From 944ad276547977fb57fb9c3fed9f58eb193605ea Mon Sep 17 00:00:00 2001 From: kevin Date: Mon, 12 Dec 2022 23:49:57 +0800 Subject: [PATCH] fix: #2684 --- core/mapping/unmarshaler.go | 1058 +++++++++++++++--------------- core/mapping/unmarshaler_test.go | 48 ++ core/mapping/utils.go | 4 +- 3 files changed, 581 insertions(+), 529 deletions(-) diff --git a/core/mapping/unmarshaler.go b/core/mapping/unmarshaler.go index e1a1c5704fe5..59977b443962 100644 --- a/core/mapping/unmarshaler.go +++ b/core/mapping/unmarshaler.go @@ -80,707 +80,711 @@ func (u *Unmarshaler) UnmarshalValuer(m Valuer, v interface{}) error { return u.unmarshalWithFullName(simpleValuer{current: m}, v, "") } -func (u *Unmarshaler) unmarshalWithFullName(m valuerWithParent, v interface{}, fullName string) error { - rv := reflect.ValueOf(v) - if err := ValidatePtr(&rv); err != nil { +func (u *Unmarshaler) fillMap(fieldType reflect.Type, value reflect.Value, mapValue interface{}) error { + if !value.CanSet() { + return errValueNotSettable + } + + fieldKeyType := fieldType.Key() + fieldElemType := fieldType.Elem() + targetValue, err := u.generateMap(fieldKeyType, fieldElemType, mapValue) + if err != nil { return err } - rte := reflect.TypeOf(v).Elem() - if rte.Kind() != reflect.Struct { - return errValueNotStruct + if !targetValue.Type().AssignableTo(value.Type()) { + return errTypeMismatch } - rve := rv.Elem() - numFields := rte.NumField() - for i := 0; i < numFields; i++ { - if err := u.processField(rte.Field(i), rve.Field(i), m, fullName); err != nil { + value.Set(targetValue) + return nil +} + +func (u *Unmarshaler) fillMapFromString(value reflect.Value, mapValue interface{}) error { + if !value.CanSet() { + return errValueNotSettable + } + + switch v := mapValue.(type) { + case fmt.Stringer: + if err := jsonx.UnmarshalFromString(v.String(), value.Addr().Interface()); err != nil { + return err + } + case string: + if err := jsonx.UnmarshalFromString(v, value.Addr().Interface()); err != nil { return err } + default: + return errUnsupportedType } return nil } -func (u *Unmarshaler) processAnonymousField(field reflect.StructField, value reflect.Value, - m valuerWithParent, fullName string) error { - key, options, err := u.parseOptionsWithContext(field, m, fullName) - if err != nil { - return err +func (u *Unmarshaler) fillSlice(fieldType reflect.Type, value reflect.Value, mapValue interface{}) error { + if !value.CanSet() { + return errValueNotSettable } - if _, hasValue := getValue(m, key); hasValue { - return fmt.Errorf("fields of %s can't be wrapped inside, because it's anonymous", key) + baseType := fieldType.Elem() + baseKind := baseType.Kind() + dereffedBaseType := Deref(baseType) + dereffedBaseKind := dereffedBaseType.Kind() + refValue := reflect.ValueOf(mapValue) + if refValue.IsNil() { + return nil } - if options.optional() { - return u.processAnonymousFieldOptional(field, value, key, m, fullName) + conv := reflect.MakeSlice(reflect.SliceOf(baseType), refValue.Len(), refValue.Cap()) + if refValue.Len() == 0 { + value.Set(conv) + return nil } - return u.processAnonymousFieldRequired(field, value, m, fullName) -} - -func (u *Unmarshaler) processAnonymousFieldOptional(field reflect.StructField, value reflect.Value, - key string, m valuerWithParent, fullName string) error { - var filled bool - var required int - var requiredFilled int - var indirectValue reflect.Value - fieldType := Deref(field.Type) - - for i := 0; i < fieldType.NumField(); i++ { - subField := fieldType.Field(i) - fieldKey, fieldOpts, err := u.parseOptionsWithContext(subField, m, fullName) - if err != nil { - return err + var valid bool + for i := 0; i < refValue.Len(); i++ { + ithValue := refValue.Index(i).Interface() + if ithValue == nil { + continue } - _, hasValue := getValue(m, fieldKey) - if hasValue { - if !filled { - filled = true - maybeNewValue(field, value) - indirectValue = reflect.Indirect(value) + valid = true + switch dereffedBaseKind { + case reflect.Struct: + target := reflect.New(dereffedBaseType) + if err := u.Unmarshal(ithValue.(map[string]interface{}), target.Interface()); err != nil { + return err } - if err = u.processField(subField, indirectValue.Field(i), m, fullName); err != nil { + + if baseKind == reflect.Ptr { + conv.Index(i).Set(target) + } else { + conv.Index(i).Set(target.Elem()) + } + case reflect.Slice: + if err := u.fillSlice(dereffedBaseType, conv.Index(i), ithValue); err != nil { return err } - } - if !fieldOpts.optional() { - required++ - if hasValue { - requiredFilled++ + default: + if err := u.fillSliceValue(conv, i, dereffedBaseKind, ithValue); err != nil { + return err } } } - if filled && required != requiredFilled { - return fmt.Errorf("%s is not fully set", key) + if valid { + value.Set(conv) } return nil } -func (u *Unmarshaler) processAnonymousFieldRequired(field reflect.StructField, value reflect.Value, - m valuerWithParent, fullName string) error { - maybeNewValue(field, value) - fieldType := Deref(field.Type) - indirectValue := reflect.Indirect(value) - - for i := 0; i < fieldType.NumField(); i++ { - if err := u.processField(fieldType.Field(i), indirectValue.Field(i), m, fullName); err != nil { +func (u *Unmarshaler) fillSliceFromString(fieldType reflect.Type, value reflect.Value, + mapValue interface{}) error { + var slice []interface{} + switch v := mapValue.(type) { + case fmt.Stringer: + if err := jsonx.UnmarshalFromString(v.String(), &slice); err != nil { return err } - } - - return nil -} - -func (u *Unmarshaler) processField(field reflect.StructField, value reflect.Value, - m valuerWithParent, fullName string) error { - if usingDifferentKeys(u.key, field) { - return nil - } - - if field.Anonymous { - return u.processAnonymousField(field, value, m, fullName) - } - - return u.processNamedField(field, value, m, fullName) -} - -func (u *Unmarshaler) processFieldNotFromString(field reflect.StructField, value reflect.Value, - vp valueWithParent, opts *fieldOptionsWithContext, fullName string) error { - fieldType := field.Type - derefedFieldType := Deref(fieldType) - typeKind := derefedFieldType.Kind() - valueKind := reflect.TypeOf(vp.value).Kind() - mapValue := vp.value - - switch { - case valueKind == reflect.Map && typeKind == reflect.Struct: - if mv, ok := mapValue.(map[string]interface{}); ok { - return u.processFieldStruct(field, value, &simpleValuer{ - current: mapValuer(mv), - parent: vp.parent, - }, fullName) - } else { - return errTypeMismatch + case string: + if err := jsonx.UnmarshalFromString(v, &slice); err != nil { + return err } - case valueKind == reflect.Map && typeKind == reflect.Map: - return u.fillMap(field, value, mapValue) - case valueKind == reflect.String && typeKind == reflect.Map: - return u.fillMapFromString(value, mapValue) - case valueKind == reflect.String && typeKind == reflect.Slice: - return u.fillSliceFromString(fieldType, value, mapValue) - case valueKind == reflect.String && derefedFieldType == durationType: - return fillDurationValue(fieldType.Kind(), value, mapValue.(string)) default: - return u.processFieldPrimitive(field, value, mapValue, opts, fullName) + return errUnsupportedType } -} - -func (u *Unmarshaler) processFieldPrimitive(field reflect.StructField, value reflect.Value, - mapValue interface{}, opts *fieldOptionsWithContext, fullName string) error { - fieldType := field.Type - typeKind := Deref(fieldType).Kind() - valueKind := reflect.TypeOf(mapValue).Kind() - switch { - case typeKind == reflect.Slice && valueKind == reflect.Slice: - return u.fillSlice(fieldType, value, mapValue) - case typeKind == reflect.Map && valueKind == reflect.Map: - return u.fillMap(field, value, mapValue) - default: - switch v := mapValue.(type) { - case json.Number: - return u.processFieldPrimitiveWithJSONNumber(field, value, v, opts, fullName) - default: - if typeKind == valueKind { - if err := validateValueInOptions(mapValue, opts.options()); err != nil { - return err - } + baseFieldType := Deref(fieldType.Elem()) + baseFieldKind := baseFieldType.Kind() + conv := reflect.MakeSlice(reflect.SliceOf(baseFieldType), len(slice), cap(slice)) - return fillWithSameType(field, value, mapValue, opts) - } + for i := 0; i < len(slice); i++ { + if err := u.fillSliceValue(conv, i, baseFieldKind, slice[i]); err != nil { + return err } } - return newTypeMismatchError(fullName) + value.Set(conv) + return nil } -func (u *Unmarshaler) processFieldPrimitiveWithJSONNumber(field reflect.StructField, value reflect.Value, - v json.Number, opts *fieldOptionsWithContext, fullName string) error { - fieldType := field.Type - fieldKind := fieldType.Kind() - typeKind := Deref(fieldType).Kind() - - if err := validateJsonNumberRange(v, opts); err != nil { - return err - } - - if err := validateValueInOptions(v, opts.options()); err != nil { - return err - } - - if fieldKind == reflect.Ptr { - value = value.Elem() - } +func (u *Unmarshaler) fillSliceValue(slice reflect.Value, index int, + baseKind reflect.Kind, value interface{}) error { + ithVal := slice.Index(index) + switch v := value.(type) { + case fmt.Stringer: + return setValue(baseKind, ithVal, v.String()) + case string: + return setValue(baseKind, ithVal, v) + case map[string]interface{}: + return u.fillMap(ithVal.Type(), ithVal, value) + default: + // don't need to consider the difference between int, int8, int16, int32, int64, + // uint, uint8, uint16, uint32, uint64, because they're handled as json.Number. + if ithVal.Kind() == reflect.Ptr { + baseType := Deref(ithVal.Type()) + if !reflect.TypeOf(value).AssignableTo(baseType) { + return errTypeMismatch + } - switch typeKind { - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - iValue, err := v.Int64() - if err != nil { - return err + target := reflect.New(baseType).Elem() + target.Set(reflect.ValueOf(value)) + ithVal.Set(target.Addr()) + return nil } - value.SetInt(iValue) - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - iValue, err := v.Int64() - if err != nil { - return err + if !reflect.TypeOf(value).AssignableTo(ithVal.Type()) { + return errTypeMismatch } - if iValue < 0 { - return fmt.Errorf("unmarshal %q with bad value %q", fullName, v.String()) - } + ithVal.Set(reflect.ValueOf(value)) + return nil + } +} - value.SetUint(uint64(iValue)) - case reflect.Float32, reflect.Float64: - fValue, err := v.Float64() - if err != nil { +func (u *Unmarshaler) fillSliceWithDefault(derefedType reflect.Type, value reflect.Value, + defaultValue string) error { + baseFieldType := Deref(derefedType.Elem()) + baseFieldKind := baseFieldType.Kind() + defaultCacheLock.Lock() + slice, ok := defaultCache[defaultValue] + defaultCacheLock.Unlock() + if !ok { + if baseFieldKind == reflect.String { + slice = parseGroupedSegments(defaultValue) + } else if err := jsonx.UnmarshalFromString(defaultValue, &slice); err != nil { return err } - value.SetFloat(fValue) - default: - return newTypeMismatchError(fullName) + defaultCacheLock.Lock() + defaultCache[defaultValue] = slice + defaultCacheLock.Unlock() } - return nil + return u.fillSlice(derefedType, value, slice) } -func (u *Unmarshaler) processFieldStruct(field reflect.StructField, value reflect.Value, - m valuerWithParent, fullName string) error { - if field.Type.Kind() == reflect.Ptr { - baseType := Deref(field.Type) - target := reflect.New(baseType).Elem() - if err := u.unmarshalWithFullName(m, target.Addr().Interface(), fullName); err != nil { - return err - } - - value.Set(target.Addr()) - } else if err := u.unmarshalWithFullName(m, value.Addr().Interface(), fullName); err != nil { - return err +func (u *Unmarshaler) generateMap(keyType, elemType reflect.Type, mapValue interface{}) (reflect.Value, error) { + mapType := reflect.MapOf(keyType, elemType) + valueType := reflect.TypeOf(mapValue) + if mapType == valueType { + return reflect.ValueOf(mapValue), nil } - return nil -} + refValue := reflect.ValueOf(mapValue) + targetValue := reflect.MakeMapWithSize(mapType, refValue.Len()) + fieldElemKind := elemType.Kind() + dereffedElemType := Deref(elemType) + dereffedElemKind := dereffedElemType.Kind() -func (u *Unmarshaler) processFieldTextUnmarshaler(field reflect.StructField, value reflect.Value, - mapValue interface{}) (bool, error) { - var tval encoding.TextUnmarshaler - var ok bool + for _, key := range refValue.MapKeys() { + keythValue := refValue.MapIndex(key) + keythData := keythValue.Interface() - if field.Type.Kind() == reflect.Ptr { - tval, ok = value.Interface().(encoding.TextUnmarshaler) - } else { - tval, ok = value.Addr().Interface().(encoding.TextUnmarshaler) - } - if ok { - switch mv := mapValue.(type) { - case string: - return true, tval.UnmarshalText([]byte(mv)) - case []byte: - return true, tval.UnmarshalText(mv) - } - } + switch dereffedElemKind { + case reflect.Slice: + target := reflect.New(dereffedElemType) + if err := u.fillSlice(elemType, target.Elem(), keythData); err != nil { + return emptyValue, err + } - return false, nil -} + targetValue.SetMapIndex(key, target.Elem()) + case reflect.Struct: + keythMap, ok := keythData.(map[string]interface{}) + if !ok { + return emptyValue, errTypeMismatch + } -func (u *Unmarshaler) processFieldWithEnvValue(field reflect.StructField, value reflect.Value, - envVal string, opts *fieldOptionsWithContext, fullName string) error { - if err := validateValueInOptions(envVal, opts.options()); err != nil { - return err - } + target := reflect.New(dereffedElemType) + if err := u.Unmarshal(keythMap, target.Interface()); err != nil { + return emptyValue, err + } - fieldKind := field.Type.Kind() - switch fieldKind { - case reflect.Bool: - val, err := strconv.ParseBool(envVal) - if err != nil { - return fmt.Errorf("unmarshal field %q with environment variable, %w", fullName, err) - } + if fieldElemKind == reflect.Ptr { + targetValue.SetMapIndex(key, target) + } else { + targetValue.SetMapIndex(key, target.Elem()) + } + case reflect.Map: + keythMap, ok := keythData.(map[string]interface{}) + if !ok { + return emptyValue, errTypeMismatch + } - value.SetBool(val) - return nil - case durationType.Kind(): - if err := fillDurationValue(fieldKind, value, envVal); err != nil { - return fmt.Errorf("unmarshal field %q with environment variable, %w", fullName, err) - } + innerValue, err := u.generateMap(elemType.Key(), elemType.Elem(), keythMap) + if err != nil { + return emptyValue, err + } - return nil - case reflect.String: - value.SetString(envVal) - return nil - default: - return u.processFieldPrimitiveWithJSONNumber(field, value, json.Number(envVal), opts, fullName) - } -} + targetValue.SetMapIndex(key, innerValue) + default: + switch v := keythData.(type) { + case bool: + if dereffedElemKind != reflect.Bool { + return emptyValue, errTypeMismatch + } -func (u *Unmarshaler) processNamedField(field reflect.StructField, value reflect.Value, - m valuerWithParent, fullName string) error { - key, opts, err := u.parseOptionsWithContext(field, m, fullName) - if err != nil { - return err - } + targetValue.SetMapIndex(key, reflect.ValueOf(v)) + case string: + if dereffedElemKind != reflect.String { + return emptyValue, errTypeMismatch + } - fullName = join(fullName, key) - if opts != nil && len(opts.EnvVar) > 0 { - envVal := proc.Env(opts.EnvVar) - if len(envVal) > 0 { - return u.processFieldWithEnvValue(field, value, envVal, opts, fullName) + targetValue.SetMapIndex(key, reflect.ValueOf(v)) + case json.Number: + target := reflect.New(dereffedElemType) + if err := setValue(dereffedElemKind, target.Elem(), v.String()); err != nil { + return emptyValue, err + } + + targetValue.SetMapIndex(key, target.Elem()) + default: + if dereffedElemKind != keythValue.Kind() { + return emptyValue, errTypeMismatch + } + + targetValue.SetMapIndex(key, keythValue) + } } } - canonicalKey := key - if u.opts.canonicalKey != nil { - canonicalKey = u.opts.canonicalKey(key) + return targetValue, nil +} + +func (u *Unmarshaler) parseOptionsWithContext(field reflect.StructField, m Valuer, fullName string) ( + string, *fieldOptionsWithContext, error) { + key, options, err := parseKeyAndOptions(u.key, field) + if err != nil { + return "", nil, err + } else if options == nil { + return key, nil, nil } - valuer := createValuer(m, opts) - mapValue, hasValue := getValue(valuer, canonicalKey) - if !hasValue { - return u.processNamedFieldWithoutValue(field, value, opts, fullName) + optsWithContext, err := options.toOptionsWithContext(key, m, fullName) + if err != nil { + return "", nil, err } - return u.processNamedFieldWithValue(field, value, valueWithParent{ - value: mapValue, - parent: valuer, - }, key, opts, fullName) + return key, optsWithContext, nil } -func (u *Unmarshaler) processNamedFieldWithValue(field reflect.StructField, value reflect.Value, - vp valueWithParent, key string, opts *fieldOptionsWithContext, fullName string) error { - mapValue := vp.value - if mapValue == nil { - if opts.optional() { - return nil - } - - return fmt.Errorf("field %s mustn't be nil", key) +func (u *Unmarshaler) processAnonymousField(field reflect.StructField, value reflect.Value, + m valuerWithParent, fullName string) error { + key, options, err := u.parseOptionsWithContext(field, m, fullName) + if err != nil { + return err } - if !value.CanSet() { - return fmt.Errorf("field %s is not settable", key) + if _, hasValue := getValue(m, key); hasValue { + return fmt.Errorf("fields of %s can't be wrapped inside, because it's anonymous", key) } - maybeNewValue(field, value) - - if yes, err := u.processFieldTextUnmarshaler(field, value, mapValue); yes { - return err + if options.optional() { + return u.processAnonymousFieldOptional(field.Type, value, key, m, fullName) } - fieldKind := Deref(field.Type).Kind() - switch fieldKind { - case reflect.Array, reflect.Map, reflect.Slice, reflect.Struct: - return u.processFieldNotFromString(field, value, vp, opts, fullName) - default: - if u.opts.fromString || opts.fromString() { - valueKind := reflect.TypeOf(mapValue).Kind() - if valueKind != reflect.String { - return fmt.Errorf("error: the value in map is not string, but %s", valueKind) - } + return u.processAnonymousFieldRequired(field.Type, value, m, fullName) +} - options := opts.options() - if len(options) > 0 { - if !stringx.Contains(options, mapValue.(string)) { - return fmt.Errorf(`error: value "%s" for field "%s" is not defined in options "%v"`, - mapValue, key, options) - } - } +func (u *Unmarshaler) processAnonymousFieldOptional(fieldType reflect.Type, value reflect.Value, + key string, m valuerWithParent, fullName string) error { + var filled bool + var required int + var requiredFilled int + var indirectValue reflect.Value + derefedFieldType := Deref(fieldType) - return fillPrimitive(field.Type, value, mapValue, opts, fullName) + for i := 0; i < derefedFieldType.NumField(); i++ { + subField := derefedFieldType.Field(i) + fieldKey, fieldOpts, err := u.parseOptionsWithContext(subField, m, fullName) + if err != nil { + return err } - return u.processFieldNotFromString(field, value, vp, opts, fullName) - } -} - -func (u *Unmarshaler) processNamedFieldWithoutValue(field reflect.StructField, value reflect.Value, - opts *fieldOptionsWithContext, fullName string) error { - derefedType := Deref(field.Type) - fieldKind := derefedType.Kind() - if defaultValue, ok := opts.getDefault(); ok { - if field.Type.Kind() == reflect.Ptr { - maybeNewValue(field, value) - value = value.Elem() + _, hasValue := getValue(m, fieldKey) + if hasValue { + if !filled { + filled = true + maybeNewValue(fieldType, value) + indirectValue = reflect.Indirect(value) + } + if err = u.processField(subField, indirectValue.Field(i), m, fullName); err != nil { + return err + } } - if derefedType == durationType { - return fillDurationValue(fieldKind, value, defaultValue) + if !fieldOpts.optional() { + required++ + if hasValue { + requiredFilled++ + } } + } - switch fieldKind { - case reflect.Array, reflect.Slice: - return u.fillSliceWithDefault(derefedType, value, defaultValue) - default: - return setValue(fieldKind, value, defaultValue) - } + if filled && required != requiredFilled { + return fmt.Errorf("%s is not fully set", key) } - switch fieldKind { - case reflect.Array, reflect.Map, reflect.Slice: - if !opts.optional() { - return u.processFieldNotFromString(field, value, valueWithParent{ - value: emptyMap, - }, opts, fullName) - } - case reflect.Struct: - if !opts.optional() { - required, err := structValueRequired(u.key, derefedType) - if err != nil { - return err - } + return nil +} - if required { - return fmt.Errorf("%q is not set", fullName) - } +func (u *Unmarshaler) processAnonymousFieldRequired(fieldType reflect.Type, value reflect.Value, + m valuerWithParent, fullName string) error { + maybeNewValue(fieldType, value) + derefedFieldType := Deref(fieldType) + indirectValue := reflect.Indirect(value) - return u.processFieldNotFromString(field, value, valueWithParent{ - value: emptyMap, - }, opts, fullName) - } - default: - if !opts.optional() { - return newInitError(fullName) + for i := 0; i < derefedFieldType.NumField(); i++ { + if err := u.processField(derefedFieldType.Field(i), indirectValue.Field(i), m, fullName); err != nil { + return err } } return nil } -func (u *Unmarshaler) fillMap(field reflect.StructField, value reflect.Value, mapValue interface{}) error { - if !value.CanSet() { - return errValueNotSettable +func (u *Unmarshaler) processField(field reflect.StructField, value reflect.Value, + m valuerWithParent, fullName string) error { + if usingDifferentKeys(u.key, field) { + return nil } - fieldKeyType := field.Type.Key() - fieldElemType := field.Type.Elem() - targetValue, err := u.generateMap(fieldKeyType, fieldElemType, mapValue) - if err != nil { - return err + if field.Anonymous { + return u.processAnonymousField(field, value, m, fullName) } - value.Set(targetValue) - return nil + return u.processNamedField(field, value, m, fullName) } -func (u *Unmarshaler) fillMapFromString(value reflect.Value, mapValue interface{}) error { - if !value.CanSet() { - return errValueNotSettable - } +func (u *Unmarshaler) processFieldNotFromString(fieldType reflect.Type, value reflect.Value, + vp valueWithParent, opts *fieldOptionsWithContext, fullName string) error { + derefedFieldType := Deref(fieldType) + typeKind := derefedFieldType.Kind() + valueKind := reflect.TypeOf(vp.value).Kind() + mapValue := vp.value - switch v := mapValue.(type) { - case fmt.Stringer: - if err := jsonx.UnmarshalFromString(v.String(), value.Addr().Interface()); err != nil { - return err - } - case string: - if err := jsonx.UnmarshalFromString(v, value.Addr().Interface()); err != nil { - return err + switch { + case valueKind == reflect.Map && typeKind == reflect.Struct: + mv, ok := mapValue.(map[string]interface{}) + if !ok { + return errTypeMismatch } + + return u.processFieldStruct(fieldType, value, &simpleValuer{ + current: mapValuer(mv), + parent: vp.parent, + }, fullName) + case valueKind == reflect.Map && typeKind == reflect.Map: + return u.fillMap(fieldType, value, mapValue) + case valueKind == reflect.String && typeKind == reflect.Map: + return u.fillMapFromString(value, mapValue) + case valueKind == reflect.String && typeKind == reflect.Slice: + return u.fillSliceFromString(fieldType, value, mapValue) + case valueKind == reflect.String && derefedFieldType == durationType: + return fillDurationValue(fieldType.Kind(), value, mapValue.(string)) default: - return errUnsupportedType + return u.processFieldPrimitive(fieldType, value, mapValue, opts, fullName) } +} - return nil +func (u *Unmarshaler) processFieldPrimitive(fieldType reflect.Type, value reflect.Value, + mapValue interface{}, opts *fieldOptionsWithContext, fullName string) error { + typeKind := Deref(fieldType).Kind() + valueKind := reflect.TypeOf(mapValue).Kind() + + switch { + case typeKind == reflect.Slice && valueKind == reflect.Slice: + return u.fillSlice(fieldType, value, mapValue) + case typeKind == reflect.Map && valueKind == reflect.Map: + return u.fillMap(fieldType, value, mapValue) + default: + switch v := mapValue.(type) { + case json.Number: + return u.processFieldPrimitiveWithJSONNumber(fieldType, value, v, opts, fullName) + default: + if typeKind == valueKind { + if err := validateValueInOptions(mapValue, opts.options()); err != nil { + return err + } + + return fillWithSameType(fieldType, value, mapValue, opts) + } + } + } + + return newTypeMismatchError(fullName) } -func (u *Unmarshaler) fillSlice(fieldType reflect.Type, value reflect.Value, mapValue interface{}) error { - if !value.CanSet() { - return errValueNotSettable +func (u *Unmarshaler) processFieldPrimitiveWithJSONNumber(fieldType reflect.Type, value reflect.Value, + v json.Number, opts *fieldOptionsWithContext, fullName string) error { + fieldKind := fieldType.Kind() + typeKind := Deref(fieldType).Kind() + + if err := validateJsonNumberRange(v, opts); err != nil { + return err } - baseType := fieldType.Elem() - baseKind := baseType.Kind() - dereffedBaseType := Deref(baseType) - dereffedBaseKind := dereffedBaseType.Kind() - refValue := reflect.ValueOf(mapValue) - if refValue.IsNil() { - return nil + if err := validateValueInOptions(v, opts.options()); err != nil { + return err } - conv := reflect.MakeSlice(reflect.SliceOf(baseType), refValue.Len(), refValue.Cap()) - if refValue.Len() == 0 { - value.Set(conv) - return nil + if fieldKind == reflect.Ptr { + value = value.Elem() } - var valid bool - for i := 0; i < refValue.Len(); i++ { - ithValue := refValue.Index(i).Interface() - if ithValue == nil { - continue + switch typeKind { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + iValue, err := v.Int64() + if err != nil { + return err } - valid = true - switch dereffedBaseKind { - case reflect.Struct: - target := reflect.New(dereffedBaseType) - if err := u.Unmarshal(ithValue.(map[string]interface{}), target.Interface()); err != nil { - return err - } + value.SetInt(iValue) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + iValue, err := v.Int64() + if err != nil { + return err + } - if baseKind == reflect.Ptr { - conv.Index(i).Set(target) - } else { - conv.Index(i).Set(target.Elem()) - } - case reflect.Slice: - if err := u.fillSlice(dereffedBaseType, conv.Index(i), ithValue); err != nil { - return err - } - default: - if err := u.fillSliceValue(conv, i, dereffedBaseKind, ithValue); err != nil { - return err - } + if iValue < 0 { + return fmt.Errorf("unmarshal %q with bad value %q", fullName, v.String()) } - } - if valid { - value.Set(conv) + value.SetUint(uint64(iValue)) + case reflect.Float32, reflect.Float64: + fValue, err := v.Float64() + if err != nil { + return err + } + + value.SetFloat(fValue) + default: + return newTypeMismatchError(fullName) } return nil } -func (u *Unmarshaler) fillSliceFromString(fieldType reflect.Type, value reflect.Value, - mapValue interface{}) error { - var slice []interface{} - switch v := mapValue.(type) { - case fmt.Stringer: - if err := jsonx.UnmarshalFromString(v.String(), &slice); err != nil { - return err - } - case string: - if err := jsonx.UnmarshalFromString(v, &slice); err != nil { +func (u *Unmarshaler) processFieldStruct(fieldType reflect.Type, value reflect.Value, + m valuerWithParent, fullName string) error { + if fieldType.Kind() == reflect.Ptr { + baseType := Deref(fieldType) + target := reflect.New(baseType).Elem() + if err := u.unmarshalWithFullName(m, target.Addr().Interface(), fullName); err != nil { return err } - default: - return errUnsupportedType + + value.Set(target.Addr()) + } else if err := u.unmarshalWithFullName(m, value.Addr().Interface(), fullName); err != nil { + return err } - baseFieldType := Deref(fieldType.Elem()) - baseFieldKind := baseFieldType.Kind() - conv := reflect.MakeSlice(reflect.SliceOf(baseFieldType), len(slice), cap(slice)) + return nil +} - for i := 0; i < len(slice); i++ { - if err := u.fillSliceValue(conv, i, baseFieldKind, slice[i]); err != nil { - return err +func (u *Unmarshaler) processFieldTextUnmarshaler(fieldType reflect.Type, value reflect.Value, + mapValue interface{}) (bool, error) { + var tval encoding.TextUnmarshaler + var ok bool + + if fieldType.Kind() == reflect.Ptr { + tval, ok = value.Interface().(encoding.TextUnmarshaler) + } else { + tval, ok = value.Addr().Interface().(encoding.TextUnmarshaler) + } + if ok { + switch mv := mapValue.(type) { + case string: + return true, tval.UnmarshalText([]byte(mv)) + case []byte: + return true, tval.UnmarshalText(mv) } } - value.Set(conv) - return nil + return false, nil } -func (u *Unmarshaler) fillSliceValue(slice reflect.Value, index int, - baseKind reflect.Kind, value interface{}) error { - ithVal := slice.Index(index) - switch v := value.(type) { - case fmt.Stringer: - return setValue(baseKind, ithVal, v.String()) - case string: - return setValue(baseKind, ithVal, v) - default: - // don't need to consider the difference between int, int8, int16, int32, int64, - // uint, uint8, uint16, uint32, uint64, because they're handled as json.Number. - if ithVal.Kind() == reflect.Ptr { - baseType := Deref(ithVal.Type()) - if baseType.Kind() != reflect.TypeOf(value).Kind() { - return errTypeMismatch - } +func (u *Unmarshaler) processFieldWithEnvValue(fieldType reflect.Type, value reflect.Value, + envVal string, opts *fieldOptionsWithContext, fullName string) error { + if err := validateValueInOptions(envVal, opts.options()); err != nil { + return err + } - target := reflect.New(baseType).Elem() - target.Set(reflect.ValueOf(value)) - ithVal.Set(target.Addr()) - return nil + fieldKind := fieldType.Kind() + switch fieldKind { + case reflect.Bool: + val, err := strconv.ParseBool(envVal) + if err != nil { + return fmt.Errorf("unmarshal field %q with environment variable, %w", fullName, err) } - if ithVal.Kind() != reflect.TypeOf(value).Kind() { - return errTypeMismatch + value.SetBool(val) + return nil + case durationType.Kind(): + if err := fillDurationValue(fieldKind, value, envVal); err != nil { + return fmt.Errorf("unmarshal field %q with environment variable, %w", fullName, err) } - ithVal.Set(reflect.ValueOf(value)) return nil + case reflect.String: + value.SetString(envVal) + return nil + default: + return u.processFieldPrimitiveWithJSONNumber(fieldType, value, json.Number(envVal), opts, fullName) } } -func (u *Unmarshaler) fillSliceWithDefault(derefedType reflect.Type, value reflect.Value, - defaultValue string) error { - baseFieldType := Deref(derefedType.Elem()) - baseFieldKind := baseFieldType.Kind() - defaultCacheLock.Lock() - slice, ok := defaultCache[defaultValue] - defaultCacheLock.Unlock() - if !ok { - if baseFieldKind == reflect.String { - slice = parseGroupedSegments(defaultValue) - } else if err := jsonx.UnmarshalFromString(defaultValue, &slice); err != nil { - return err +func (u *Unmarshaler) processNamedField(field reflect.StructField, value reflect.Value, + m valuerWithParent, fullName string) error { + key, opts, err := u.parseOptionsWithContext(field, m, fullName) + if err != nil { + return err + } + + fullName = join(fullName, key) + if opts != nil && len(opts.EnvVar) > 0 { + envVal := proc.Env(opts.EnvVar) + if len(envVal) > 0 { + return u.processFieldWithEnvValue(field.Type, value, envVal, opts, fullName) } + } - defaultCacheLock.Lock() - defaultCache[defaultValue] = slice - defaultCacheLock.Unlock() + canonicalKey := key + if u.opts.canonicalKey != nil { + canonicalKey = u.opts.canonicalKey(key) } - return u.fillSlice(derefedType, value, slice) + valuer := createValuer(m, opts) + mapValue, hasValue := getValue(valuer, canonicalKey) + if !hasValue { + return u.processNamedFieldWithoutValue(field.Type, value, opts, fullName) + } + + return u.processNamedFieldWithValue(field.Type, value, valueWithParent{ + value: mapValue, + parent: valuer, + }, key, opts, fullName) } -func (u *Unmarshaler) generateMap(keyType, elemType reflect.Type, mapValue interface{}) (reflect.Value, error) { - mapType := reflect.MapOf(keyType, elemType) - valueType := reflect.TypeOf(mapValue) - if mapType == valueType { - return reflect.ValueOf(mapValue), nil - } +func (u *Unmarshaler) processNamedFieldWithValue(fieldType reflect.Type, value reflect.Value, + vp valueWithParent, key string, opts *fieldOptionsWithContext, fullName string) error { + mapValue := vp.value + if mapValue == nil { + if opts.optional() { + return nil + } - refValue := reflect.ValueOf(mapValue) - targetValue := reflect.MakeMapWithSize(mapType, refValue.Len()) - fieldElemKind := elemType.Kind() - dereffedElemType := Deref(elemType) - dereffedElemKind := dereffedElemType.Kind() + return fmt.Errorf("field %s mustn't be nil", key) + } - for _, key := range refValue.MapKeys() { - keythValue := refValue.MapIndex(key) - keythData := keythValue.Interface() + if !value.CanSet() { + return fmt.Errorf("field %s is not settable", key) + } - switch dereffedElemKind { - case reflect.Slice: - target := reflect.New(dereffedElemType) - if err := u.fillSlice(elemType, target.Elem(), keythData); err != nil { - return emptyValue, err - } + maybeNewValue(fieldType, value) - targetValue.SetMapIndex(key, target.Elem()) - case reflect.Struct: - keythMap, ok := keythData.(map[string]interface{}) - if !ok { - return emptyValue, errTypeMismatch - } + if yes, err := u.processFieldTextUnmarshaler(fieldType, value, mapValue); yes { + return err + } - target := reflect.New(dereffedElemType) - if err := u.Unmarshal(keythMap, target.Interface()); err != nil { - return emptyValue, err + fieldKind := Deref(fieldType).Kind() + switch fieldKind { + case reflect.Array, reflect.Map, reflect.Slice, reflect.Struct: + return u.processFieldNotFromString(fieldType, value, vp, opts, fullName) + default: + if u.opts.fromString || opts.fromString() { + valueKind := reflect.TypeOf(mapValue).Kind() + if valueKind != reflect.String { + return fmt.Errorf("error: the value in map is not string, but %s", valueKind) } - if fieldElemKind == reflect.Ptr { - targetValue.SetMapIndex(key, target) - } else { - targetValue.SetMapIndex(key, target.Elem()) - } - case reflect.Map: - keythMap, ok := keythData.(map[string]interface{}) - if !ok { - return emptyValue, errTypeMismatch + options := opts.options() + if len(options) > 0 { + if !stringx.Contains(options, mapValue.(string)) { + return fmt.Errorf(`error: value "%s" for field "%s" is not defined in options "%v"`, + mapValue, key, options) + } } - innerValue, err := u.generateMap(elemType.Key(), elemType.Elem(), keythMap) - if err != nil { - return emptyValue, err - } + return fillPrimitive(fieldType, value, mapValue, opts, fullName) + } - targetValue.SetMapIndex(key, innerValue) - default: - switch v := keythData.(type) { - case bool: - if dereffedElemKind != reflect.Bool { - return emptyValue, errTypeMismatch - } + return u.processFieldNotFromString(fieldType, value, vp, opts, fullName) + } +} - targetValue.SetMapIndex(key, reflect.ValueOf(v)) - case string: - if dereffedElemKind != reflect.String { - return emptyValue, errTypeMismatch - } +func (u *Unmarshaler) processNamedFieldWithoutValue(fieldType reflect.Type, value reflect.Value, + opts *fieldOptionsWithContext, fullName string) error { + derefedType := Deref(fieldType) + fieldKind := derefedType.Kind() + if defaultValue, ok := opts.getDefault(); ok { + if fieldType.Kind() == reflect.Ptr { + maybeNewValue(fieldType, value) + value = value.Elem() + } + if derefedType == durationType { + return fillDurationValue(fieldKind, value, defaultValue) + } - targetValue.SetMapIndex(key, reflect.ValueOf(v)) - case json.Number: - target := reflect.New(dereffedElemType) - if err := setValue(dereffedElemKind, target.Elem(), v.String()); err != nil { - return emptyValue, err - } + switch fieldKind { + case reflect.Array, reflect.Slice: + return u.fillSliceWithDefault(derefedType, value, defaultValue) + default: + return setValue(fieldKind, value, defaultValue) + } + } - targetValue.SetMapIndex(key, target.Elem()) - default: - if dereffedElemKind != keythValue.Kind() { - return emptyValue, errTypeMismatch - } + switch fieldKind { + case reflect.Array, reflect.Map, reflect.Slice: + if !opts.optional() { + return u.processFieldNotFromString(fieldType, value, valueWithParent{ + value: emptyMap, + }, opts, fullName) + } + case reflect.Struct: + if !opts.optional() { + required, err := structValueRequired(u.key, derefedType) + if err != nil { + return err + } - targetValue.SetMapIndex(key, keythValue) + if required { + return fmt.Errorf("%q is not set", fullName) } + + return u.processFieldNotFromString(fieldType, value, valueWithParent{ + value: emptyMap, + }, opts, fullName) + } + default: + if !opts.optional() { + return newInitError(fullName) } } - return targetValue, nil + return nil } -func (u *Unmarshaler) parseOptionsWithContext(field reflect.StructField, m Valuer, fullName string) ( - string, *fieldOptionsWithContext, error) { - key, options, err := parseKeyAndOptions(u.key, field) - if err != nil { - return "", nil, err - } else if options == nil { - return key, nil, nil +func (u *Unmarshaler) unmarshalWithFullName(m valuerWithParent, v interface{}, fullName string) error { + rv := reflect.ValueOf(v) + if err := ValidatePtr(&rv); err != nil { + return err } - optsWithContext, err := options.toOptionsWithContext(key, m, fullName) - if err != nil { - return "", nil, err + rte := reflect.TypeOf(v).Elem() + if rte.Kind() != reflect.Struct { + return errValueNotStruct } - return key, optsWithContext, nil + rve := rv.Elem() + numFields := rte.NumField() + for i := 0; i < numFields; i++ { + if err := u.processField(rte.Field(i), rve.Field(i), m, fullName); err != nil { + return err + } + } + + return nil } // WithStringValues customizes an Unmarshaler with number values from strings. @@ -855,7 +859,7 @@ func fillPrimitive(fieldType reflect.Type, value reflect.Value, mapValue interfa } } -func fillWithSameType(field reflect.StructField, value reflect.Value, mapValue interface{}, +func fillWithSameType(fieldType reflect.Type, value reflect.Value, mapValue interface{}, opts *fieldOptionsWithContext) error { if !value.CanSet() { return errValueNotSettable @@ -865,13 +869,13 @@ func fillWithSameType(field reflect.StructField, value reflect.Value, mapValue i return err } - if field.Type.Kind() == reflect.Ptr { - baseType := Deref(field.Type) + if fieldType.Kind() == reflect.Ptr { + baseType := Deref(fieldType) target := reflect.New(baseType).Elem() setSameKindValue(baseType, target, mapValue) value.Set(target.Addr()) } else { - setSameKindValue(field.Type, value, mapValue) + setSameKindValue(fieldType, value, mapValue) } return nil diff --git a/core/mapping/unmarshaler_test.go b/core/mapping/unmarshaler_test.go index 96f841fb3349..356cbb3ac137 100644 --- a/core/mapping/unmarshaler_test.go +++ b/core/mapping/unmarshaler_test.go @@ -3628,6 +3628,54 @@ func TestUnmarshalJsonReaderWithMismatchTypeBoolMap(t *testing.T) { }, &req)) } +func TestUnmarshalJsonBytesSliceOfMaps(t *testing.T) { + input := []byte(`{ + "order_id": "1234567", + "refund_reason": { + "reason_code": [ + 123, + 234 + ], + "desc": "not wanted", + "show_reason": [ + { + "123": "not enough", + "234": "closed" + } + ] + }, + "product_detail": { + "product_id": "123", + "sku_id": "123", + "name": "cake", + "actual_amount": 100 + } +}`) + + type ( + RefundReasonData struct { + ReasonCode []int `json:"reason_code"` + Desc string `json:"desc"` + ShowReason []map[string]string `json:"show_reason"` + } + + ProductDetailData struct { + ProductId string `json:"product_id"` + SkuId string `json:"sku_id"` + Name string `json:"name"` + ActualAmount int `json:"actual_amount"` + } + OrderApplyRefundReq struct { + OrderId string `json:"order_id"` + RefundReason RefundReasonData `json:"refund_reason,optional"` + ProductDetail ProductDetailData `json:"product_detail,optional"` + } + ) + + var req OrderApplyRefundReq + assert.NoError(t, UnmarshalJsonBytes(input, &req)) +} + func BenchmarkDefaultValue(b *testing.B) { for i := 0; i < b.N; i++ { var a struct { diff --git a/core/mapping/utils.go b/core/mapping/utils.go index d7c9becaac86..5425961460fb 100644 --- a/core/mapping/utils.go +++ b/core/mapping/utils.go @@ -210,8 +210,8 @@ func isRightInclude(b byte) (bool, error) { } } -func maybeNewValue(field reflect.StructField, value reflect.Value) { - if field.Type.Kind() == reflect.Ptr && value.IsNil() { +func maybeNewValue(fieldType reflect.Type, value reflect.Value) { + if fieldType.Kind() == reflect.Ptr && value.IsNil() { value.Set(reflect.New(value.Type().Elem())) } }