feat: accept camelcase for config keys (#2651)

* feat: accept camelcase for config keys

* chore: refactor

* chore: refactor

* chore: add more tests

* chore: refactor

* fix: map elements of array

core/conf/config.go

@@ -7,9 +7,13 @@ import (
 	"path"
 	"strings"
 
+	"github.com/zeromicro/go-zero/core/jsonx"
 	"github.com/zeromicro/go-zero/core/mapping"
+	"github.com/zeromicro/go-zero/internal/encoding"
 )
 
+const distanceBetweenUpperAndLower = 32
+
 var loaders = map[string]func([]byte, interface{}) error{
 	".json": LoadFromJsonBytes,
 	".toml": LoadFromTomlBytes,
@@ -49,7 +53,12 @@ func LoadConfig(file string, v interface{}, opts ...Option) error {
 
 // LoadFromJsonBytes loads config into v from content json bytes.
 func LoadFromJsonBytes(content []byte, v interface{}) error {
-	return mapping.UnmarshalJsonBytes(content, v)
+	var m map[string]interface{}
+	if err := jsonx.Unmarshal(content, &m); err != nil {
+		return err
+	}
+
+	return mapping.UnmarshalJsonMap(toCamelCaseKeyMap(m), v, mapping.WithCanonicalKeyFunc(toCamelCase))
 }
 
 // LoadConfigFromJsonBytes loads config into v from content json bytes.
@@ -60,12 +69,22 @@ func LoadConfigFromJsonBytes(content []byte, v interface{}) error {
 
 // LoadFromTomlBytes loads config into v from content toml bytes.
 func LoadFromTomlBytes(content []byte, v interface{}) error {
-	return mapping.UnmarshalTomlBytes(content, v)
+	b, err := encoding.TomlToJson(content)
+	if err != nil {
+		return err
+	}
+
+	return LoadFromJsonBytes(b, v)
 }
 
 // LoadFromYamlBytes loads config into v from content yaml bytes.
 func LoadFromYamlBytes(content []byte, v interface{}) error {
-	return mapping.UnmarshalYamlBytes(content, v)
+	b, err := encoding.YamlToJson(content)
+	if err != nil {
+		return err
+	}
+
+	return LoadFromJsonBytes(b, v)
 }
 
 // LoadConfigFromYamlBytes loads config into v from content yaml bytes.
@@ -80,3 +99,66 @@ func MustLoad(path string, v interface{}, opts ...Option) {
 		log.Fatalf("error: config file %s, %s", path, err.Error())
 	}
 }
+
+func toCamelCase(s string) string {
+	var buf strings.Builder
+	buf.Grow(len(s))
+	var capNext bool
+	boundary := true
+	for _, v := range s {
+		isCap := v >= 'A' && v <= 'Z'
+		isLow := v >= 'a' && v <= 'z'
+		if boundary && (isCap || isLow) {
+			if capNext {
+				if isLow {
+					v -= distanceBetweenUpperAndLower
+				}
+			} else {
+				if isCap {
+					v += distanceBetweenUpperAndLower
+				}
+			}
+			boundary = false
+		}
+		if isCap || isLow {
+			buf.WriteRune(v)
+			capNext = false
+		} else if v == ' ' || v == '\t' {
+			buf.WriteRune(v)
+			capNext = false
+			boundary = true
+		} else if v == '_' {
+			capNext = true
+			boundary = true
+		} else {
+			buf.WriteRune(v)
+			capNext = true
+		}
+	}
+
+	return buf.String()
+}
+
+func toCamelCaseInterface(v interface{}) interface{} {
+	switch vv := v.(type) {
+	case map[string]interface{}:
+		return toCamelCaseKeyMap(vv)
+	case []interface{}:
+		var arr []interface{}
+		for _, vvv := range vv {
+			arr = append(arr, toCamelCaseInterface(vvv))
+		}
+		return arr
+	default:
+		return v
+	}
+}
+
+func toCamelCaseKeyMap(m map[string]interface{}) map[string]interface{} {
+	res := make(map[string]interface{})
+	for k, v := range m {
+		res[toCamelCase(k)] = toCamelCaseInterface(v)
+	}
+
+	return res
+}

core/conf/config_test.go

@@ -56,6 +56,22 @@ func TestConfigJson(t *testing.T) {
 	}
 }
 
+func TestLoadFromJsonBytesArray(t *testing.T) {
+	input := []byte(`{"users": [{"name": "foo"}, {"Name": "bar"}]}`)
+	var val struct {
+		Users []struct {
+			Name string
+		}
+	}
+
+	assert.NoError(t, LoadFromJsonBytes(input, &val))
+	var expect []string
+	for _, user := range val.Users {
+		expect = append(expect, user.Name)
+	}
+	assert.EqualValues(t, []string{"foo", "bar"}, expect)
+}
+
 func TestConfigToml(t *testing.T) {
 	text := `a = "foo"
 b = 1
@@ -81,6 +97,65 @@ d = "abcd!@#$112"
 	assert.Equal(t, "abcd!@#$112", val.D)
 }
 
+func TestConfigJsonCanonical(t *testing.T) {
+	text := []byte(`{"a": "foo", "B": "bar"}`)
+
+	var val1 struct {
+		A string `json:"a"`
+		B string `json:"b"`
+	}
+	var val2 struct {
+		A string
+		B string
+	}
+	assert.NoError(t, LoadFromJsonBytes(text, &val1))
+	assert.Equal(t, "foo", val1.A)
+	assert.Equal(t, "bar", val1.B)
+	assert.NoError(t, LoadFromJsonBytes(text, &val2))
+	assert.Equal(t, "foo", val2.A)
+	assert.Equal(t, "bar", val2.B)
+}
+
+func TestConfigTomlCanonical(t *testing.T) {
+	text := []byte(`a = "foo"
+B = "bar"`)
+
+	var val1 struct {
+		A string `json:"a"`
+		B string `json:"b"`
+	}
+	var val2 struct {
+		A string
+		B string
+	}
+	assert.NoError(t, LoadFromTomlBytes(text, &val1))
+	assert.Equal(t, "foo", val1.A)
+	assert.Equal(t, "bar", val1.B)
+	assert.NoError(t, LoadFromTomlBytes(text, &val2))
+	assert.Equal(t, "foo", val2.A)
+	assert.Equal(t, "bar", val2.B)
+}
+
+func TestConfigYamlCanonical(t *testing.T) {
+	text := []byte(`a: foo
+B: bar`)
+
+	var val1 struct {
+		A string `json:"a"`
+		B string `json:"b"`
+	}
+	var val2 struct {
+		A string
+		B string
+	}
+	assert.NoError(t, LoadFromYamlBytes(text, &val1))
+	assert.Equal(t, "foo", val1.A)
+	assert.Equal(t, "bar", val1.B)
+	assert.NoError(t, LoadFromYamlBytes(text, &val2))
+	assert.Equal(t, "foo", val2.A)
+	assert.Equal(t, "bar", val2.B)
+}
+
 func TestConfigTomlEnv(t *testing.T) {
 	text := `a = "foo"
 b = 1
@@ -143,6 +218,116 @@ func TestConfigJsonEnv(t *testing.T) {
 	}
 }
 
+func TestToCamelCase(t *testing.T) {
+	tests := []struct {
+		input  string
+		expect string
+	}{
+		{
+			input:  "",
+			expect: "",
+		},
+		{
+			input:  "A",
+			expect: "a",
+		},
+		{
+			input:  "a",
+			expect: "a",
+		},
+		{
+			input:  "hello_world",
+			expect: "helloWorld",
+		},
+		{
+			input:  "Hello_world",
+			expect: "helloWorld",
+		},
+		{
+			input:  "hello_World",
+			expect: "helloWorld",
+		},
+		{
+			input:  "helloWorld",
+			expect: "helloWorld",
+		},
+		{
+			input:  "HelloWorld",
+			expect: "helloWorld",
+		},
+		{
+			input:  "hello World",
+			expect: "hello world",
+		},
+		{
+			input:  "Hello World",
+			expect: "hello world",
+		},
+		{
+			input:  "Hello World",
+			expect: "hello world",
+		},
+		{
+			input:  "Hello World foo_bar",
+			expect: "hello world fooBar",
+		},
+		{
+			input:  "Hello World foo_Bar",
+			expect: "hello world fooBar",
+		},
+		{
+			input:  "Hello World Foo_bar",
+			expect: "hello world fooBar",
+		},
+		{
+			input:  "Hello World Foo_Bar",
+			expect: "hello world fooBar",
+		},
+		{
+			input:  "你好 World Foo_Bar",
+			expect: "你好 world fooBar",
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.input, func(t *testing.T) {
+			assert.Equal(t, test.expect, toCamelCase(test.input))
+		})
+	}
+}
+
+func TestLoadFromJsonBytesError(t *testing.T) {
+	var val struct{}
+	assert.Error(t, LoadFromJsonBytes([]byte(`hello`), &val))
+}
+
+func TestLoadFromTomlBytesError(t *testing.T) {
+	var val struct{}
+	assert.Error(t, LoadFromTomlBytes([]byte(`hello`), &val))
+}
+
+func TestLoadFromYamlBytesError(t *testing.T) {
+	var val struct{}
+	assert.Error(t, LoadFromYamlBytes([]byte(`':hello`), &val))
+}
+
+func TestLoadFromYamlBytes(t *testing.T) {
+	input := []byte(`layer1:
+  layer2:
+    layer3: foo`)
+	var val struct {
+		Layer1 struct {
+			Layer2 struct {
+				Layer3 string
+			}
+		}
+	}
+
+	assert.NoError(t, LoadFromYamlBytes(input, &val))
+	assert.Equal(t, "foo", val.Layer1.Layer2.Layer3)
+}
+
 func createTempFile(ext, text string) (string, error) {
 	tmpfile, err := os.CreateTemp(os.TempDir(), hash.Md5Hex([]byte(text))+"*"+ext)
 	if err != nil {

core/mapping/jsonunmarshaler.go

@@ -11,18 +11,26 @@ const jsonTagKey = "json"
 var jsonUnmarshaler = NewUnmarshaler(jsonTagKey)
 
 // UnmarshalJsonBytes unmarshals content into v.
-func UnmarshalJsonBytes(content []byte, v interface{}) error {
-	return unmarshalJsonBytes(content, v, jsonUnmarshaler)
+func UnmarshalJsonBytes(content []byte, v interface{}, opts ...UnmarshalOption) error {
+	return unmarshalJsonBytes(content, v, getJsonUnmarshaler(opts...))
 }
 
 // UnmarshalJsonMap unmarshals content from m into v.
-func UnmarshalJsonMap(m map[string]interface{}, v interface{}) error {
-	return jsonUnmarshaler.Unmarshal(m, v)
+func UnmarshalJsonMap(m map[string]interface{}, v interface{}, opts ...UnmarshalOption) error {
+	return getJsonUnmarshaler(opts...).Unmarshal(m, v)
 }
 
 // UnmarshalJsonReader unmarshals content from reader into v.
-func UnmarshalJsonReader(reader io.Reader, v interface{}) error {
-	return unmarshalJsonReader(reader, v, jsonUnmarshaler)
+func UnmarshalJsonReader(reader io.Reader, v interface{}, opts ...UnmarshalOption) error {
+	return unmarshalJsonReader(reader, v, getJsonUnmarshaler(opts...))
+}
+
+func getJsonUnmarshaler(opts ...UnmarshalOption) *Unmarshaler {
+	if len(opts) > 0 {
+		return NewUnmarshaler(jsonTagKey, opts...)
+	}
+
+	return jsonUnmarshaler
 }
 
 func unmarshalJsonBytes(content []byte, v interface{}, unmarshaler *Unmarshaler) error {

core/mapping/jsonunmarshaler_test.go

@@ -900,7 +900,9 @@ func TestUnmarshalMap(t *testing.T) {
 			Any string `json:",optional"`
 		}
 
-		err := UnmarshalJsonMap(m, &v)
+		err := UnmarshalJsonMap(m, &v, WithCanonicalKeyFunc(func(s string) string {
+			return s
+		}))
 		assert.Nil(t, err)
 		assert.True(t, len(v.Any) == 0)
 	})