From f793e03be37e05137fd2338b86e8b6843473872d Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Thu, 15 Jul 2021 23:47:20 +0200 Subject: [PATCH 01/10] feat(encoding)!: accept a map in the decoder interface This interface is specific to decoding data into Viper's internal, so it's okay to make it Viper specific. BREAKING CHANGE: the decoder interface now accepts a map instead of an interface Signed-off-by: Mark Sagi-Kazar --- internal/encoding/decoder.go | 6 +++--- internal/encoding/decoder_test.go | 26 +++++++++++++++----------- internal/encoding/hcl/codec.go | 4 ++-- internal/encoding/json/codec.go | 4 ++-- internal/encoding/toml/codec.go | 15 +++++---------- internal/encoding/yaml/codec.go | 4 ++-- viper.go | 2 +- 7 files changed, 30 insertions(+), 31 deletions(-) diff --git a/internal/encoding/decoder.go b/internal/encoding/decoder.go index 08b1bb66b..f472e9ff1 100644 --- a/internal/encoding/decoder.go +++ b/internal/encoding/decoder.go @@ -4,10 +4,10 @@ import ( "sync" ) -// Decoder decodes the contents of b into a v representation. +// Decoder decodes the contents of b into v. // It's primarily used for decoding contents of a file into a map[string]interface{}. type Decoder interface { - Decode(b []byte, v interface{}) error + Decode(b []byte, v map[string]interface{}) error } const ( @@ -48,7 +48,7 @@ func (e *DecoderRegistry) RegisterDecoder(format string, enc Decoder) error { } // Decode calls the underlying Decoder based on the format. -func (e *DecoderRegistry) Decode(format string, b []byte, v interface{}) error { +func (e *DecoderRegistry) Decode(format string, b []byte, v map[string]interface{}) error { e.mu.RLock() decoder, ok := e.decoders[format] e.mu.RUnlock() diff --git a/internal/encoding/decoder_test.go b/internal/encoding/decoder_test.go index 80e668887..6cb56d021 100644 --- a/internal/encoding/decoder_test.go +++ b/internal/encoding/decoder_test.go @@ -1,16 +1,18 @@ package encoding import ( + "reflect" "testing" ) type decoder struct { - v interface{} + v map[string]interface{} } -func (d decoder) Decode(_ []byte, v interface{}) error { - rv := v.(*string) - *rv = d.v.(string) +func (d decoder) Decode(_ []byte, v map[string]interface{}) error { + for key, value := range d.v { + v[key] = value + } return nil } @@ -44,7 +46,9 @@ func TestDecoderRegistry_Decode(t *testing.T) { t.Run("OK", func(t *testing.T) { registry := NewDecoderRegistry() decoder := decoder{ - v: "decoded value", + v: map[string]interface{}{ + "key": "value", + }, } err := registry.RegisterDecoder("myformat", decoder) @@ -52,24 +56,24 @@ func TestDecoderRegistry_Decode(t *testing.T) { t.Fatal(err) } - var v string + v := map[string]interface{}{} - err = registry.Decode("myformat", []byte("some value"), &v) + err = registry.Decode("myformat", []byte("key: value"), v) if err != nil { t.Fatal(err) } - if v != "decoded value" { - t.Fatalf("expected 'decoded value', got: %#v", v) + if !reflect.DeepEqual(decoder.v, v) { + t.Fatalf("decoded value does not match the expected one\nactual: %+v\nexpected: %+v", v, decoder.v) } }) t.Run("DecoderNotFound", func(t *testing.T) { registry := NewDecoderRegistry() - var v string + v := map[string]interface{}{} - err := registry.Decode("myformat", []byte("some value"), &v) + err := registry.Decode("myformat", nil, v) if err != ErrDecoderNotFound { t.Fatalf("expected ErrDecoderNotFound, got: %v", err) } diff --git a/internal/encoding/hcl/codec.go b/internal/encoding/hcl/codec.go index f3e4ab122..6e2d50847 100644 --- a/internal/encoding/hcl/codec.go +++ b/internal/encoding/hcl/codec.go @@ -35,6 +35,6 @@ func (Codec) Encode(v interface{}) ([]byte, error) { return buf.Bytes(), nil } -func (Codec) Decode(b []byte, v interface{}) error { - return hcl.Unmarshal(b, v) +func (Codec) Decode(b []byte, v map[string]interface{}) error { + return hcl.Unmarshal(b, &v) } diff --git a/internal/encoding/json/codec.go b/internal/encoding/json/codec.go index dff9ec982..f8fbdd089 100644 --- a/internal/encoding/json/codec.go +++ b/internal/encoding/json/codec.go @@ -12,6 +12,6 @@ func (Codec) Encode(v interface{}) ([]byte, error) { return json.MarshalIndent(v, "", " ") } -func (Codec) Decode(b []byte, v interface{}) error { - return json.Unmarshal(b, v) +func (Codec) Decode(b []byte, v map[string]interface{}) error { + return json.Unmarshal(b, &v) } diff --git a/internal/encoding/toml/codec.go b/internal/encoding/toml/codec.go index c043802b9..48b05628a 100644 --- a/internal/encoding/toml/codec.go +++ b/internal/encoding/toml/codec.go @@ -25,21 +25,16 @@ func (Codec) Encode(v interface{}) ([]byte, error) { return toml.Marshal(v) } -func (Codec) Decode(b []byte, v interface{}) error { +func (Codec) Decode(b []byte, v map[string]interface{}) error { tree, err := toml.LoadBytes(b) if err != nil { return err } - if m, ok := v.(*map[string]interface{}); ok { - vmap := *m - tmap := tree.ToMap() - for k, v := range tmap { - vmap[k] = v - } - - return nil + tmap := tree.ToMap() + for key, value := range tmap { + v[key] = value } - return tree.Unmarshal(v) + return nil } diff --git a/internal/encoding/yaml/codec.go b/internal/encoding/yaml/codec.go index f94b26996..a46a882e7 100644 --- a/internal/encoding/yaml/codec.go +++ b/internal/encoding/yaml/codec.go @@ -9,6 +9,6 @@ func (Codec) Encode(v interface{}) ([]byte, error) { return yaml.Marshal(v) } -func (Codec) Decode(b []byte, v interface{}) error { - return yaml.Unmarshal(b, v) +func (Codec) Decode(b []byte, v map[string]interface{}) error { + return yaml.Unmarshal(b, &v) } diff --git a/viper.go b/viper.go index 4a9935899..3f14b9461 100644 --- a/viper.go +++ b/viper.go @@ -1635,7 +1635,7 @@ func (v *Viper) unmarshalReader(in io.Reader, c map[string]interface{}) error { switch format := strings.ToLower(v.getConfigType()); format { case "yaml", "yml", "json", "toml", "hcl", "tfvars": - err := decoderRegistry.Decode(format, buf.Bytes(), &c) + err := decoderRegistry.Decode(format, buf.Bytes(), c) if err != nil { return ConfigParseError{err} } From 6980ee0b0493afb0098a07820b6d502c6d2a2f0b Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Fri, 16 Jul 2021 00:26:30 +0200 Subject: [PATCH 02/10] feat(encoding)!: accept a map in the encoder interface This interface is specific to encoding data from Viper's internal, so it's okay to make it Viper specific. BREAKING CHANGE: the encoder interface now accepts a map instead of an interface Signed-off-by: Mark Sagi-Kazar --- internal/encoding/encoder.go | 4 ++-- internal/encoding/encoder_test.go | 12 ++++++------ internal/encoding/hcl/codec.go | 2 +- internal/encoding/json/codec.go | 2 +- internal/encoding/toml/codec.go | 24 ++++++++++-------------- internal/encoding/yaml/codec.go | 2 +- 6 files changed, 21 insertions(+), 25 deletions(-) diff --git a/internal/encoding/encoder.go b/internal/encoding/encoder.go index 82c7996cb..2341bf235 100644 --- a/internal/encoding/encoder.go +++ b/internal/encoding/encoder.go @@ -7,7 +7,7 @@ import ( // Encoder encodes the contents of v into a byte representation. // It's primarily used for encoding a map[string]interface{} into a file format. type Encoder interface { - Encode(v interface{}) ([]byte, error) + Encode(v map[string]interface{}) ([]byte, error) } const ( @@ -47,7 +47,7 @@ func (e *EncoderRegistry) RegisterEncoder(format string, enc Encoder) error { return nil } -func (e *EncoderRegistry) Encode(format string, v interface{}) ([]byte, error) { +func (e *EncoderRegistry) Encode(format string, v map[string]interface{}) ([]byte, error) { e.mu.RLock() encoder, ok := e.encoders[format] e.mu.RUnlock() diff --git a/internal/encoding/encoder_test.go b/internal/encoding/encoder_test.go index e2472ad7c..adee6d090 100644 --- a/internal/encoding/encoder_test.go +++ b/internal/encoding/encoder_test.go @@ -8,7 +8,7 @@ type encoder struct { b []byte } -func (e encoder) Encode(_ interface{}) ([]byte, error) { +func (e encoder) Encode(_ map[string]interface{}) ([]byte, error) { return e.b, nil } @@ -41,7 +41,7 @@ func TestEncoderRegistry_Decode(t *testing.T) { t.Run("OK", func(t *testing.T) { registry := NewEncoderRegistry() encoder := encoder{ - b: []byte("encoded value"), + b: []byte("key: value"), } err := registry.RegisterEncoder("myformat", encoder) @@ -49,20 +49,20 @@ func TestEncoderRegistry_Decode(t *testing.T) { t.Fatal(err) } - b, err := registry.Encode("myformat", "some value") + b, err := registry.Encode("myformat", map[string]interface{}{"key": "value"}) if err != nil { t.Fatal(err) } - if string(b) != "encoded value" { - t.Fatalf("expected 'encoded value', got: %#v", string(b)) + if string(b) != "key: value" { + t.Fatalf("expected 'key: value', got: %#v", string(b)) } }) t.Run("EncoderNotFound", func(t *testing.T) { registry := NewEncoderRegistry() - _, err := registry.Encode("myformat", "some value") + _, err := registry.Encode("myformat", map[string]interface{}{"key": "value"}) if err != ErrEncoderNotFound { t.Fatalf("expected ErrEncoderNotFound, got: %v", err) } diff --git a/internal/encoding/hcl/codec.go b/internal/encoding/hcl/codec.go index 6e2d50847..7fde8e4bc 100644 --- a/internal/encoding/hcl/codec.go +++ b/internal/encoding/hcl/codec.go @@ -12,7 +12,7 @@ import ( // TODO: add printer config to the codec? type Codec struct{} -func (Codec) Encode(v interface{}) ([]byte, error) { +func (Codec) Encode(v map[string]interface{}) ([]byte, error) { b, err := json.Marshal(v) if err != nil { return nil, err diff --git a/internal/encoding/json/codec.go b/internal/encoding/json/codec.go index f8fbdd089..1b7caaceb 100644 --- a/internal/encoding/json/codec.go +++ b/internal/encoding/json/codec.go @@ -7,7 +7,7 @@ import ( // Codec implements the encoding.Encoder and encoding.Decoder interfaces for JSON encoding. type Codec struct{} -func (Codec) Encode(v interface{}) ([]byte, error) { +func (Codec) Encode(v map[string]interface{}) ([]byte, error) { // TODO: expose prefix and indent in the Codec as setting? return json.MarshalIndent(v, "", " ") } diff --git a/internal/encoding/toml/codec.go b/internal/encoding/toml/codec.go index 48b05628a..ad0a18b35 100644 --- a/internal/encoding/toml/codec.go +++ b/internal/encoding/toml/codec.go @@ -7,22 +7,18 @@ import ( // Codec implements the encoding.Encoder and encoding.Decoder interfaces for TOML encoding. type Codec struct{} -func (Codec) Encode(v interface{}) ([]byte, error) { - if m, ok := v.(map[string]interface{}); ok { - t, err := toml.TreeFromMap(m) - if err != nil { - return nil, err - } - - s, err := t.ToTomlString() - if err != nil { - return nil, err - } - - return []byte(s), nil +func (Codec) Encode(v map[string]interface{}) ([]byte, error) { + t, err := toml.TreeFromMap(v) + if err != nil { + return nil, err + } + + s, err := t.ToTomlString() + if err != nil { + return nil, err } - return toml.Marshal(v) + return []byte(s), nil } func (Codec) Decode(b []byte, v map[string]interface{}) error { diff --git a/internal/encoding/yaml/codec.go b/internal/encoding/yaml/codec.go index a46a882e7..5fa4b3924 100644 --- a/internal/encoding/yaml/codec.go +++ b/internal/encoding/yaml/codec.go @@ -5,7 +5,7 @@ import "gopkg.in/yaml.v2" // Codec implements the encoding.Encoder and encoding.Decoder interfaces for YAML encoding. type Codec struct{} -func (Codec) Encode(v interface{}) ([]byte, error) { +func (Codec) Encode(v map[string]interface{}) ([]byte, error) { return yaml.Marshal(v) } From 0a4e72cb738b610f33d01887d2442996a3888113 Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Fri, 16 Jul 2021 01:47:03 +0200 Subject: [PATCH 03/10] test(encoding): add tests for existing encoding implementations Signed-off-by: Mark Sagi-Kazar --- internal/encoding/hcl/codec_test.go | 140 +++++++++++++++++++++++++++ internal/encoding/json/codec_test.go | 95 ++++++++++++++++++ internal/encoding/toml/codec_test.go | 106 ++++++++++++++++++++ internal/encoding/yaml/codec_test.go | 136 ++++++++++++++++++++++++++ 4 files changed, 477 insertions(+) create mode 100644 internal/encoding/hcl/codec_test.go create mode 100644 internal/encoding/json/codec_test.go create mode 100644 internal/encoding/toml/codec_test.go create mode 100644 internal/encoding/yaml/codec_test.go diff --git a/internal/encoding/hcl/codec_test.go b/internal/encoding/hcl/codec_test.go new file mode 100644 index 000000000..7e48057c0 --- /dev/null +++ b/internal/encoding/hcl/codec_test.go @@ -0,0 +1,140 @@ +package hcl + +import ( + "reflect" + "testing" +) + +// original form of the data +const original = `# key-value pair +"key" = "value" + +// list +"list" = ["item1", "item2", "item3"] + +/* map */ +"map" = { + "key" = "value" +} + +/* +nested map +*/ +"nested_map" "map" { + "key" = "value" + + "list" = ["item1", "item2", "item3"] +}` + +// encoded form of the data +const encoded = `"key" = "value" + +"list" = ["item1", "item2", "item3"] + +"map" = { + "key" = "value" +} + +"nested_map" "map" { + "key" = "value" + + "list" = ["item1", "item2", "item3"] +}` + +// decoded form of the data +// +// in case of HCL it's slightly different from Viper's internal representation +// (eg. map is decoded into a list of maps) +var decoded = map[string]interface{}{ + "key": "value", + "list": []interface{}{ + "item1", + "item2", + "item3", + }, + "map": []map[string]interface{}{ + { + "key": "value", + }, + }, + "nested_map": []map[string]interface{}{ + { + "map": []map[string]interface{}{ + { + "key": "value", + "list": []interface{}{ + "item1", + "item2", + "item3", + }, + }, + }, + }, + }, +} + +// Viper's internal representation +var data = map[string]interface{}{ + "key": "value", + "list": []interface{}{ + "item1", + "item2", + "item3", + }, + "map": map[string]interface{}{ + "key": "value", + }, + "nested_map": map[string]interface{}{ + "map": map[string]interface{}{ + "key": "value", + "list": []interface{}{ + "item1", + "item2", + "item3", + }, + }, + }, +} + +func TestCodec_Encode(t *testing.T) { + codec := Codec{} + + b, err := codec.Encode(data) + if err != nil { + t.Fatal(err) + } + + if encoded != string(b) { + t.Fatalf("decoded value does not match the expected one\nactual: %#v\nexpected: %#v", string(b), encoded) + } +} + +func TestCodec_Decode(t *testing.T) { + t.Run("OK", func(t *testing.T) { + codec := Codec{} + + v := map[string]interface{}{} + + err := codec.Decode([]byte(original), v) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(decoded, v) { + t.Fatalf("decoded value does not match the expected one\nactual: %#v\nexpected: %#v", v, decoded) + } + }) + + t.Run("InvalidData", func(t *testing.T) { + codec := Codec{} + + v := map[string]interface{}{} + + err := codec.Decode([]byte(`invalid data`), v) + if err == nil { + t.Fatal("expected decoding to fail") + } + + t.Logf("decoding failed as expected: %s", err) + }) +} diff --git a/internal/encoding/json/codec_test.go b/internal/encoding/json/codec_test.go new file mode 100644 index 000000000..f4a71df49 --- /dev/null +++ b/internal/encoding/json/codec_test.go @@ -0,0 +1,95 @@ +package json + +import ( + "reflect" + "testing" +) + +// encoded form of the data +const encoded = `{ + "key": "value", + "list": [ + "item1", + "item2", + "item3" + ], + "map": { + "key": "value" + }, + "nested_map": { + "map": { + "key": "value", + "list": [ + "item1", + "item2", + "item3" + ] + } + } +}` + +// Viper's internal representation +var data = map[string]interface{}{ + "key": "value", + "list": []interface{}{ + "item1", + "item2", + "item3", + }, + "map": map[string]interface{}{ + "key": "value", + }, + "nested_map": map[string]interface{}{ + "map": map[string]interface{}{ + "key": "value", + "list": []interface{}{ + "item1", + "item2", + "item3", + }, + }, + }, +} + +func TestCodec_Encode(t *testing.T) { + codec := Codec{} + + b, err := codec.Encode(data) + if err != nil { + t.Fatal(err) + } + + if encoded != string(b) { + t.Fatalf("decoded value does not match the expected one\nactual: %#v\nexpected: %#v", string(b), encoded) + } +} + +func TestCodec_Decode(t *testing.T) { + t.Run("OK", func(t *testing.T) { + codec := Codec{} + + v := map[string]interface{}{} + + err := codec.Decode([]byte(encoded), v) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(data, v) { + t.Fatalf("decoded value does not match the expected one\nactual: %#v\nexpected: %#v", v, data) + } + }) + + t.Run("InvalidData", func(t *testing.T) { + codec := Codec{} + + v := map[string]interface{}{} + + err := codec.Decode([]byte(`invalid data`), v) + if err == nil { + t.Fatal("expected decoding to fail") + } + + t.Logf("decoding failed as expected: %s", err) + }) +} diff --git a/internal/encoding/toml/codec_test.go b/internal/encoding/toml/codec_test.go new file mode 100644 index 000000000..33435e69d --- /dev/null +++ b/internal/encoding/toml/codec_test.go @@ -0,0 +1,106 @@ +package toml + +import ( + "reflect" + "testing" +) + +// original form of the data +const original = `# key-value pair +key = "value" +list = ["item1", "item2", "item3"] + +[map] +key = "value" + +# nested +# map +[nested_map] +[nested_map.map] +key = "value" +list = [ + "item1", + "item2", + "item3", +] +` + +// encoded form of the data +const encoded = `key = "value" +list = ["item1", "item2", "item3"] + +[map] + key = "value" + +[nested_map] + + [nested_map.map] + key = "value" + list = ["item1", "item2", "item3"] +` + +// Viper's internal representation +var data = map[string]interface{}{ + "key": "value", + "list": []interface{}{ + "item1", + "item2", + "item3", + }, + "map": map[string]interface{}{ + "key": "value", + }, + "nested_map": map[string]interface{}{ + "map": map[string]interface{}{ + "key": "value", + "list": []interface{}{ + "item1", + "item2", + "item3", + }, + }, + }, +} + +func TestCodec_Encode(t *testing.T) { + codec := Codec{} + + b, err := codec.Encode(data) + if err != nil { + t.Fatal(err) + } + + if encoded != string(b) { + t.Fatalf("decoded value does not match the expected one\nactual: %#v\nexpected: %#v", string(b), encoded) + } +} + +func TestCodec_Decode(t *testing.T) { + t.Run("OK", func(t *testing.T) { + codec := Codec{} + + v := map[string]interface{}{} + + err := codec.Decode([]byte(original), v) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(data, v) { + t.Fatalf("decoded value does not match the expected one\nactual: %#v\nexpected: %#v", v, data) + } + }) + + t.Run("InvalidData", func(t *testing.T) { + codec := Codec{} + + v := map[string]interface{}{} + + err := codec.Decode([]byte(`invalid data`), v) + if err == nil { + t.Fatal("expected decoding to fail") + } + + t.Logf("decoding failed as expected: %s", err) + }) +} diff --git a/internal/encoding/yaml/codec_test.go b/internal/encoding/yaml/codec_test.go new file mode 100644 index 000000000..d76a6c657 --- /dev/null +++ b/internal/encoding/yaml/codec_test.go @@ -0,0 +1,136 @@ +package yaml + +import ( + "reflect" + "testing" +) + +// original form of the data +const original = `# key-value pair +key: value +list: +- item1 +- item2 +- item3 +map: + key: value + +# nested +# map +nested_map: + map: + key: value + list: + - item1 + - item2 + - item3 +` + +// encoded form of the data +const encoded = `key: value +list: +- item1 +- item2 +- item3 +map: + key: value +nested_map: + map: + key: value + list: + - item1 + - item2 + - item3 +` + +// decoded form of the data +// +// in case of YAML it's slightly different from Viper's internal representation +// (eg. map is decoded into a map with interface key) +var decoded = map[string]interface{}{ + "key": "value", + "list": []interface{}{ + "item1", + "item2", + "item3", + }, + "map": map[interface{}]interface{}{ + "key": "value", + }, + "nested_map": map[interface{}]interface{}{ + "map": map[interface{}]interface{}{ + "key": "value", + "list": []interface{}{ + "item1", + "item2", + "item3", + }, + }, + }, +} + +// Viper's internal representation +var data = map[string]interface{}{ + "key": "value", + "list": []interface{}{ + "item1", + "item2", + "item3", + }, + "map": map[string]interface{}{ + "key": "value", + }, + "nested_map": map[string]interface{}{ + "map": map[string]interface{}{ + "key": "value", + "list": []interface{}{ + "item1", + "item2", + "item3", + }, + }, + }, +} + +func TestCodec_Encode(t *testing.T) { + codec := Codec{} + + b, err := codec.Encode(data) + if err != nil { + t.Fatal(err) + } + + if encoded != string(b) { + t.Fatalf("decoded value does not match the expected one\nactual: %#v\nexpected: %#v", string(b), encoded) + } +} + +func TestCodec_Decode(t *testing.T) { + t.Run("OK", func(t *testing.T) { + codec := Codec{} + + v := map[string]interface{}{} + + err := codec.Decode([]byte(original), v) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(decoded, v) { + t.Fatalf("decoded value does not match the expected one\nactual: %#v\nexpected: %#v", v, decoded) + } + }) + + t.Run("InvalidData", func(t *testing.T) { + codec := Codec{} + + v := map[string]interface{}{} + + err := codec.Decode([]byte(`invalid data`), v) + if err == nil { + t.Fatal("expected decoding to fail") + } + + t.Logf("decoding failed as expected: %s", err) + }) +} From 12057d1ef7db49dd5eb0ead910555ea9842bfa0c Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Fri, 16 Jul 2021 02:59:29 +0200 Subject: [PATCH 04/10] feat(encoding): add ini codec Signed-off-by: Mark Sagi-Kazar --- internal/encoding/ini/codec.go | 99 ++++++++++++++++++++++++ internal/encoding/ini/codec_test.go | 112 ++++++++++++++++++++++++++++ internal/encoding/ini/map_utils.go | 74 ++++++++++++++++++ 3 files changed, 285 insertions(+) create mode 100644 internal/encoding/ini/codec.go create mode 100644 internal/encoding/ini/codec_test.go create mode 100644 internal/encoding/ini/map_utils.go diff --git a/internal/encoding/ini/codec.go b/internal/encoding/ini/codec.go new file mode 100644 index 000000000..9acd87fc3 --- /dev/null +++ b/internal/encoding/ini/codec.go @@ -0,0 +1,99 @@ +package ini + +import ( + "bytes" + "sort" + "strings" + + "github.com/spf13/cast" + "gopkg.in/ini.v1" +) + +// LoadOptions contains all customized options used for load data source(s). +// This type is added here for convenience: this way consumers can import a single package called "ini". +type LoadOptions = ini.LoadOptions + +// Codec implements the encoding.Encoder and encoding.Decoder interfaces for INI encoding. +type Codec struct { + KeyDelimiter string + LoadOptions LoadOptions +} + +func (c Codec) Encode(v map[string]interface{}) ([]byte, error) { + cfg := ini.Empty() + ini.PrettyFormat = false + + flattened := map[string]interface{}{} + + flattened = flattenAndMergeMap(flattened, v, "", c.keyDelimiter()) + + keys := make([]string, 0, len(flattened)) + + for key := range flattened { + keys = append(keys, key) + } + + sort.Strings(keys) + + for _, key := range keys { + sectionName, keyName := "", key + + lastSep := strings.LastIndex(key, ".") + if lastSep != -1 { + sectionName = key[:(lastSep)] + keyName = key[(lastSep + 1):] + } + + // TODO: is this a good idea? + if sectionName == "default" { + sectionName = "" + } + + cfg.Section(sectionName).Key(keyName).SetValue(cast.ToString(flattened[key])) + } + + var buf bytes.Buffer + + _, err := cfg.WriteTo(&buf) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func (c Codec) Decode(b []byte, v map[string]interface{}) error { + cfg := ini.Empty(c.LoadOptions) + + err := cfg.Append(b) + if err != nil { + return err + } + + sections := cfg.Sections() + + for i := 0; i < len(sections); i++ { + section := sections[i] + keys := section.Keys() + + for j := 0; j < len(keys); j++ { + key := keys[j] + value := cfg.Section(section.Name()).Key(key.Name()).String() + + deepestMap := deepSearch(v, strings.Split(section.Name(), c.keyDelimiter())) + + // set innermost value + deepestMap[key.Name()] = value + } + } + + return nil +} + +func (c Codec) keyDelimiter() string { + if c.KeyDelimiter == "" { + return "." + } + + return c.KeyDelimiter +} diff --git a/internal/encoding/ini/codec_test.go b/internal/encoding/ini/codec_test.go new file mode 100644 index 000000000..ca48617cf --- /dev/null +++ b/internal/encoding/ini/codec_test.go @@ -0,0 +1,112 @@ +package ini + +import ( + "reflect" + "testing" +) + +// original form of the data +const original = `; key-value pair +key=value ; key-value pair + +# map +[map] # map +key=%(key)s + +` + +// encoded form of the data +const encoded = `key=value + +[map] +key=value + +` + +// decoded form of the data +// +// in case of INI it's slightly different from Viper's internal representation +// (eg. top level keys land in a section called default) +var decoded = map[string]interface{}{ + "DEFAULT": map[string]interface{}{ + "key": "value", + }, + "map": map[string]interface{}{ + "key": "value", + }, +} + +// Viper's internal representation +var data = map[string]interface{}{ + "key": "value", + "map": map[string]interface{}{ + "key": "value", + }, +} + +func TestCodec_Encode(t *testing.T) { + t.Run("OK", func(t *testing.T) { + codec := Codec{} + + b, err := codec.Encode(data) + if err != nil { + t.Fatal(err) + } + + if encoded != string(b) { + t.Fatalf("decoded value does not match the expected one\nactual: %#v\nexpected: %#v", string(b), encoded) + } + }) + + t.Run("Default", func(t *testing.T) { + codec := Codec{} + + data := map[string]interface{}{ + "default": map[string]interface{}{ + "key": "value", + }, + "map": map[string]interface{}{ + "key": "value", + }, + } + + b, err := codec.Encode(data) + if err != nil { + t.Fatal(err) + } + + if encoded != string(b) { + t.Fatalf("decoded value does not match the expected one\nactual: %#v\nexpected: %#v", string(b), encoded) + } + }) +} + +func TestCodec_Decode(t *testing.T) { + t.Run("OK", func(t *testing.T) { + codec := Codec{} + + v := map[string]interface{}{} + + err := codec.Decode([]byte(original), v) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(decoded, v) { + t.Fatalf("decoded value does not match the expected one\nactual: %#v\nexpected: %#v", v, decoded) + } + }) + + t.Run("InvalidData", func(t *testing.T) { + codec := Codec{} + + v := map[string]interface{}{} + + err := codec.Decode([]byte(`invalid data`), v) + if err == nil { + t.Fatal("expected decoding to fail") + } + + t.Logf("decoding failed as expected: %s", err) + }) +} diff --git a/internal/encoding/ini/map_utils.go b/internal/encoding/ini/map_utils.go new file mode 100644 index 000000000..8329856b5 --- /dev/null +++ b/internal/encoding/ini/map_utils.go @@ -0,0 +1,74 @@ +package ini + +import ( + "strings" + + "github.com/spf13/cast" +) + +// THIS CODE IS COPIED HERE: IT SHOULD NOT BE MODIFIED +// AT SOME POINT IT WILL BE MOVED TO A COMMON PLACE +// deepSearch scans deep maps, following the key indexes listed in the +// sequence "path". +// The last value is expected to be another map, and is returned. +// +// In case intermediate keys do not exist, or map to a non-map value, +// a new map is created and inserted, and the search continues from there: +// the initial map "m" may be modified! +func deepSearch(m map[string]interface{}, path []string) map[string]interface{} { + for _, k := range path { + m2, ok := m[k] + if !ok { + // intermediate key does not exist + // => create it and continue from there + m3 := make(map[string]interface{}) + m[k] = m3 + m = m3 + continue + } + m3, ok := m2.(map[string]interface{}) + if !ok { + // intermediate key is a value + // => replace with a new map + m3 = make(map[string]interface{}) + m[k] = m3 + } + // continue search from here + m = m3 + } + return m +} + +// flattenAndMergeMap recursively flattens the given map into a new map +// Code is based on the function with the same name in tha main package. +// TODO: move it to a common place +func flattenAndMergeMap(shadow map[string]interface{}, m map[string]interface{}, prefix string, delimiter string) map[string]interface{} { + if shadow != nil && prefix != "" && shadow[prefix] != nil { + // prefix is shadowed => nothing more to flatten + return shadow + } + if shadow == nil { + shadow = make(map[string]interface{}) + } + + var m2 map[string]interface{} + if prefix != "" { + prefix += delimiter + } + for k, val := range m { + fullKey := prefix + k + switch val.(type) { + case map[string]interface{}: + m2 = val.(map[string]interface{}) + case map[interface{}]interface{}: + m2 = cast.ToStringMap(val) + default: + // immediate value + shadow[strings.ToLower(fullKey)] = val + continue + } + // recursively merge to shadow map + shadow = flattenAndMergeMap(shadow, m2, fullKey, delimiter) + } + return shadow +} From 9a19bc4e6c7ee7d88277977bcb2f6536f72e3d94 Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Fri, 16 Jul 2021 03:09:43 +0200 Subject: [PATCH 05/10] refactor(encoding): initialize codecs per Viper Some codecs might have options that rely on Viper in the future (eg. key delimiter) which requires initializing codecs for each Viper instance. Signed-off-by: Mark Sagi-Kazar --- viper.go | 94 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 53 insertions(+), 41 deletions(-) diff --git a/viper.go b/viper.go index 3f14b9461..587ca6a67 100644 --- a/viper.go +++ b/viper.go @@ -67,47 +67,8 @@ type RemoteResponse struct { Error error } -var ( - encoderRegistry = encoding.NewEncoderRegistry() - decoderRegistry = encoding.NewDecoderRegistry() -) - func init() { v = New() - - { - codec := yaml.Codec{} - - encoderRegistry.RegisterEncoder("yaml", codec) - decoderRegistry.RegisterDecoder("yaml", codec) - - encoderRegistry.RegisterEncoder("yml", codec) - decoderRegistry.RegisterDecoder("yml", codec) - } - - { - codec := json.Codec{} - - encoderRegistry.RegisterEncoder("json", codec) - decoderRegistry.RegisterDecoder("json", codec) - } - - { - codec := toml.Codec{} - - encoderRegistry.RegisterEncoder("toml", codec) - decoderRegistry.RegisterDecoder("toml", codec) - } - - { - codec := hcl.Codec{} - - encoderRegistry.RegisterEncoder("hcl", codec) - decoderRegistry.RegisterDecoder("hcl", codec) - - encoderRegistry.RegisterEncoder("tfvars", codec) - decoderRegistry.RegisterDecoder("tfvars", codec) - } } type remoteConfigFactory interface { @@ -261,6 +222,10 @@ type Viper struct { onConfigChange func(fsnotify.Event) logger Logger + + // TODO: should probably be protected with a mutex + encoderRegistry *encoding.EncoderRegistry + decoderRegistry *encoding.DecoderRegistry } // New returns an initialized Viper instance. @@ -280,6 +245,8 @@ func New() *Viper { v.typeByDefValue = false v.logger = jwwLogger{} + v.resetEncoding() + return v } @@ -326,6 +293,8 @@ func NewWithOptions(opts ...Option) *Viper { opt.apply(v) } + v.resetEncoding() + return v } @@ -338,6 +307,49 @@ func Reset() { SupportedRemoteProviders = []string{"etcd", "consul", "firestore"} } +// TODO: make this lazy initialization instead +func (v *Viper) resetEncoding() { + encoderRegistry := encoding.NewEncoderRegistry() + decoderRegistry := encoding.NewDecoderRegistry() + + { + codec := yaml.Codec{} + + encoderRegistry.RegisterEncoder("yaml", codec) + decoderRegistry.RegisterDecoder("yaml", codec) + + encoderRegistry.RegisterEncoder("yml", codec) + decoderRegistry.RegisterDecoder("yml", codec) + } + + { + codec := json.Codec{} + + encoderRegistry.RegisterEncoder("json", codec) + decoderRegistry.RegisterDecoder("json", codec) + } + + { + codec := toml.Codec{} + + encoderRegistry.RegisterEncoder("toml", codec) + decoderRegistry.RegisterDecoder("toml", codec) + } + + { + codec := hcl.Codec{} + + encoderRegistry.RegisterEncoder("hcl", codec) + decoderRegistry.RegisterDecoder("hcl", codec) + + encoderRegistry.RegisterEncoder("tfvars", codec) + decoderRegistry.RegisterDecoder("tfvars", codec) + } + + v.encoderRegistry = encoderRegistry + v.decoderRegistry = decoderRegistry +} + type defaultRemoteProvider struct { provider string endpoint string @@ -1635,7 +1647,7 @@ func (v *Viper) unmarshalReader(in io.Reader, c map[string]interface{}) error { switch format := strings.ToLower(v.getConfigType()); format { case "yaml", "yml", "json", "toml", "hcl", "tfvars": - err := decoderRegistry.Decode(format, buf.Bytes(), c) + err := v.decoderRegistry.Decode(format, buf.Bytes(), c) if err != nil { return ConfigParseError{err} } @@ -1692,7 +1704,7 @@ func (v *Viper) marshalWriter(f afero.File, configType string) error { c := v.AllSettings() switch configType { case "yaml", "yml", "json", "toml", "hcl", "tfvars": - b, err := encoderRegistry.Encode(configType, c) + b, err := v.encoderRegistry.Encode(configType, c) if err != nil { return ConfigMarshalError{err} } From e32369a152ae71dc550f0699f139929f504d2faf Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Fri, 16 Jul 2021 03:14:41 +0200 Subject: [PATCH 06/10] feat(encoding): integrate ini codec into Viper Signed-off-by: Mark Sagi-Kazar --- viper.go | 49 +++++++++++++------------------------------------ 1 file changed, 13 insertions(+), 36 deletions(-) diff --git a/viper.go b/viper.go index 587ca6a67..0c10093f2 100644 --- a/viper.go +++ b/viper.go @@ -41,10 +41,10 @@ import ( "github.com/spf13/cast" "github.com/spf13/pflag" "github.com/subosito/gotenv" - "gopkg.in/ini.v1" "github.com/spf13/viper/internal/encoding" "github.com/spf13/viper/internal/encoding/hcl" + "github.com/spf13/viper/internal/encoding/ini" "github.com/spf13/viper/internal/encoding/json" "github.com/spf13/viper/internal/encoding/toml" "github.com/spf13/viper/internal/encoding/yaml" @@ -346,6 +346,16 @@ func (v *Viper) resetEncoding() { decoderRegistry.RegisterDecoder("tfvars", codec) } + { + codec := ini.Codec{ + KeyDelimiter: v.keyDelim, + LoadOptions: v.iniLoadOptions, + } + + encoderRegistry.RegisterEncoder("ini", codec) + decoderRegistry.RegisterDecoder("ini", codec) + } + v.encoderRegistry = encoderRegistry v.decoderRegistry = decoderRegistry } @@ -1646,7 +1656,7 @@ func (v *Viper) unmarshalReader(in io.Reader, c map[string]interface{}) error { buf.ReadFrom(in) switch format := strings.ToLower(v.getConfigType()); format { - case "yaml", "yml", "json", "toml", "hcl", "tfvars": + case "yaml", "yml", "json", "toml", "hcl", "tfvars", "ini": err := v.decoderRegistry.Decode(format, buf.Bytes(), c) if err != nil { return ConfigParseError{err} @@ -1676,23 +1686,6 @@ func (v *Viper) unmarshalReader(in io.Reader, c map[string]interface{}) error { // set innermost value deepestMap[lastKey] = value } - - case "ini": - cfg := ini.Empty(v.iniLoadOptions) - err := cfg.Append(buf.Bytes()) - if err != nil { - return ConfigParseError{err} - } - sections := cfg.Sections() - for i := 0; i < len(sections); i++ { - section := sections[i] - keys := section.Keys() - for j := 0; j < len(keys); j++ { - key := keys[j] - value := cfg.Section(section.Name()).Key(key.Name()).String() - c[section.Name()+"."+key.Name()] = value - } - } } insensitiviseMap(c) @@ -1703,7 +1696,7 @@ func (v *Viper) unmarshalReader(in io.Reader, c map[string]interface{}) error { func (v *Viper) marshalWriter(f afero.File, configType string) error { c := v.AllSettings() switch configType { - case "yaml", "yml", "json", "toml", "hcl", "tfvars": + case "yaml", "yml", "json", "toml", "hcl", "tfvars", "ini": b, err := v.encoderRegistry.Encode(configType, c) if err != nil { return ConfigMarshalError{err} @@ -1741,22 +1734,6 @@ func (v *Viper) marshalWriter(f afero.File, configType string) error { if _, err := f.WriteString(s); err != nil { return ConfigMarshalError{err} } - - case "ini": - keys := v.AllKeys() - cfg := ini.Empty() - ini.PrettyFormat = false - for i := 0; i < len(keys); i++ { - key := keys[i] - lastSep := strings.LastIndex(key, ".") - sectionName := key[:(lastSep)] - keyName := key[(lastSep + 1):] - if sectionName == "default" { - sectionName = "" - } - cfg.Section(sectionName).Key(keyName).SetValue(v.GetString(key)) - } - cfg.WriteTo(f) } return nil } From bf4765206a34ac1bad063194eb1a8f5f3d1c1ab1 Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Fri, 16 Jul 2021 03:38:17 +0200 Subject: [PATCH 07/10] feat(encoding): add Java properties codec Signed-off-by: Mark Sagi-Kazar --- internal/encoding/javaproperties/codec.go | 77 +++++++++++++++++++ .../encoding/javaproperties/codec_test.go | 69 +++++++++++++++++ internal/encoding/javaproperties/map_utils.go | 74 ++++++++++++++++++ 3 files changed, 220 insertions(+) create mode 100644 internal/encoding/javaproperties/codec.go create mode 100644 internal/encoding/javaproperties/codec_test.go create mode 100644 internal/encoding/javaproperties/map_utils.go diff --git a/internal/encoding/javaproperties/codec.go b/internal/encoding/javaproperties/codec.go new file mode 100644 index 000000000..ac3b66b68 --- /dev/null +++ b/internal/encoding/javaproperties/codec.go @@ -0,0 +1,77 @@ +package javaproperties + +import ( + "bytes" + "sort" + "strings" + + "github.com/magiconair/properties" + "github.com/spf13/cast" +) + +// Codec implements the encoding.Encoder and encoding.Decoder interfaces for Java properties encoding. +type Codec struct { + KeyDelimiter string +} + +func (c Codec) Encode(v map[string]interface{}) ([]byte, error) { + p := properties.NewProperties() + + flattened := map[string]interface{}{} + + flattened = flattenAndMergeMap(flattened, v, "", c.keyDelimiter()) + + keys := make([]string, 0, len(flattened)) + + for key := range flattened { + keys = append(keys, key) + } + + sort.Strings(keys) + + for _, key := range keys { + _, _, err := p.Set(key, cast.ToString(flattened[key])) + if err != nil { + return nil, err + } + } + + var buf bytes.Buffer + + _, err := p.WriteComment(&buf, "#", properties.UTF8) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func (c Codec) Decode(b []byte, v map[string]interface{}) error { + p, err := properties.Load(b, properties.UTF8) + if err != nil { + return err + } + + for _, key := range p.Keys() { + // ignore existence check: we know it's there + value, _ := p.Get(key) + + // recursively build nested maps + path := strings.Split(key, c.keyDelimiter()) + lastKey := strings.ToLower(path[len(path)-1]) + deepestMap := deepSearch(v, path[0:len(path)-1]) + + // set innermost value + deepestMap[lastKey] = value + } + + return nil +} + +func (c Codec) keyDelimiter() string { + if c.KeyDelimiter == "" { + return "." + } + + return c.KeyDelimiter +} diff --git a/internal/encoding/javaproperties/codec_test.go b/internal/encoding/javaproperties/codec_test.go new file mode 100644 index 000000000..80097078c --- /dev/null +++ b/internal/encoding/javaproperties/codec_test.go @@ -0,0 +1,69 @@ +package javaproperties + +import ( + "reflect" + "testing" +) + +// original form of the data +const original = `# key-value pair +key = value +map.key = value +` + +// encoded form of the data +const encoded = `key = value +map.key = value +` + +// Viper's internal representation +var data = map[string]interface{}{ + "key": "value", + "map": map[string]interface{}{ + "key": "value", + }, +} + +func TestCodec_Encode(t *testing.T) { + codec := Codec{} + + b, err := codec.Encode(data) + if err != nil { + t.Fatal(err) + } + + if encoded != string(b) { + t.Fatalf("decoded value does not match the expected one\nactual: %#v\nexpected: %#v", string(b), encoded) + } +} + +func TestCodec_Decode(t *testing.T) { + t.Run("OK", func(t *testing.T) { + codec := Codec{} + + v := map[string]interface{}{} + + err := codec.Decode([]byte(original), v) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(data, v) { + t.Fatalf("decoded value does not match the expected one\nactual: %#v\nexpected: %#v", v, data) + } + }) + + t.Run("InvalidData", func(t *testing.T) { + t.Skip("TODO: needs invalid data example") + + codec := Codec{} + + v := map[string]interface{}{} + + codec.Decode([]byte(``), v) + + if len(v) > 0 { + t.Fatalf("expected map to be empty when data is invalid\nactual: %#v", v) + } + }) +} diff --git a/internal/encoding/javaproperties/map_utils.go b/internal/encoding/javaproperties/map_utils.go new file mode 100644 index 000000000..93755cac1 --- /dev/null +++ b/internal/encoding/javaproperties/map_utils.go @@ -0,0 +1,74 @@ +package javaproperties + +import ( + "strings" + + "github.com/spf13/cast" +) + +// THIS CODE IS COPIED HERE: IT SHOULD NOT BE MODIFIED +// AT SOME POINT IT WILL BE MOVED TO A COMMON PLACE +// deepSearch scans deep maps, following the key indexes listed in the +// sequence "path". +// The last value is expected to be another map, and is returned. +// +// In case intermediate keys do not exist, or map to a non-map value, +// a new map is created and inserted, and the search continues from there: +// the initial map "m" may be modified! +func deepSearch(m map[string]interface{}, path []string) map[string]interface{} { + for _, k := range path { + m2, ok := m[k] + if !ok { + // intermediate key does not exist + // => create it and continue from there + m3 := make(map[string]interface{}) + m[k] = m3 + m = m3 + continue + } + m3, ok := m2.(map[string]interface{}) + if !ok { + // intermediate key is a value + // => replace with a new map + m3 = make(map[string]interface{}) + m[k] = m3 + } + // continue search from here + m = m3 + } + return m +} + +// flattenAndMergeMap recursively flattens the given map into a new map +// Code is based on the function with the same name in tha main package. +// TODO: move it to a common place +func flattenAndMergeMap(shadow map[string]interface{}, m map[string]interface{}, prefix string, delimiter string) map[string]interface{} { + if shadow != nil && prefix != "" && shadow[prefix] != nil { + // prefix is shadowed => nothing more to flatten + return shadow + } + if shadow == nil { + shadow = make(map[string]interface{}) + } + + var m2 map[string]interface{} + if prefix != "" { + prefix += delimiter + } + for k, val := range m { + fullKey := prefix + k + switch val.(type) { + case map[string]interface{}: + m2 = val.(map[string]interface{}) + case map[interface{}]interface{}: + m2 = cast.ToStringMap(val) + default: + // immediate value + shadow[strings.ToLower(fullKey)] = val + continue + } + // recursively merge to shadow map + shadow = flattenAndMergeMap(shadow, m2, fullKey, delimiter) + } + return shadow +} From fb6c6788e0537bc2d139a1dcf4ae5bce3fa4a826 Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Fri, 16 Jul 2021 03:51:02 +0200 Subject: [PATCH 08/10] feat(encoding): integrate Java properties codec into Viper Signed-off-by: Mark Sagi-Kazar --- internal/encoding/javaproperties/codec.go | 25 +++++--- .../encoding/javaproperties/codec_test.go | 22 ++++++- viper.go | 57 ++++++------------- 3 files changed, 56 insertions(+), 48 deletions(-) diff --git a/internal/encoding/javaproperties/codec.go b/internal/encoding/javaproperties/codec.go index ac3b66b68..b8a2251c1 100644 --- a/internal/encoding/javaproperties/codec.go +++ b/internal/encoding/javaproperties/codec.go @@ -12,10 +12,18 @@ import ( // Codec implements the encoding.Encoder and encoding.Decoder interfaces for Java properties encoding. type Codec struct { KeyDelimiter string + + // Store read properties on the object so that we can write back in order with comments. + // This will only be used if the configuration read is a properties file. + // TODO: drop this feature in v2 + // TODO: make use of the global properties object optional + Properties *properties.Properties } -func (c Codec) Encode(v map[string]interface{}) ([]byte, error) { - p := properties.NewProperties() +func (c *Codec) Encode(v map[string]interface{}) ([]byte, error) { + if c.Properties == nil { + c.Properties = properties.NewProperties() + } flattened := map[string]interface{}{} @@ -30,7 +38,7 @@ func (c Codec) Encode(v map[string]interface{}) ([]byte, error) { sort.Strings(keys) for _, key := range keys { - _, _, err := p.Set(key, cast.ToString(flattened[key])) + _, _, err := c.Properties.Set(key, cast.ToString(flattened[key])) if err != nil { return nil, err } @@ -38,7 +46,7 @@ func (c Codec) Encode(v map[string]interface{}) ([]byte, error) { var buf bytes.Buffer - _, err := p.WriteComment(&buf, "#", properties.UTF8) + _, err := c.Properties.WriteComment(&buf, "#", properties.UTF8) if err != nil { return nil, err } @@ -46,15 +54,16 @@ func (c Codec) Encode(v map[string]interface{}) ([]byte, error) { return buf.Bytes(), nil } -func (c Codec) Decode(b []byte, v map[string]interface{}) error { - p, err := properties.Load(b, properties.UTF8) +func (c *Codec) Decode(b []byte, v map[string]interface{}) error { + var err error + c.Properties, err = properties.Load(b, properties.UTF8) if err != nil { return err } - for _, key := range p.Keys() { + for _, key := range c.Properties.Keys() { // ignore existence check: we know it's there - value, _ := p.Get(key) + value, _ := c.Properties.Get(key) // recursively build nested maps path := strings.Split(key, c.keyDelimiter()) diff --git a/internal/encoding/javaproperties/codec_test.go b/internal/encoding/javaproperties/codec_test.go index 80097078c..0a33ebfdb 100644 --- a/internal/encoding/javaproperties/codec_test.go +++ b/internal/encoding/javaproperties/codec_test.go @@ -6,7 +6,7 @@ import ( ) // original form of the data -const original = `# key-value pair +const original = `#key-value pair key = value map.key = value ` @@ -67,3 +67,23 @@ func TestCodec_Decode(t *testing.T) { } }) } + +func TestCodec_DecodeEncode(t *testing.T) { + codec := Codec{} + + v := map[string]interface{}{} + + err := codec.Decode([]byte(original), v) + if err != nil { + t.Fatal(err) + } + + b, err := codec.Encode(data) + if err != nil { + t.Fatal(err) + } + + if original != string(b) { + t.Fatalf("encoded value does not match the original\nactual: %#v\nexpected: %#v", string(b), original) + } +} diff --git a/viper.go b/viper.go index 0c10093f2..88450c3e9 100644 --- a/viper.go +++ b/viper.go @@ -35,7 +35,6 @@ import ( "time" "github.com/fsnotify/fsnotify" - "github.com/magiconair/properties" "github.com/mitchellh/mapstructure" "github.com/spf13/afero" "github.com/spf13/cast" @@ -45,6 +44,7 @@ import ( "github.com/spf13/viper/internal/encoding" "github.com/spf13/viper/internal/encoding/hcl" "github.com/spf13/viper/internal/encoding/ini" + "github.com/spf13/viper/internal/encoding/javaproperties" "github.com/spf13/viper/internal/encoding/json" "github.com/spf13/viper/internal/encoding/toml" "github.com/spf13/viper/internal/encoding/yaml" @@ -215,10 +215,6 @@ type Viper struct { aliases map[string]string typeByDefValue bool - // Store read properties on the object so that we can write back in order with comments. - // This will only be used if the configuration read is a properties file. - properties *properties.Properties - onConfigChange func(fsnotify.Event) logger Logger @@ -356,6 +352,21 @@ func (v *Viper) resetEncoding() { decoderRegistry.RegisterDecoder("ini", codec) } + { + codec := &javaproperties.Codec{ + KeyDelimiter: v.keyDelim, + } + + encoderRegistry.RegisterEncoder("properties", codec) + decoderRegistry.RegisterDecoder("properties", codec) + + encoderRegistry.RegisterEncoder("props", codec) + decoderRegistry.RegisterDecoder("props", codec) + + encoderRegistry.RegisterEncoder("prop", codec) + decoderRegistry.RegisterDecoder("prop", codec) + } + v.encoderRegistry = encoderRegistry v.decoderRegistry = decoderRegistry } @@ -1656,7 +1667,7 @@ func (v *Viper) unmarshalReader(in io.Reader, c map[string]interface{}) error { buf.ReadFrom(in) switch format := strings.ToLower(v.getConfigType()); format { - case "yaml", "yml", "json", "toml", "hcl", "tfvars", "ini": + case "yaml", "yml", "json", "toml", "hcl", "tfvars", "ini", "properties", "props", "prop": err := v.decoderRegistry.Decode(format, buf.Bytes(), c) if err != nil { return ConfigParseError{err} @@ -1670,22 +1681,6 @@ func (v *Viper) unmarshalReader(in io.Reader, c map[string]interface{}) error { for k, v := range env { c[k] = v } - - case "properties", "props", "prop": - v.properties = properties.NewProperties() - var err error - if v.properties, err = properties.Load(buf.Bytes(), properties.UTF8); err != nil { - return ConfigParseError{err} - } - for _, key := range v.properties.Keys() { - value, _ := v.properties.Get(key) - // recursively build nested maps - path := strings.Split(key, ".") - lastKey := strings.ToLower(path[len(path)-1]) - deepestMap := deepSearch(c, path[0:len(path)-1]) - // set innermost value - deepestMap[lastKey] = value - } } insensitiviseMap(c) @@ -1696,7 +1691,7 @@ func (v *Viper) unmarshalReader(in io.Reader, c map[string]interface{}) error { func (v *Viper) marshalWriter(f afero.File, configType string) error { c := v.AllSettings() switch configType { - case "yaml", "yml", "json", "toml", "hcl", "tfvars", "ini": + case "yaml", "yml", "json", "toml", "hcl", "tfvars", "ini", "prop", "props", "properties": b, err := v.encoderRegistry.Encode(configType, c) if err != nil { return ConfigMarshalError{err} @@ -1707,22 +1702,6 @@ func (v *Viper) marshalWriter(f afero.File, configType string) error { return ConfigMarshalError{err} } - case "prop", "props", "properties": - if v.properties == nil { - v.properties = properties.NewProperties() - } - p := v.properties - for _, key := range v.AllKeys() { - _, _, err := p.Set(key, v.GetString(key)) - if err != nil { - return ConfigMarshalError{err} - } - } - _, err := p.WriteComment(f, "#", properties.UTF8) - if err != nil { - return ConfigMarshalError{err} - } - case "dotenv", "env": lines := []string{} for _, key := range v.AllKeys() { From 9aa08ab2e637d6468c1484b3775f60b25b22ff4a Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Tue, 20 Jul 2021 01:04:13 +0200 Subject: [PATCH 09/10] feat(encoding): add dotenv codec Signed-off-by: Mark Sagi-Kazar --- internal/encoding/dotenv/codec.go | 61 +++++++++++++++++++++++++ internal/encoding/dotenv/codec_test.go | 63 ++++++++++++++++++++++++++ internal/encoding/dotenv/map_utils.go | 41 +++++++++++++++++ 3 files changed, 165 insertions(+) create mode 100644 internal/encoding/dotenv/codec.go create mode 100644 internal/encoding/dotenv/codec_test.go create mode 100644 internal/encoding/dotenv/map_utils.go diff --git a/internal/encoding/dotenv/codec.go b/internal/encoding/dotenv/codec.go new file mode 100644 index 000000000..4485063b6 --- /dev/null +++ b/internal/encoding/dotenv/codec.go @@ -0,0 +1,61 @@ +package dotenv + +import ( + "bytes" + "fmt" + "sort" + "strings" + + "github.com/subosito/gotenv" +) + +const keyDelimiter = "_" + +// Codec implements the encoding.Encoder and encoding.Decoder interfaces for encoding data containing environment variables +// (commonly called as dotenv format). +type Codec struct{} + +func (Codec) Encode(v map[string]interface{}) ([]byte, error) { + flattened := map[string]interface{}{} + + flattened = flattenAndMergeMap(flattened, v, "", keyDelimiter) + + keys := make([]string, 0, len(flattened)) + + for key := range flattened { + keys = append(keys, key) + } + + sort.Strings(keys) + + var buf bytes.Buffer + + for _, key := range keys { + _, err := buf.WriteString(fmt.Sprintf("%v=%v\n", strings.ToUpper(key), flattened[key])) + if err != nil { + return nil, err + } + } + + return buf.Bytes(), nil +} + +func (Codec) Decode(b []byte, v map[string]interface{}) error { + var buf bytes.Buffer + + _, err := buf.Write(b) + if err != nil { + return err + } + + env, err := gotenv.StrictParse(&buf) + if err != nil { + return err + } + + for key, value := range env { + v[key] = value + } + + return nil +} diff --git a/internal/encoding/dotenv/codec_test.go b/internal/encoding/dotenv/codec_test.go new file mode 100644 index 000000000..d297071c1 --- /dev/null +++ b/internal/encoding/dotenv/codec_test.go @@ -0,0 +1,63 @@ +package dotenv + +import ( + "reflect" + "testing" +) + +// original form of the data +const original = `# key-value pair +KEY=value +` + +// encoded form of the data +const encoded = `KEY=value +` + +// Viper's internal representation +var data = map[string]interface{}{ + "KEY": "value", +} + +func TestCodec_Encode(t *testing.T) { + codec := Codec{} + + b, err := codec.Encode(data) + if err != nil { + t.Fatal(err) + } + + if encoded != string(b) { + t.Fatalf("decoded value does not match the expected one\nactual: %#v\nexpected: %#v", string(b), encoded) + } +} + +func TestCodec_Decode(t *testing.T) { + t.Run("OK", func(t *testing.T) { + codec := Codec{} + + v := map[string]interface{}{} + + err := codec.Decode([]byte(original), v) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(data, v) { + t.Fatalf("decoded value does not match the expected one\nactual: %#v\nexpected: %#v", v, data) + } + }) + + t.Run("InvalidData", func(t *testing.T) { + codec := Codec{} + + v := map[string]interface{}{} + + err := codec.Decode([]byte(`invalid data`), v) + if err == nil { + t.Fatal("expected decoding to fail") + } + + t.Logf("decoding failed as expected: %s", err) + }) +} diff --git a/internal/encoding/dotenv/map_utils.go b/internal/encoding/dotenv/map_utils.go new file mode 100644 index 000000000..ce6e6efa3 --- /dev/null +++ b/internal/encoding/dotenv/map_utils.go @@ -0,0 +1,41 @@ +package dotenv + +import ( + "strings" + + "github.com/spf13/cast" +) + +// flattenAndMergeMap recursively flattens the given map into a new map +// Code is based on the function with the same name in tha main package. +// TODO: move it to a common place +func flattenAndMergeMap(shadow map[string]interface{}, m map[string]interface{}, prefix string, delimiter string) map[string]interface{} { + if shadow != nil && prefix != "" && shadow[prefix] != nil { + // prefix is shadowed => nothing more to flatten + return shadow + } + if shadow == nil { + shadow = make(map[string]interface{}) + } + + var m2 map[string]interface{} + if prefix != "" { + prefix += delimiter + } + for k, val := range m { + fullKey := prefix + k + switch val.(type) { + case map[string]interface{}: + m2 = val.(map[string]interface{}) + case map[interface{}]interface{}: + m2 = cast.ToStringMap(val) + default: + // immediate value + shadow[strings.ToLower(fullKey)] = val + continue + } + // recursively merge to shadow map + shadow = flattenAndMergeMap(shadow, m2, fullKey, delimiter) + } + return shadow +} From aa10a085e4e456ebf0927b634ff683e143bcd28a Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Tue, 20 Jul 2021 01:46:45 +0200 Subject: [PATCH 10/10] feat(encoding): integrate dotenv codec into Viper Signed-off-by: Mark Sagi-Kazar --- viper.go | 37 +++++++++++++------------------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/viper.go b/viper.go index 88450c3e9..a5188208d 100644 --- a/viper.go +++ b/viper.go @@ -39,9 +39,9 @@ import ( "github.com/spf13/afero" "github.com/spf13/cast" "github.com/spf13/pflag" - "github.com/subosito/gotenv" "github.com/spf13/viper/internal/encoding" + "github.com/spf13/viper/internal/encoding/dotenv" "github.com/spf13/viper/internal/encoding/hcl" "github.com/spf13/viper/internal/encoding/ini" "github.com/spf13/viper/internal/encoding/javaproperties" @@ -367,6 +367,16 @@ func (v *Viper) resetEncoding() { decoderRegistry.RegisterDecoder("prop", codec) } + { + codec := &dotenv.Codec{} + + encoderRegistry.RegisterEncoder("dotenv", codec) + decoderRegistry.RegisterDecoder("dotenv", codec) + + encoderRegistry.RegisterEncoder("env", codec) + decoderRegistry.RegisterDecoder("env", codec) + } + v.encoderRegistry = encoderRegistry v.decoderRegistry = decoderRegistry } @@ -1667,20 +1677,11 @@ func (v *Viper) unmarshalReader(in io.Reader, c map[string]interface{}) error { buf.ReadFrom(in) switch format := strings.ToLower(v.getConfigType()); format { - case "yaml", "yml", "json", "toml", "hcl", "tfvars", "ini", "properties", "props", "prop": + case "yaml", "yml", "json", "toml", "hcl", "tfvars", "ini", "properties", "props", "prop", "dotenv", "env": err := v.decoderRegistry.Decode(format, buf.Bytes(), c) if err != nil { return ConfigParseError{err} } - - case "dotenv", "env": - env, err := gotenv.StrictParse(buf) - if err != nil { - return ConfigParseError{err} - } - for k, v := range env { - c[k] = v - } } insensitiviseMap(c) @@ -1691,7 +1692,7 @@ func (v *Viper) unmarshalReader(in io.Reader, c map[string]interface{}) error { func (v *Viper) marshalWriter(f afero.File, configType string) error { c := v.AllSettings() switch configType { - case "yaml", "yml", "json", "toml", "hcl", "tfvars", "ini", "prop", "props", "properties": + case "yaml", "yml", "json", "toml", "hcl", "tfvars", "ini", "prop", "props", "properties", "dotenv", "env": b, err := v.encoderRegistry.Encode(configType, c) if err != nil { return ConfigMarshalError{err} @@ -1701,18 +1702,6 @@ func (v *Viper) marshalWriter(f afero.File, configType string) error { if err != nil { return ConfigMarshalError{err} } - - case "dotenv", "env": - lines := []string{} - for _, key := range v.AllKeys() { - envName := strings.ToUpper(strings.Replace(key, ".", "_", -1)) - val := v.Get(key) - lines = append(lines, fmt.Sprintf("%v=%v", envName, val)) - } - s := strings.Join(lines, "\n") - if _, err := f.WriteString(s); err != nil { - return ConfigMarshalError{err} - } } return nil }