Skip to content

Commit

Permalink
Add yaml util to unmarshal numbers into int/float
Browse files Browse the repository at this point in the history
  • Loading branch information
nodo authored and kevindelgado committed Apr 5, 2021
1 parent 5ebc98d commit 56b9595
Show file tree
Hide file tree
Showing 6 changed files with 93 additions and 17 deletions.
33 changes: 18 additions & 15 deletions staging/src/k8s.io/apimachinery/pkg/util/json/json.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ func Marshal(v interface{}) ([]byte, error) {
const maxDepth = 10000

// Unmarshal unmarshals the given data
// If v is a *map[string]interface{}, numbers are converted to int64 or float64
// If v is a *map[string]interface{}, *[]interface{}, or *interface{} numbers
// are converted to int64 or float64
func Unmarshal(data []byte, v interface{}) error {
switch v := v.(type) {
case *map[string]interface{}:
Expand All @@ -52,7 +53,7 @@ func Unmarshal(data []byte, v interface{}) error {
return err
}
// If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64
return convertMapNumbers(*v, 0)
return ConvertMapNumbers(*v, 0)

case *[]interface{}:
// Build a decoder from the given data
Expand All @@ -64,7 +65,7 @@ func Unmarshal(data []byte, v interface{}) error {
return err
}
// If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64
return convertSliceNumbers(*v, 0)
return ConvertSliceNumbers(*v, 0)

case *interface{}:
// Build a decoder from the given data
Expand All @@ -76,29 +77,31 @@ func Unmarshal(data []byte, v interface{}) error {
return err
}
// If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64
return convertInterfaceNumbers(v, 0)
return ConvertInterfaceNumbers(v, 0)

default:
return json.Unmarshal(data, v)
}
}

func convertInterfaceNumbers(v *interface{}, depth int) error {
// ConvertInterfaceNumbers converts any json.Number values to int64 or float64.
// Values which are map[string]interface{} or []interface{} are recursively visited
func ConvertInterfaceNumbers(v *interface{}, depth int) error {
var err error
switch v2 := (*v).(type) {
case json.Number:
*v, err = convertNumber(v2)
case map[string]interface{}:
err = convertMapNumbers(v2, depth+1)
err = ConvertMapNumbers(v2, depth+1)
case []interface{}:
err = convertSliceNumbers(v2, depth+1)
err = ConvertSliceNumbers(v2, depth+1)
}
return err
}

// convertMapNumbers traverses the map, converting any json.Number values to int64 or float64.
// ConvertMapNumbers traverses the map, converting any json.Number values to int64 or float64.
// values which are map[string]interface{} or []interface{} are recursively visited
func convertMapNumbers(m map[string]interface{}, depth int) error {
func ConvertMapNumbers(m map[string]interface{}, depth int) error {
if depth > maxDepth {
return fmt.Errorf("exceeded max depth of %d", maxDepth)
}
Expand All @@ -109,9 +112,9 @@ func convertMapNumbers(m map[string]interface{}, depth int) error {
case json.Number:
m[k], err = convertNumber(v)
case map[string]interface{}:
err = convertMapNumbers(v, depth+1)
err = ConvertMapNumbers(v, depth+1)
case []interface{}:
err = convertSliceNumbers(v, depth+1)
err = ConvertSliceNumbers(v, depth+1)
}
if err != nil {
return err
Expand All @@ -120,9 +123,9 @@ func convertMapNumbers(m map[string]interface{}, depth int) error {
return nil
}

// convertSliceNumbers traverses the slice, converting any json.Number values to int64 or float64.
// ConvertSliceNumbers traverses the slice, converting any json.Number values to int64 or float64.
// values which are map[string]interface{} or []interface{} are recursively visited
func convertSliceNumbers(s []interface{}, depth int) error {
func ConvertSliceNumbers(s []interface{}, depth int) error {
if depth > maxDepth {
return fmt.Errorf("exceeded max depth of %d", maxDepth)
}
Expand All @@ -133,9 +136,9 @@ func convertSliceNumbers(s []interface{}, depth int) error {
case json.Number:
s[i], err = convertNumber(v)
case map[string]interface{}:
err = convertMapNumbers(v, depth+1)
err = ConvertMapNumbers(v, depth+1)
case []interface{}:
err = convertSliceNumbers(v, depth+1)
err = ConvertSliceNumbers(v, depth+1)
}
if err != nil {
return err
Expand Down
1 change: 1 addition & 0 deletions staging/src/k8s.io/apimachinery/pkg/util/yaml/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ go_library(
importpath = "k8s.io/apimachinery/pkg/util/yaml",
deps = [
"//vendor/k8s.io/klog:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/json:go_default_library",
"//vendor/sigs.k8s.io/yaml:go_default_library",
],
)
Expand Down
31 changes: 31 additions & 0 deletions staging/src/k8s.io/apimachinery/pkg/util/yaml/decoder.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,41 @@ import (
"strings"
"unicode"

jsonutil "k8s.io/apimachinery/pkg/util/json"
"k8s.io/klog"

"sigs.k8s.io/yaml"
)

// Unmarshal unmarshals the given data
// If v is a *map[string]interface{}, *[]interface{}, or *interface{} numbers
// are converted to int64 or float64
func Unmarshal(data []byte, v interface{}) error {
preserveIntFloat := func(d *json.Decoder) *json.Decoder {
d.UseNumber()
return d
}
switch v := v.(type) {
case *map[string]interface{}:
if err := yaml.Unmarshal(data, v, preserveIntFloat); err != nil {
return err
}
return jsonutil.ConvertMapNumbers(*v, 0)
case *[]interface{}:
if err := yaml.Unmarshal(data, v, preserveIntFloat); err != nil {
return err
}
return jsonutil.ConvertSliceNumbers(*v, 0)
case *interface{}:
if err := yaml.Unmarshal(data, v, preserveIntFloat); err != nil {
return err
}
return jsonutil.ConvertInterfaceNumbers(v, 0)
default:
return yaml.Unmarshal(data, v)
}
}

// ToJSON converts a single YAML document into a JSON document
// or returns an error. If the document appears to be JSON the
// YAML decoding path is not used (so that error messages are
Expand Down
41 changes: 41 additions & 0 deletions staging/src/k8s.io/apimachinery/pkg/util/yaml/decoder_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -403,3 +403,44 @@ stuff: 1
t.Fatalf("expected %q to be of type YAMLSyntaxError", err.Error())
}
}

func TestUnmarshal(t *testing.T) {
mapWithIntegerBytes := []byte(`replicas: 1`)
mapWithInteger := make(map[string]interface{})
if err := Unmarshal(mapWithIntegerBytes, &mapWithInteger); err != nil {
t.Fatalf("unexpected error unmarshaling yaml: %v", err)
}
if _, ok := mapWithInteger["replicas"].(int64); !ok {
t.Fatalf(`Expected number in map to be int64 but got "%T"`, mapWithInteger["replicas"])
}

sliceWithIntegerBytes := []byte(`- 1`)
var sliceWithInteger []interface{}
if err := Unmarshal(sliceWithIntegerBytes, &sliceWithInteger); err != nil {
t.Fatalf("unexpected error unmarshaling yaml: %v", err)
}
if _, ok := sliceWithInteger[0].(int64); !ok {
t.Fatalf(`Expected number in slice to be int64 but got "%T"`, sliceWithInteger[0])
}

integerBytes := []byte(`1`)
var integer interface{}
if err := Unmarshal(integerBytes, &integer); err != nil {
t.Fatalf("unexpected error unmarshaling yaml: %v", err)
}
if _, ok := integer.(int64); !ok {
t.Fatalf(`Expected number to be int64 but got "%T"`, integer)
}

otherTypeBytes := []byte(`123: 2`)
otherType := make(map[int]interface{})
if err := Unmarshal(otherTypeBytes, &otherType); err != nil {
t.Fatalf("unexpected error unmarshaling yaml: %v", err)
}
if _, ok := otherType[123].(int64); ok {
t.Fatalf(`Expected number not to be converted to int64`)
}
if _, ok := otherType[123].(float64); !ok {
t.Fatalf(`Expected number to be float64 but got "%T"`, otherType[123])
}
}
2 changes: 1 addition & 1 deletion staging/src/k8s.io/apiserver/pkg/endpoints/handlers/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/strategicpatch:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/admission:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/audit:go_default_library",
Expand All @@ -107,7 +108,6 @@ go_library(
"//vendor/google.golang.org/grpc/status:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/utils/trace:go_default_library",
"//vendor/sigs.k8s.io/yaml:go_default_library",
],
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import (
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/strategicpatch"
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/apimachinery/pkg/util/yaml"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/authorization/authorizer"
Expand All @@ -49,7 +50,6 @@ import (
"k8s.io/apiserver/pkg/util/dryrun"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utiltrace "k8s.io/utils/trace"
"sigs.k8s.io/yaml"
)

const (
Expand Down

0 comments on commit 56b9595

Please sign in to comment.