From 71dfab651b1eeec300bc985230b8999a8c100bba Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Thu, 28 Jul 2022 16:06:44 +0100 Subject: [PATCH] Allow ignoring undefined attributes during unmarshalling (#213) * Adding alternative implementation of Unmarshal which allows supplying of opts which can be used to modify the unmarshalling behaviour, such as ignoring undefined attributes (#212) * Upper case JSONOpts field (#212) * Code review changes (#212) * Apply suggestions from code review Co-authored-by: Brian Flad Co-authored-by: Brian Flad --- .changelog/213.txt | 7 ++++ tfprotov5/state.go | 19 +++++++++ tfprotov5/state_test.go | 83 ++++++++++++++++++++++++++++++++++++++ tfprotov6/state.go | 19 +++++++++ tfprotov6/state_test.go | 83 ++++++++++++++++++++++++++++++++++++++ tftypes/value_json.go | 66 ++++++++++++++++++++---------- tftypes/value_json_test.go | 62 ++++++++++++++++++++++++++++ 7 files changed, 318 insertions(+), 21 deletions(-) create mode 100644 .changelog/213.txt create mode 100644 tfprotov5/state_test.go create mode 100644 tfprotov6/state_test.go diff --git a/.changelog/213.txt b/.changelog/213.txt new file mode 100644 index 00000000..284d5a2f --- /dev/null +++ b/.changelog/213.txt @@ -0,0 +1,7 @@ +```release-note:enhancement +tfprotov5: Added `RawState` type `UnmarshalWithOpts` method to facilitate configurable behaviour during unmarshalling +``` + +```release-note:enhancement +tfprotov6: Added `RawState` type `UnmarshalWithOpts` method to facilitate configurable behaviour during unmarshalling +``` diff --git a/tfprotov5/state.go b/tfprotov5/state.go index 7b7de2f7..1f85dcad 100644 --- a/tfprotov5/state.go +++ b/tfprotov5/state.go @@ -77,3 +77,22 @@ func (s RawState) Unmarshal(typ tftypes.Type) (tftypes.Value, error) { } return tftypes.Value{}, ErrUnknownRawStateType } + +// UnmarshalOpts contains options that can be used to modify the behaviour when +// unmarshalling. Currently, this only contains a struct for opts for JSON but +// could have a field for Flatmap in the future. +type UnmarshalOpts struct { + ValueFromJSONOpts tftypes.ValueFromJSONOpts +} + +// UnmarshalWithOpts is identical to Unmarshal but also accepts a tftypes.UnmarshalOpts which contains +// options that can be used to modify the behaviour when unmarshalling JSON or Flatmap. +func (s RawState) UnmarshalWithOpts(typ tftypes.Type, opts UnmarshalOpts) (tftypes.Value, error) { + if s.JSON != nil { + return tftypes.ValueFromJSONWithOpts(s.JSON, typ, opts.ValueFromJSONOpts) //nolint:staticcheck + } + if s.Flatmap != nil { + return tftypes.Value{}, fmt.Errorf("flatmap states cannot be unmarshaled, only states written by Terraform 0.12 and higher can be unmarshaled") + } + return tftypes.Value{}, ErrUnknownRawStateType +} diff --git a/tfprotov5/state_test.go b/tfprotov5/state_test.go new file mode 100644 index 00000000..e4ad0fac --- /dev/null +++ b/tfprotov5/state_test.go @@ -0,0 +1,83 @@ +package tfprotov5_test + +import ( + "math/big" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/hashicorp/terraform-plugin-go/tfprotov5" + "github.com/hashicorp/terraform-plugin-go/tftypes" +) + +func TestRawStateUnmarshalWithOpts(t *testing.T) { + t.Parallel() + type testCase struct { + rawState tfprotov5.RawState + value tftypes.Value + typ tftypes.Type + opts tfprotov5.UnmarshalOpts + } + tests := map[string]testCase{ + "object-of-bool-number": { + rawState: tfprotov5.RawState{ + JSON: []byte(`{"bool":true,"number":0}`), + }, + value: tftypes.NewValue(tftypes.Object{ + AttributeTypes: map[string]tftypes.Type{ + "bool": tftypes.Bool, + "number": tftypes.Number, + }, + }, map[string]tftypes.Value{ + "bool": tftypes.NewValue(tftypes.Bool, true), + "number": tftypes.NewValue(tftypes.Number, big.NewFloat(0)), + }), + typ: tftypes.Object{ + AttributeTypes: map[string]tftypes.Type{ + "bool": tftypes.Bool, + "number": tftypes.Number, + }, + }, + }, + "object-with-missing-attribute": { + rawState: tfprotov5.RawState{ + JSON: []byte(`{"bool":true,"number":0,"unknown":"whatever"}`), + }, + value: tftypes.NewValue(tftypes.Object{ + AttributeTypes: map[string]tftypes.Type{ + "bool": tftypes.Bool, + "number": tftypes.Number, + }, + }, map[string]tftypes.Value{ + "bool": tftypes.NewValue(tftypes.Bool, true), + "number": tftypes.NewValue(tftypes.Number, big.NewFloat(0)), + }), + typ: tftypes.Object{ + AttributeTypes: map[string]tftypes.Type{ + "bool": tftypes.Bool, + "number": tftypes.Number, + }, + }, + opts: tfprotov5.UnmarshalOpts{ + ValueFromJSONOpts: tftypes.ValueFromJSONOpts{ + IgnoreUndefinedAttributes: true, + }, + }, + }, + } + for name, test := range tests { + name, test := name, test + t.Run(name, func(t *testing.T) { + t.Parallel() + + val, err := test.rawState.UnmarshalWithOpts(test.typ, test.opts) + if err != nil { + t.Fatalf("unexpected error unmarshaling: %s", err) + } + + if diff := cmp.Diff(test.value, val); diff != "" { + t.Errorf("Unexpected results (-wanted +got): %s", diff) + } + }) + } +} diff --git a/tfprotov6/state.go b/tfprotov6/state.go index 3b5d9b11..41506b8d 100644 --- a/tfprotov6/state.go +++ b/tfprotov6/state.go @@ -77,3 +77,22 @@ func (s RawState) Unmarshal(typ tftypes.Type) (tftypes.Value, error) { } return tftypes.Value{}, ErrUnknownRawStateType } + +// UnmarshalOpts contains options that can be used to modify the behaviour when +// unmarshalling. Currently, this only contains a struct for opts for JSON but +// could have a field for Flatmap in the future. +type UnmarshalOpts struct { + ValueFromJSONOpts tftypes.ValueFromJSONOpts +} + +// UnmarshalWithOpts is identical to Unmarshal but also accepts a tftypes.UnmarshalOpts which contains +// options that can be used to modify the behaviour when unmarshalling JSON or Flatmap. +func (s RawState) UnmarshalWithOpts(typ tftypes.Type, opts UnmarshalOpts) (tftypes.Value, error) { + if s.JSON != nil { + return tftypes.ValueFromJSONWithOpts(s.JSON, typ, opts.ValueFromJSONOpts) //nolint:staticcheck + } + if s.Flatmap != nil { + return tftypes.Value{}, fmt.Errorf("flatmap states cannot be unmarshaled, only states written by Terraform 0.12 and higher can be unmarshaled") + } + return tftypes.Value{}, ErrUnknownRawStateType +} diff --git a/tfprotov6/state_test.go b/tfprotov6/state_test.go new file mode 100644 index 00000000..4285b59c --- /dev/null +++ b/tfprotov6/state_test.go @@ -0,0 +1,83 @@ +package tfprotov6_test + +import ( + "math/big" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/hashicorp/terraform-plugin-go/tfprotov6" + "github.com/hashicorp/terraform-plugin-go/tftypes" +) + +func TestRawStateUnmarshalWithOpts(t *testing.T) { + t.Parallel() + type testCase struct { + rawState tfprotov6.RawState + value tftypes.Value + typ tftypes.Type + opts tfprotov6.UnmarshalOpts + } + tests := map[string]testCase{ + "object-of-bool-number": { + rawState: tfprotov6.RawState{ + JSON: []byte(`{"bool":true,"number":0}`), + }, + value: tftypes.NewValue(tftypes.Object{ + AttributeTypes: map[string]tftypes.Type{ + "bool": tftypes.Bool, + "number": tftypes.Number, + }, + }, map[string]tftypes.Value{ + "bool": tftypes.NewValue(tftypes.Bool, true), + "number": tftypes.NewValue(tftypes.Number, big.NewFloat(0)), + }), + typ: tftypes.Object{ + AttributeTypes: map[string]tftypes.Type{ + "bool": tftypes.Bool, + "number": tftypes.Number, + }, + }, + }, + "object-with-missing-attribute": { + rawState: tfprotov6.RawState{ + JSON: []byte(`{"bool":true,"number":0,"unknown":"whatever"}`), + }, + value: tftypes.NewValue(tftypes.Object{ + AttributeTypes: map[string]tftypes.Type{ + "bool": tftypes.Bool, + "number": tftypes.Number, + }, + }, map[string]tftypes.Value{ + "bool": tftypes.NewValue(tftypes.Bool, true), + "number": tftypes.NewValue(tftypes.Number, big.NewFloat(0)), + }), + typ: tftypes.Object{ + AttributeTypes: map[string]tftypes.Type{ + "bool": tftypes.Bool, + "number": tftypes.Number, + }, + }, + opts: tfprotov6.UnmarshalOpts{ + ValueFromJSONOpts: tftypes.ValueFromJSONOpts{ + IgnoreUndefinedAttributes: true, + }, + }, + }, + } + for name, test := range tests { + name, test := name, test + t.Run(name, func(t *testing.T) { + t.Parallel() + + val, err := test.rawState.UnmarshalWithOpts(test.typ, test.opts) + if err != nil { + t.Fatalf("unexpected error unmarshaling: %s", err) + } + + if diff := cmp.Diff(test.value, val); diff != "" { + t.Errorf("Unexpected results (-wanted +got): %s", diff) + } + }) + } +} diff --git a/tftypes/value_json.go b/tftypes/value_json.go index 30843477..d21c880f 100644 --- a/tftypes/value_json.go +++ b/tftypes/value_json.go @@ -16,7 +16,24 @@ import ( // terraform-plugin-go. Third parties should not use it, and its behavior is // not covered under the API compatibility guarantees. Don't use this. func ValueFromJSON(data []byte, typ Type) (Value, error) { - return jsonUnmarshal(data, typ, NewAttributePath()) + return jsonUnmarshal(data, typ, NewAttributePath(), ValueFromJSONOpts{}) +} + +// ValueFromJSONOpts contains options that can be used to modify the behaviour when +// unmarshalling JSON. +type ValueFromJSONOpts struct { + // IgnoreUndefinedAttributes is used to ignore any attributes which appear in the + // JSON but do not have a corresponding entry in the schema. For example, raw state + // where an attribute has been removed from the schema. + IgnoreUndefinedAttributes bool +} + +// ValueFromJSONWithOpts is identical to ValueFromJSON with the exception that it +// accepts ValueFromJSONOpts which can be used to modify the unmarshalling behaviour, such +// as ignoring undefined attributes, for instance. This can occur when the JSON +// being unmarshalled does not have a corresponding attribute in the schema. +func ValueFromJSONWithOpts(data []byte, typ Type, opts ValueFromJSONOpts) (Value, error) { + return jsonUnmarshal(data, typ, NewAttributePath(), opts) } func jsonByteDecoder(buf []byte) *json.Decoder { @@ -26,7 +43,7 @@ func jsonByteDecoder(buf []byte) *json.Decoder { return dec } -func jsonUnmarshal(buf []byte, typ Type, p *AttributePath) (Value, error) { +func jsonUnmarshal(buf []byte, typ Type, p *AttributePath, opts ValueFromJSONOpts) (Value, error) { dec := jsonByteDecoder(buf) tok, err := dec.Token() @@ -46,18 +63,17 @@ func jsonUnmarshal(buf []byte, typ Type, p *AttributePath) (Value, error) { case typ.Is(Bool): return jsonUnmarshalBool(buf, typ, p) case typ.Is(DynamicPseudoType): - return jsonUnmarshalDynamicPseudoType(buf, typ, p) + return jsonUnmarshalDynamicPseudoType(buf, typ, p, opts) case typ.Is(List{}): - return jsonUnmarshalList(buf, typ.(List).ElementType, p) + return jsonUnmarshalList(buf, typ.(List).ElementType, p, opts) case typ.Is(Set{}): - return jsonUnmarshalSet(buf, typ.(Set).ElementType, p) - + return jsonUnmarshalSet(buf, typ.(Set).ElementType, p, opts) case typ.Is(Map{}): - return jsonUnmarshalMap(buf, typ.(Map).ElementType, p) + return jsonUnmarshalMap(buf, typ.(Map).ElementType, p, opts) case typ.Is(Tuple{}): - return jsonUnmarshalTuple(buf, typ.(Tuple).ElementTypes, p) + return jsonUnmarshalTuple(buf, typ.(Tuple).ElementTypes, p, opts) case typ.Is(Object{}): - return jsonUnmarshalObject(buf, typ.(Object).AttributeTypes, p) + return jsonUnmarshalObject(buf, typ.(Object).AttributeTypes, p, opts) } return Value{}, p.NewErrorf("unknown type %s", typ) } @@ -140,7 +156,7 @@ func jsonUnmarshalBool(buf []byte, _ Type, p *AttributePath) (Value, error) { return Value{}, p.NewErrorf("unsupported type %T sent as %s", tok, Bool) } -func jsonUnmarshalDynamicPseudoType(buf []byte, _ Type, p *AttributePath) (Value, error) { +func jsonUnmarshalDynamicPseudoType(buf []byte, _ Type, p *AttributePath, opts ValueFromJSONOpts) (Value, error) { dec := jsonByteDecoder(buf) tok, err := dec.Token() if err != nil { @@ -190,10 +206,10 @@ func jsonUnmarshalDynamicPseudoType(buf []byte, _ Type, p *AttributePath) (Value if valBody == nil { return Value{}, p.NewErrorf("missing value in dynamically-typed value") } - return jsonUnmarshal(valBody, t, p) + return jsonUnmarshal(valBody, t, p, opts) } -func jsonUnmarshalList(buf []byte, elementType Type, p *AttributePath) (Value, error) { +func jsonUnmarshalList(buf []byte, elementType Type, p *AttributePath, opts ValueFromJSONOpts) (Value, error) { dec := jsonByteDecoder(buf) tok, err := dec.Token() @@ -227,7 +243,7 @@ func jsonUnmarshalList(buf []byte, elementType Type, p *AttributePath) (Value, e if err != nil { return Value{}, innerPath.NewErrorf("error decoding value: %w", err) } - val, err := jsonUnmarshal(rawVal, elementType, innerPath) + val, err := jsonUnmarshal(rawVal, elementType, innerPath, opts) if err != nil { return Value{}, err } @@ -254,7 +270,7 @@ func jsonUnmarshalList(buf []byte, elementType Type, p *AttributePath) (Value, e }, vals), nil } -func jsonUnmarshalSet(buf []byte, elementType Type, p *AttributePath) (Value, error) { +func jsonUnmarshalSet(buf []byte, elementType Type, p *AttributePath, opts ValueFromJSONOpts) (Value, error) { dec := jsonByteDecoder(buf) tok, err := dec.Token() @@ -284,7 +300,7 @@ func jsonUnmarshalSet(buf []byte, elementType Type, p *AttributePath) (Value, er if err != nil { return Value{}, innerPath.NewErrorf("error decoding value: %w", err) } - val, err := jsonUnmarshal(rawVal, elementType, innerPath) + val, err := jsonUnmarshal(rawVal, elementType, innerPath, opts) if err != nil { return Value{}, err } @@ -310,7 +326,7 @@ func jsonUnmarshalSet(buf []byte, elementType Type, p *AttributePath) (Value, er }, vals), nil } -func jsonUnmarshalMap(buf []byte, attrType Type, p *AttributePath) (Value, error) { +func jsonUnmarshalMap(buf []byte, attrType Type, p *AttributePath, opts ValueFromJSONOpts) (Value, error) { dec := jsonByteDecoder(buf) tok, err := dec.Token() @@ -341,7 +357,7 @@ func jsonUnmarshalMap(buf []byte, attrType Type, p *AttributePath) (Value, error if err != nil { return Value{}, innerPath.NewErrorf("error decoding value: %w", err) } - val, err := jsonUnmarshal(rawVal, attrType, innerPath) + val, err := jsonUnmarshal(rawVal, attrType, innerPath, opts) if err != nil { return Value{}, err } @@ -360,7 +376,7 @@ func jsonUnmarshalMap(buf []byte, attrType Type, p *AttributePath) (Value, error }, vals), nil } -func jsonUnmarshalTuple(buf []byte, elementTypes []Type, p *AttributePath) (Value, error) { +func jsonUnmarshalTuple(buf []byte, elementTypes []Type, p *AttributePath, opts ValueFromJSONOpts) (Value, error) { dec := jsonByteDecoder(buf) tok, err := dec.Token() @@ -398,7 +414,7 @@ func jsonUnmarshalTuple(buf []byte, elementTypes []Type, p *AttributePath) (Valu if err != nil { return Value{}, innerPath.NewErrorf("error decoding value: %w", err) } - val, err := jsonUnmarshal(rawVal, elementType, innerPath) + val, err := jsonUnmarshal(rawVal, elementType, innerPath, opts) if err != nil { return Value{}, err } @@ -422,7 +438,9 @@ func jsonUnmarshalTuple(buf []byte, elementTypes []Type, p *AttributePath) (Valu }, vals), nil } -func jsonUnmarshalObject(buf []byte, attrTypes map[string]Type, p *AttributePath) (Value, error) { +// jsonUnmarshalObject attempts to decode JSON object structure to tftypes.Value object. +// opts contains fields that can be used to modify the behaviour of JSON unmarshalling. +func jsonUnmarshalObject(buf []byte, attrTypes map[string]Type, p *AttributePath, opts ValueFromJSONOpts) (Value, error) { dec := jsonByteDecoder(buf) tok, err := dec.Token() @@ -446,6 +464,12 @@ func jsonUnmarshalObject(buf []byte, attrTypes map[string]Type, p *AttributePath } attrType, ok := attrTypes[key] if !ok { + if opts.IgnoreUndefinedAttributes { + // We are trying to ignore the key and value of any unsupported attribute. + _ = dec.Decode(new(json.RawMessage)) + continue + } + return Value{}, innerPath.NewErrorf("unsupported attribute %q", key) } innerPath = p.WithAttributeName(key) @@ -455,7 +479,7 @@ func jsonUnmarshalObject(buf []byte, attrTypes map[string]Type, p *AttributePath if err != nil { return Value{}, innerPath.NewErrorf("error decoding value: %w", err) } - val, err := jsonUnmarshal(rawVal, attrType, innerPath) + val, err := jsonUnmarshal(rawVal, attrType, innerPath, opts) if err != nil { return Value{}, err } diff --git a/tftypes/value_json_test.go b/tftypes/value_json_test.go index 4e552f41..f3253fc6 100644 --- a/tftypes/value_json_test.go +++ b/tftypes/value_json_test.go @@ -380,3 +380,65 @@ func TestValueFromJSON(t *testing.T) { }) } } + +func TestValueFromJSONWithOpts(t *testing.T) { + t.Parallel() + type testCase struct { + value Value + typ Type + json string + } + tests := map[string]testCase{ + "object-of-bool-number": { + value: NewValue(Object{ + AttributeTypes: map[string]Type{ + "bool": Bool, + "number": Number, + }, + }, map[string]Value{ + "bool": NewValue(Bool, true), + "number": NewValue(Number, big.NewFloat(0)), + }), + typ: Object{ + AttributeTypes: map[string]Type{ + "bool": Bool, + "number": Number, + }, + }, + json: `{"bool":true,"number":0}`, + }, + "object-with-missing-attribute": { + value: NewValue(Object{ + AttributeTypes: map[string]Type{ + "bool": Bool, + "number": Number, + }, + }, map[string]Value{ + "bool": NewValue(Bool, true), + "number": NewValue(Number, big.NewFloat(0)), + }), + typ: Object{ + AttributeTypes: map[string]Type{ + "bool": Bool, + "number": Number, + }, + }, + json: `{"bool":true,"number":0,"unknown":"whatever"}`, + }, + } + for name, test := range tests { + name, test := name, test + t.Run(name, func(t *testing.T) { + t.Parallel() + val, err := ValueFromJSONWithOpts([]byte(test.json), test.typ, ValueFromJSONOpts{ + IgnoreUndefinedAttributes: true, + }) + if err != nil { + t.Fatalf("unexpected error unmarshaling: %s", err) + } + if diff := cmp.Diff(test.value, val); diff != "" { + t.Errorf("Unexpected results (-wanted +got): %s", diff) + } + }) + } +}