From ba292d0cf9fcfc2798c156ef68773db49adf6ce4 Mon Sep 17 00:00:00 2001 From: Hengfeng Li Date: Thu, 13 May 2021 11:33:05 +1000 Subject: [PATCH 01/14] feat(spanner): add json support --- spanner/protoutils.go | 4 ++++ spanner/value.go | 27 +++++++++++++++++++++++++++ spanner/value_test.go | 23 +++++++++++++++++++++++ 3 files changed, 54 insertions(+) diff --git a/spanner/protoutils.go b/spanner/protoutils.go index 6465b2c3342..66eea4b5c29 100644 --- a/spanner/protoutils.go +++ b/spanner/protoutils.go @@ -73,6 +73,10 @@ func numericType() *sppb.Type { return &sppb.Type{Code: sppb.TypeCode_NUMERIC} } +func jsonType() *sppb.Type { + return &sppb.Type{Code: sppb.TypeCode_JSON} +} + func bytesProto(b []byte) *proto3.Value { return &proto3.Value{Kind: &proto3.Value_StringValue{StringValue: base64.StdEncoding.EncodeToString(b)}} } diff --git a/spanner/value.go b/spanner/value.go index 4f0ca8cfc37..f6e8c10f0b4 100644 --- a/spanner/value.go +++ b/spanner/value.go @@ -19,6 +19,7 @@ package spanner import ( "bytes" "encoding/base64" + "encoding/json" "fmt" "math" "math/big" @@ -1327,6 +1328,7 @@ func decodeValue(v *proto3.Value, t *sppb.Type, ptr interface{}) error { case *GenericColumnValue: *p = GenericColumnValue{Type: t, Value: v} default: + fmt.Println("coming here!!!") // Check if the pointer is a custom type that implements spanner.Decoder // interface. if decodedVal, ok := ptr.(Decoder); ok { @@ -1346,6 +1348,21 @@ func decodeValue(v *proto3.Value, t *sppb.Type, ptr interface{}) error { return decodableType.decodeValueToCustomType(v, t, acode, ptr) } + fmt.Println("coming here!!! - 1") + + if code == sppb.TypeCode_JSON { + x, err := getStringValue(v) + if err != nil { + return fmt.Errorf("failed to read json string: %s", err) + } + // Check if it can be unmarshaled to the given type. + err = json.Unmarshal([]byte(x), ptr) + if err != nil { + return fmt.Errorf("failed to unmarshal the json string: %s", err) + } + break + } + // Check if the proto encoding is for an array of structs. if !(code == sppb.TypeCode_ARRAY && acode == sppb.TypeCode_STRUCT) { return errTypeMismatch(code, acode, ptr) @@ -1358,6 +1375,7 @@ func decodeValue(v *proto3.Value, t *sppb.Type, ptr interface{}) error { // The container is not a slice of struct pointers. return fmt.Errorf("the container is not a slice of struct pointers: %v", errTypeMismatch(code, acode, ptr)) } + fmt.Println("coming here!!! - 2") // Only use reflection for nil detection on slow path. // Also, IsNil panics on many types, so check it after the type check. if vp.IsNil() { @@ -1369,6 +1387,7 @@ func decodeValue(v *proto3.Value, t *sppb.Type, ptr interface{}) error { vp.Elem().Set(reflect.Zero(vp.Elem().Type())) break } + fmt.Println("coming here!!! - 3") x, err := getListValue(v) if err != nil { return err @@ -2813,6 +2832,14 @@ func encodeValue(v interface{}) (*proto3.Value, *sppb.Type, error) { return encodeValue(converted) } + // Check if it can be marshaled to a json string. + b, err := json.Marshal(v) + if err == nil { + pb.Kind = stringKind(string(b)) + pt = jsonType() + return pb, pt, nil + } + if !isStructOrArrayOfStructValue(v) { return nil, nil, errEncoderUnsupportedType(v) } diff --git a/spanner/value_test.go b/spanner/value_test.go index 5dccbe9d738..c9cc784197d 100644 --- a/spanner/value_test.go +++ b/spanner/value_test.go @@ -213,6 +213,14 @@ func TestEncodeValue(t *testing.T) { type CustomNullDate NullDate type CustomNullNumeric NullNumeric + type Message struct { + Name string + Body string + Time int64 + } + msg := Message{"Alice", "Hello", 1294706395881547000} + jsonStr := `{"Name":"Alice","Body":"Hello","Time":1294706395881547000}` + sValue := "abc" var sNilPtr *string iValue := int64(7) @@ -238,6 +246,7 @@ func TestEncodeValue(t *testing.T) { tTime = timeType() tDate = dateType() tNumeric = numericType() + tJSON = jsonType() ) for i, test := range []struct { in interface{} @@ -304,6 +313,8 @@ func TestEncodeValue(t *testing.T) { {[]NullNumeric{{*numValuePtr, true}, {*numValuePtr, false}}, listProto(numericProto(numValuePtr), nullProto()), listType(tNumeric), "[]NullNumeric"}, {[]*big.Rat{nil, numValuePtr}, listProto(nullProto(), numericProto(numValuePtr)), listType(tNumeric), "[]*big.Rat"}, {[]*big.Rat(nil), nullProto(), listType(tNumeric), "null []*big.Rat"}, + // JSON + {msg, stringProto(jsonStr), tJSON, "json"}, // TIMESTAMP / TIMESTAMP ARRAY {t1, timeProto(t1), tTime, "time"}, {NullTime{t1, true}, timeProto(t1), tTime, "NullTime with value"}, @@ -1264,6 +1275,15 @@ func TestDecodeValue(t *testing.T) { type CustomNullDate NullDate type CustomNullNumeric NullNumeric + type Message struct { + Name string + Body string + Time int64 + } + msg := Message{"Alice", "Hello", 1294706395881547000} + jsonStr := `{"Name":"Alice","Body":"Hello","Time":1294706395881547000}` + invalidJsonStr := `{wrong_json_string}` + // Pointer values. sValue := "abc" var sNilPtr *string @@ -1381,6 +1401,9 @@ func TestDecodeValue(t *testing.T) { // NUMERIC ARRAY with []*big.Rat {desc: "decode ARRAY to []*big.Rat", proto: listProto(numericProto(numValuePtr), nullProto(), numericProto(num2ValuePtr)), protoType: listType(numericType()), want: []*big.Rat{numValuePtr, nil, num2ValuePtr}}, {desc: "decode NULL to []*big.Rat", proto: nullProto(), protoType: listType(numericType()), want: []*big.Rat(nil)}, + // JSON + {desc: "decode json to a struct", proto: stringProto(jsonStr), protoType: jsonType(), want: msg}, + {desc: "decode an invalid json string", proto: stringProto(invalidJsonStr), protoType: jsonType(), want: msg, wantErr: true}, // TIMESTAMP {desc: "decode TIMESTAMP to time.Time", proto: timeProto(t1), protoType: timeType(), want: t1}, {desc: "decode TIMESTAMP to NullTime", proto: timeProto(t1), protoType: timeType(), want: NullTime{t1, true}}, From af5413e684f3f326c2a8a8cc89eb2ceb390e70e5 Mon Sep 17 00:00:00 2001 From: Hengfeng Li Date: Thu, 13 May 2021 14:24:58 +1000 Subject: [PATCH 02/14] Add NullJSON. --- spanner/value.go | 82 ++++++++++++++++++++++++++++++++++++++++--- spanner/value_test.go | 8 ++++- 2 files changed, 84 insertions(+), 6 deletions(-) diff --git a/spanner/value.go b/spanner/value.go index f6e8c10f0b4..98406196e89 100644 --- a/spanner/value.go +++ b/spanner/value.go @@ -457,6 +457,60 @@ func (n *NullNumeric) UnmarshalJSON(payload []byte) error { return nil } +// NullJSON represents a Cloud Spanner JSON that may be NULL. +type NullJSON struct { + Value interface{} // Val contains the value when it is non-NULL, and nil when NULL. + Valid bool // Valid is true if Numeric is not NULL. +} + +// IsNull implements NullableValue.IsNull for NullJSON. +func (n NullJSON) IsNull() bool { + return !n.Valid +} + +// String implements Stringer.String for NullJSON +func (n NullJSON) String() string { + if !n.Valid { + return nullString + } + b, err := json.Marshal(n.Value) + if err != nil { + return nullString + } + return fmt.Sprintf("%v", string(b)) +} + +// MarshalJSON implements json.Marshaler.MarshalJSON for NullJSON. +func (n NullJSON) MarshalJSON() ([]byte, error) { + if n.Valid { + return json.Marshal(n.Value) + } + return jsonNullBytes, nil +} + +// UnmarshalJSON implements json.Unmarshaler.UnmarshalJSON for NullJSON. +func (n *NullJSON) UnmarshalJSON(payload []byte) error { + if payload == nil { + return fmt.Errorf("payload should not be nil") + } + if bytes.Equal(payload, jsonNullBytes) { + n.Valid = false + return nil + } + payload, err := trimDoubleQuotes(payload) + if err != nil { + return err + } + var v interface{} + err = json.Unmarshal(payload, &v) + if err != nil { + return fmt.Errorf("payload cannot be converted to a struct: got %v, err: %s", string(payload), err) + } + n.Value = v + n.Valid = true + return nil +} + // NullRow represents a Cloud Spanner STRUCT that may be NULL. // See also the document for Row. // Note that NullRow is not a valid Cloud Spanner column Type. @@ -1033,6 +1087,24 @@ func decodeValue(v *proto3.Value, t *sppb.Type, ptr interface{}) error { return errUnexpectedNumericStr(x) } *p = *y + case *NullJSON: + if p == nil { + return errNilDst(p) + } + if code != sppb.TypeCode_JSON { + return errTypeMismatch(code, acode, ptr) + } + if isNull { + *p = NullJSON{} + break + } + x := v.GetStringValue() + var y interface{} + err := json.Unmarshal([]byte(x), &y) + if err != nil { + return err + } + *p = NullJSON{y, true} case *NullNumeric: if p == nil { return errNilDst(p) @@ -1328,7 +1400,6 @@ func decodeValue(v *proto3.Value, t *sppb.Type, ptr interface{}) error { case *GenericColumnValue: *p = GenericColumnValue{Type: t, Value: v} default: - fmt.Println("coming here!!!") // Check if the pointer is a custom type that implements spanner.Decoder // interface. if decodedVal, ok := ptr.(Decoder); ok { @@ -1348,8 +1419,6 @@ func decodeValue(v *proto3.Value, t *sppb.Type, ptr interface{}) error { return decodableType.decodeValueToCustomType(v, t, acode, ptr) } - fmt.Println("coming here!!! - 1") - if code == sppb.TypeCode_JSON { x, err := getStringValue(v) if err != nil { @@ -1375,7 +1444,6 @@ func decodeValue(v *proto3.Value, t *sppb.Type, ptr interface{}) error { // The container is not a slice of struct pointers. return fmt.Errorf("the container is not a slice of struct pointers: %v", errTypeMismatch(code, acode, ptr)) } - fmt.Println("coming here!!! - 2") // Only use reflection for nil detection on slow path. // Also, IsNil panics on many types, so check it after the type check. if vp.IsNil() { @@ -1387,7 +1455,6 @@ func decodeValue(v *proto3.Value, t *sppb.Type, ptr interface{}) error { vp.Elem().Set(reflect.Zero(vp.Elem().Type())) break } - fmt.Println("coming here!!! - 3") x, err := getListValue(v) if err != nil { return err @@ -2713,6 +2780,11 @@ func encodeValue(v interface{}) (*proto3.Value, *sppb.Type, error) { } } pt = listType(numericType()) + case NullJSON: + if v.Valid { + return encodeValue(v.Value) + } + pt = jsonType() case *big.Rat: if v != nil { pb.Kind = stringKind(NumericString(v)) diff --git a/spanner/value_test.go b/spanner/value_test.go index c9cc784197d..a5c5b0bfcd1 100644 --- a/spanner/value_test.go +++ b/spanner/value_test.go @@ -315,6 +315,8 @@ func TestEncodeValue(t *testing.T) { {[]*big.Rat(nil), nullProto(), listType(tNumeric), "null []*big.Rat"}, // JSON {msg, stringProto(jsonStr), tJSON, "json"}, + {NullJSON{msg, true}, stringProto(jsonStr), tJSON, "NullJSON with value"}, + {NullJSON{msg, false}, nullProto(), tJSON, "NullJSON with null"}, // TIMESTAMP / TIMESTAMP ARRAY {t1, timeProto(t1), tTime, "time"}, {NullTime{t1, true}, timeProto(t1), tTime, "NullTime with value"}, @@ -1282,6 +1284,8 @@ func TestDecodeValue(t *testing.T) { } msg := Message{"Alice", "Hello", 1294706395881547000} jsonStr := `{"Name":"Alice","Body":"Hello","Time":1294706395881547000}` + var unmarshaledJSONstruct interface{} + json.Unmarshal([]byte(jsonStr), &unmarshaledJSONstruct) invalidJsonStr := `{wrong_json_string}` // Pointer values. @@ -1402,7 +1406,9 @@ func TestDecodeValue(t *testing.T) { {desc: "decode ARRAY to []*big.Rat", proto: listProto(numericProto(numValuePtr), nullProto(), numericProto(num2ValuePtr)), protoType: listType(numericType()), want: []*big.Rat{numValuePtr, nil, num2ValuePtr}}, {desc: "decode NULL to []*big.Rat", proto: nullProto(), protoType: listType(numericType()), want: []*big.Rat(nil)}, // JSON - {desc: "decode json to a struct", proto: stringProto(jsonStr), protoType: jsonType(), want: msg}, + {desc: "decode json to struct", proto: stringProto(jsonStr), protoType: jsonType(), want: msg}, + {desc: "decode json to NullJSON", proto: stringProto(jsonStr), protoType: jsonType(), want: NullJSON{unmarshaledJSONstruct, true}}, + {desc: "decode NULL to NullJSON", proto: nullProto(), protoType: jsonType(), want: NullJSON{}}, {desc: "decode an invalid json string", proto: stringProto(invalidJsonStr), protoType: jsonType(), want: msg, wantErr: true}, // TIMESTAMP {desc: "decode TIMESTAMP to time.Time", proto: timeProto(t1), protoType: timeType(), want: t1}, From 7786b365b283b4b144c0aaaeacdd03eb21f03bfb Mon Sep 17 00:00:00 2001 From: Hengfeng Li Date: Fri, 14 May 2021 23:04:32 +1000 Subject: [PATCH 03/14] Add array and custom types support. --- spanner/value.go | 63 +++++++++++++++++++++++++++++++++++++++++++ spanner/value_test.go | 17 ++++++++++++ 2 files changed, 80 insertions(+) diff --git a/spanner/value.go b/spanner/value.go index 98406196e89..2b22eb52c27 100644 --- a/spanner/value.go +++ b/spanner/value.go @@ -1488,6 +1488,7 @@ const ( spannerTypeNullTime spannerTypeNullDate spannerTypeNullNumeric + spannerTypeNullJSON spannerTypeArrayOfNonNullString spannerTypeArrayOfByteArray spannerTypeArrayOfNonNullInt64 @@ -1501,6 +1502,7 @@ const ( spannerTypeArrayOfNullBool spannerTypeArrayOfNullFloat64 spannerTypeArrayOfNullNumeric + spannerTypeArrayOfNullJSON spannerTypeArrayOfNullTime spannerTypeArrayOfNullDate ) @@ -1533,6 +1535,7 @@ var typeOfNullFloat64 = reflect.TypeOf(NullFloat64{}) var typeOfNullTime = reflect.TypeOf(NullTime{}) var typeOfNullDate = reflect.TypeOf(NullDate{}) var typeOfNullNumeric = reflect.TypeOf(NullNumeric{}) +var typeOfNullJSON = reflect.TypeOf(NullJSON{}) // getDecodableSpannerType returns the corresponding decodableSpannerType of // the given pointer. @@ -1564,6 +1567,9 @@ func getDecodableSpannerType(ptr interface{}, isPtr bool) decodableSpannerType { if t.ConvertibleTo(typeOfNullNumeric) { return spannerTypeNullNumeric } + if t.ConvertibleTo(typeOfNullJSON) { + return spannerTypeNullJSON + } case reflect.Struct: t := val.Type() if t.ConvertibleTo(typeOfNonNullNumeric) { @@ -1596,6 +1602,9 @@ func getDecodableSpannerType(ptr interface{}, isPtr bool) decodableSpannerType { if t.ConvertibleTo(typeOfNullNumeric) { return spannerTypeNullNumeric } + if t.ConvertibleTo(typeOfNullJSON) { + return spannerTypeNullJSON + } case reflect.Slice: kind := val.Type().Elem().Kind() switch kind { @@ -1648,6 +1657,9 @@ func getDecodableSpannerType(ptr interface{}, isPtr bool) decodableSpannerType { if t.ConvertibleTo(typeOfNullNumeric) { return spannerTypeArrayOfNullNumeric } + if t.ConvertibleTo(typeOfNullJSON) { + return spannerTypeArrayOfNullJSON + } case reflect.Slice: // The only array-of-array type that is supported is [][]byte. kind := val.Type().Elem().Elem().Kind() @@ -1783,6 +1795,21 @@ func (dsc decodableSpannerType) decodeValueToCustomType(v *proto3.Value, t *sppb } else { result = &NullNumeric{*y, true} } + case spannerTypeNullJSON: + if code != sppb.TypeCode_JSON { + return errTypeMismatch(code, acode, ptr) + } + if isNull { + result = &NullJSON{} + break + } + x := v.GetStringValue() + var y interface{} + err := json.Unmarshal([]byte(x), &y) + if err != nil { + return err + } + result = &NullJSON{y, true} case spannerTypeNonNullTime, spannerTypeNullTime: var nt NullTime err := parseNullTime(v, &nt, code, isNull) @@ -1917,6 +1944,23 @@ func (dsc decodableSpannerType) decodeValueToCustomType(v *proto3.Value, t *sppb return err } result = y + case spannerTypeArrayOfNullJSON: + if acode != sppb.TypeCode_JSON { + return errTypeMismatch(code, acode, ptr) + } + if isNull { + ptr = nil + return nil + } + x, err := getListValue(v) + if err != nil { + return err + } + y, err := decodeGenericArray(reflect.TypeOf(ptr).Elem(), x, jsonType(), "JSON") + if err != nil { + return err + } + result = y case spannerTypeArrayOfNonNullTime, spannerTypeArrayOfNullTime: if acode != sppb.TypeCode_TIMESTAMP { return errTypeMismatch(code, acode, ptr) @@ -2785,6 +2829,14 @@ func encodeValue(v interface{}) (*proto3.Value, *sppb.Type, error) { return encodeValue(v.Value) } pt = jsonType() + case []NullJSON: + if v != nil { + pb, err = encodeArray(len(v), func(i int) interface{} { return v[i] }) + if err != nil { + return nil, nil, err + } + } + pt = listType(jsonType()) case *big.Rat: if v != nil { pb.Kind = stringKind(NumericString(v)) @@ -2894,8 +2946,10 @@ func encodeValue(v interface{}) (*proto3.Value, *sppb.Type, error) { return encodeValue(nv) } + fmt.Println("coming here - 1") // Check if the value is a variant of a base type. decodableType := getDecodableSpannerType(v, false) + fmt.Printf("%T %v\n", v, decodableType) if decodableType != spannerTypeUnknown && decodableType != spannerTypeInvalid { converted, err := convertCustomTypeValue(decodableType, v) if err != nil { @@ -2903,6 +2957,7 @@ func encodeValue(v interface{}) (*proto3.Value, *sppb.Type, error) { } return encodeValue(converted) } + fmt.Println("coming here - 2") // Check if it can be marshaled to a json string. b, err := json.Marshal(v) @@ -2911,6 +2966,7 @@ func encodeValue(v interface{}) (*proto3.Value, *sppb.Type, error) { pt = jsonType() return pb, pt, nil } + fmt.Println("coming here - 3") if !isStructOrArrayOfStructValue(v) { return nil, nil, errEncoderUnsupportedType(v) @@ -2973,6 +3029,8 @@ func convertCustomTypeValue(sourceType decodableSpannerType, v interface{}) (int destination = reflect.Indirect(reflect.New(reflect.TypeOf(big.Rat{}))) case spannerTypeNullNumeric: destination = reflect.Indirect(reflect.New(reflect.TypeOf(NullNumeric{}))) + case spannerTypeNullJSON: + destination = reflect.Indirect(reflect.New(reflect.TypeOf(NullJSON{}))) case spannerTypeArrayOfNonNullString: if reflect.ValueOf(v).IsNil() { return []string(nil), nil @@ -3048,6 +3106,11 @@ func convertCustomTypeValue(sourceType decodableSpannerType, v interface{}) (int return []NullNumeric(nil), nil } destination = reflect.MakeSlice(reflect.TypeOf([]NullNumeric{}), reflect.ValueOf(v).Len(), reflect.ValueOf(v).Cap()) + case spannerTypeArrayOfNullJSON: + if reflect.ValueOf(v).IsNil() { + return []NullJSON(nil), nil + } + destination = reflect.MakeSlice(reflect.TypeOf([]NullJSON{}), reflect.ValueOf(v).Len(), reflect.ValueOf(v).Cap()) default: // This should not be possible. return nil, fmt.Errorf("unknown decodable type found: %v", sourceType) diff --git a/spanner/value_test.go b/spanner/value_test.go index a5c5b0bfcd1..31492505aae 100644 --- a/spanner/value_test.go +++ b/spanner/value_test.go @@ -212,6 +212,7 @@ func TestEncodeValue(t *testing.T) { type CustomNullTime NullTime type CustomNullDate NullDate type CustomNullNumeric NullNumeric + type CustomNullJSON NullJSON type Message struct { Name string @@ -317,6 +318,8 @@ func TestEncodeValue(t *testing.T) { {msg, stringProto(jsonStr), tJSON, "json"}, {NullJSON{msg, true}, stringProto(jsonStr), tJSON, "NullJSON with value"}, {NullJSON{msg, false}, nullProto(), tJSON, "NullJSON with null"}, + {[]NullJSON(nil), nullProto(), listType(tJSON), "null []NullJSON"}, + {[]NullJSON{{msg, true}, {msg, false}}, listProto(stringProto(jsonStr), nullProto()), listType(tJSON), "[]NullJSON"}, // TIMESTAMP / TIMESTAMP ARRAY {t1, timeProto(t1), tTime, "time"}, {NullTime{t1, true}, timeProto(t1), tTime, "NullTime with value"}, @@ -428,6 +431,11 @@ func TestEncodeValue(t *testing.T) { {[]CustomNumeric{CustomNumeric(*numValuePtr), CustomNumeric(*num2ValuePtr)}, listProto(numericProto(numValuePtr), numericProto(num2ValuePtr)), listType(tNumeric), "[]CustomNumeric"}, {[]CustomNullNumeric(nil), nullProto(), listType(tNumeric), "null []CustomNullNumeric"}, {[]CustomNullNumeric{{*numValuePtr, true}, {*num2ValuePtr, false}}, listProto(numericProto(numValuePtr), nullProto()), listType(tNumeric), "[]CustomNullNumeric"}, + // CUSTOM JSON + {CustomNullJSON{msg, true}, stringProto(jsonStr), tJSON, "CustomNullJSON with value"}, + {CustomNullJSON{msg, false}, nullProto(), tJSON, "CustomNullJSON with null"}, + {[]CustomNullJSON(nil), nullProto(), listType(tJSON), "null []CustomNullJSON"}, + {[]CustomNullJSON{{msg, true}, {msg, false}}, listProto(stringProto(jsonStr), nullProto()), listType(tJSON), "[]CustomNullJSON"}, } { got, gotType, err := encodeValue(test.in) if err != nil { @@ -1276,6 +1284,7 @@ func TestDecodeValue(t *testing.T) { type CustomNullTime NullTime type CustomNullDate NullDate type CustomNullNumeric NullNumeric + type CustomNullJSON NullJSON type Message struct { Name string @@ -1410,6 +1419,9 @@ func TestDecodeValue(t *testing.T) { {desc: "decode json to NullJSON", proto: stringProto(jsonStr), protoType: jsonType(), want: NullJSON{unmarshaledJSONstruct, true}}, {desc: "decode NULL to NullJSON", proto: nullProto(), protoType: jsonType(), want: NullJSON{}}, {desc: "decode an invalid json string", proto: stringProto(invalidJsonStr), protoType: jsonType(), want: msg, wantErr: true}, + // JSON ARRAY with []NullJSON + {desc: "decode ARRAY to []NullJSON", proto: listProto(stringProto(jsonStr), stringProto(jsonStr), nullProto()), protoType: listType(jsonType()), want: []NullJSON{{unmarshaledJSONstruct, true}, {unmarshaledJSONstruct, true}, {}}}, + {desc: "decode NULL to []NullJSON", proto: nullProto(), protoType: listType(jsonType()), want: []NullJSON(nil)}, // TIMESTAMP {desc: "decode TIMESTAMP to time.Time", proto: timeProto(t1), protoType: timeType(), want: t1}, {desc: "decode TIMESTAMP to NullTime", proto: timeProto(t1), protoType: timeType(), want: NullTime{t1, true}}, @@ -1620,6 +1632,7 @@ func TestDecodeValue(t *testing.T) { {desc: "decode BOOL to CustomNullBool", proto: boolProto(true), protoType: boolType(), want: CustomNullBool{true, true}}, {desc: "decode FLOAT64 to CustomNullFloat64", proto: floatProto(6.626), protoType: floatType(), want: CustomNullFloat64{6.626, true}}, {desc: "decode NUMERIC to CustomNullNumeric", proto: numericProto(numValuePtr), protoType: numericType(), want: CustomNullNumeric{*numValuePtr, true}}, + {desc: "decode JSON to CustomNullJSON", proto: stringProto(jsonStr), protoType: jsonType(), want: CustomNullJSON{unmarshaledJSONstruct, true}}, {desc: "decode TIMESTAMP to CustomNullTime", proto: timeProto(t1), protoType: timeType(), want: CustomNullTime{t1, true}}, {desc: "decode DATE to CustomNullDate", proto: dateProto(d1), protoType: dateType(), want: CustomNullDate{d1, true}}, @@ -1628,6 +1641,7 @@ func TestDecodeValue(t *testing.T) { {desc: "decode NULL to CustomNullBool", proto: nullProto(), protoType: boolType(), want: CustomNullBool{}}, {desc: "decode NULL to CustomNullFloat64", proto: nullProto(), protoType: floatType(), want: CustomNullFloat64{}}, {desc: "decode NULL to CustomNullNumeric", proto: nullProto(), protoType: numericType(), want: CustomNullNumeric{}}, + {desc: "decode NULL to CustomNullJSON", proto: nullProto(), protoType: jsonType(), want: CustomNullJSON{}}, {desc: "decode NULL to CustomNullTime", proto: nullProto(), protoType: timeType(), want: CustomNullTime{}}, {desc: "decode NULL to CustomNullDate", proto: nullProto(), protoType: dateType(), want: CustomNullDate{}}, @@ -1664,6 +1678,9 @@ func TestDecodeValue(t *testing.T) { {desc: "decode ARRAY to []CustomNumeric", proto: listProto(numericProto(numValuePtr), numericProto(num2ValuePtr)), protoType: listType(numericType()), want: []CustomNumeric{CustomNumeric(*numValuePtr), CustomNumeric(*num2ValuePtr)}}, {desc: "decode NULL to []CustomNullNumeric", proto: nullProto(), protoType: listType(numericType()), want: []CustomNullNumeric(nil)}, {desc: "decode ARRAY to []CustomNullNumeric", proto: listProto(numericProto(numValuePtr), nullProto(), numericProto(num2ValuePtr)), protoType: listType(numericType()), want: []CustomNullNumeric{{*numValuePtr, true}, {}, {*num2ValuePtr, true}}}, + // JSON ARRAY + {desc: "decode NULL to []CustomNullJSON", proto: nullProto(), protoType: listType(jsonType()), want: []CustomNullJSON(nil)}, + {desc: "decode ARRAY to []CustomNullJSON", proto: listProto(stringProto(jsonStr), stringProto(jsonStr), nullProto()), protoType: listType(jsonType()), want: []CustomNullJSON{{unmarshaledJSONstruct, true}, {unmarshaledJSONstruct, true}, {}}}, // TIME ARRAY {desc: "decode NULL to []CustomTime", proto: nullProto(), protoType: listType(timeType()), want: []CustomTime(nil)}, {desc: "decode ARRAY with NULL values to []CustomTime", proto: listProto(timeProto(t1), nullProto(), timeProto(t2)), protoType: listType(timeType()), want: []CustomTime{}, wantErr: true}, From 4d84196cc37d01dced5db361678574a94eb9fd5a Mon Sep 17 00:00:00 2001 From: Hengfeng Li Date: Sat, 15 May 2021 22:22:08 +1000 Subject: [PATCH 04/14] Add tests for json encoding and decoding for NullJSON. --- spanner/value.go | 6 +----- spanner/value_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/spanner/value.go b/spanner/value.go index 2b22eb52c27..ff685aefdc3 100644 --- a/spanner/value.go +++ b/spanner/value.go @@ -497,12 +497,8 @@ func (n *NullJSON) UnmarshalJSON(payload []byte) error { n.Valid = false return nil } - payload, err := trimDoubleQuotes(payload) - if err != nil { - return err - } var v interface{} - err = json.Unmarshal(payload, &v) + err := json.Unmarshal(payload, &v) if err != nil { return fmt.Errorf("payload cannot be converted to a struct: got %v, err: %s", string(payload), err) } diff --git a/spanner/value_test.go b/spanner/value_test.go index 31492505aae..c458d5f9bb4 100644 --- a/spanner/value_test.go +++ b/spanner/value_test.go @@ -2437,6 +2437,14 @@ func TestBindParamsDynamic(t *testing.T) { // Test converting nullable types to json strings. func TestJSONMarshal_NullTypes(t *testing.T) { + type Message struct { + Name string + Body string + Time int64 + } + msg := Message{"Alice", "Hello", 1294706395881547000} + jsonStr := `{"Name":"Alice","Body":"Hello","Time":1294706395881547000}` + type testcase struct { input interface{} expect string @@ -2509,6 +2517,15 @@ func TestJSONMarshal_NullTypes(t *testing.T) { {input: NullNumeric{}, expect: "null"}, }, }, + { + "NullJSON", + []testcase{ + {input: NullJSON{msg, true}, expect: jsonStr}, + {input: &NullJSON{msg, true}, expect: jsonStr}, + {input: &NullJSON{msg, false}, expect: "null"}, + {input: NullJSON{}, expect: "null"}, + }, + }, } { t.Run(test.name, func(t *testing.T) { for _, tc := range test.cases { @@ -2524,6 +2541,8 @@ func TestJSONMarshal_NullTypes(t *testing.T) { // Test converting json strings to nullable types. func TestJSONUnmarshal_NullTypes(t *testing.T) { + jsonStr := `{"Body":"Hello","Name":"Alice","Time":1294706395881547000}` + type testcase struct { input []byte got interface{} @@ -2607,6 +2626,16 @@ func TestJSONUnmarshal_NullTypes(t *testing.T) { {input: []byte(`"1234.123456789`), got: NullNumeric{}, isNull: true, expect: nullString, expectError: true}, }, }, + { + "NullJSON", + []testcase{ + {input: []byte(jsonStr), got: NullJSON{}, isNull: false, expect: jsonStr, expectError: false}, + {input: []byte("null"), got: NullJSON{}, isNull: true, expect: nullString, expectError: false}, + {input: nil, got: NullJSON{}, isNull: true, expect: nullString, expectError: true}, + {input: []byte(""), got: NullJSON{}, isNull: true, expect: nullString, expectError: true}, + {input: []byte(`{invalid_json_string}`), got: NullJSON{}, isNull: true, expect: nullString, expectError: true}, + }, + }, } { t.Run(test.name, func(t *testing.T) { for _, tc := range test.cases { @@ -2632,6 +2661,9 @@ func TestJSONUnmarshal_NullTypes(t *testing.T) { case NullNumeric: err := json.Unmarshal(tc.input, &v) expectUnmarshalNullableTypes(t, err, v, tc.isNull, tc.expect, tc.expectError) + case NullJSON: + err := json.Unmarshal(tc.input, &v) + expectUnmarshalNullableTypes(t, err, v, tc.isNull, tc.expect, tc.expectError) default: t.Fatalf("Unknown type: %T", v) } From 9040284373a898ef371d087a11e8853cc3f1f643 Mon Sep 17 00:00:00 2001 From: Hengfeng Li Date: Sat, 15 May 2021 22:57:24 +1000 Subject: [PATCH 05/14] Add integration test. --- spanner/integration_test.go | 51 ++++++++++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/spanner/integration_test.go b/spanner/integration_test.go index 00351f244b7..cd6f514e532 100644 --- a/spanner/integration_test.go +++ b/spanner/integration_test.go @@ -18,6 +18,7 @@ package spanner import ( "context" + "encoding/json" "errors" "flag" "fmt" @@ -113,6 +114,8 @@ var ( DateArray ARRAY, Timestamp TIMESTAMP, TimestampArray ARRAY, + Numeric NUMERIC, + NumericArray ARRAY ) PRIMARY KEY (RowID)`, } @@ -169,6 +172,8 @@ var ( DateArray ARRAY, Timestamp TIMESTAMP, TimestampArray ARRAY, + Numeric NUMERIC, + NumericArray ARRAY ) PRIMARY KEY (RowID)`, } @@ -1561,21 +1566,22 @@ func TestIntegration_BasicTypes(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() stmts := singerDBStatements - stmts = []string{ - `CREATE TABLE Singers ( + if !isEmulatorEnvSet() { + stmts = []string{ + `CREATE TABLE Singers ( SingerId INT64 NOT NULL, FirstName STRING(1024), LastName STRING(1024), SingerInfo BYTES(MAX) ) PRIMARY KEY (SingerId)`, - `CREATE INDEX SingerByName ON Singers(FirstName, LastName)`, - `CREATE TABLE Accounts ( + `CREATE INDEX SingerByName ON Singers(FirstName, LastName)`, + `CREATE TABLE Accounts ( AccountId INT64 NOT NULL, Nickname STRING(100), Balance INT64 NOT NULL, ) PRIMARY KEY (AccountId)`, - `CREATE INDEX AccountByNickname ON Accounts(Nickname) STORING (Balance)`, - `CREATE TABLE Types ( + `CREATE INDEX AccountByNickname ON Accounts(Nickname) STORING (Balance)`, + `CREATE TABLE Types ( RowID INT64 NOT NULL, String STRING(MAX), StringArray ARRAY, @@ -1592,8 +1598,11 @@ func TestIntegration_BasicTypes(t *testing.T) { Timestamp TIMESTAMP, TimestampArray ARRAY, Numeric NUMERIC, - NumericArray ARRAY + NumericArray ARRAY, + JSON JSON, + JSONArray ARRAY ) PRIMARY KEY (RowID)`, + } } client, _, cleanup := prepareIntegrationTest(ctx, t, DefaultSessionPoolConfig, stmts) defer cleanup() @@ -1613,6 +1622,16 @@ func TestIntegration_BasicTypes(t *testing.T) { n1 := *n1p n2 := *n2p + type Message struct { + Name string + Body string + Time int64 + } + msg := Message{"Alice", "Hello", 1294706395881547000} + jsonStr := `{"Name":"Alice","Body":"Hello","Time":1294706395881547000}` + var unmarshaledJSONstruct interface{} + json.Unmarshal([]byte(jsonStr), &unmarshaledJSONstruct) + tests := []struct { col string val interface{} @@ -1771,6 +1790,24 @@ func TestIntegration_BasicTypes(t *testing.T) { } } + if !isEmulatorEnvSet() { + tests = append(tests, []struct { + col string + val interface{} + want interface{} + }{ + {col: "JSON", val: msg, want: msg}, + {col: "JSON", val: msg, want: NullJSON{unmarshaledJSONstruct, true}}, + {col: "JSON", val: NullJSON{msg, true}, want: msg}, + {col: "JSON", val: NullJSON{msg, true}, want: NullJSON{unmarshaledJSONstruct, true}}, + {col: "JSON", val: NullJSON{msg, false}}, + {col: "JSON", val: nil, want: NullJSON{}}, + {col: "JSONArray", val: []NullJSON(nil)}, + {col: "JSONArray", val: []NullJSON{}}, + {col: "JSONArray", val: []NullJSON{{msg, true}, {msg, true}, {}}}, + }...) + } + // Verify that we can insert the rows using mutations. var muts []*Mutation for i, test := range tests { From 56305a0e796210fe479aeaeb886ef1c1e3c88b67 Mon Sep 17 00:00:00 2001 From: Hengfeng Li Date: Sat, 15 May 2021 23:10:54 +1000 Subject: [PATCH 06/14] Fix comments. --- spanner/integration_test.go | 8 ++++---- spanner/value.go | 12 ++++-------- spanner/value_test.go | 12 ++++++------ 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/spanner/integration_test.go b/spanner/integration_test.go index cd6f514e532..4d8a42576f7 100644 --- a/spanner/integration_test.go +++ b/spanner/integration_test.go @@ -1629,8 +1629,8 @@ func TestIntegration_BasicTypes(t *testing.T) { } msg := Message{"Alice", "Hello", 1294706395881547000} jsonStr := `{"Name":"Alice","Body":"Hello","Time":1294706395881547000}` - var unmarshaledJSONstruct interface{} - json.Unmarshal([]byte(jsonStr), &unmarshaledJSONstruct) + var unmarshalledJSONstruct interface{} + json.Unmarshal([]byte(jsonStr), &unmarshalledJSONstruct) tests := []struct { col string @@ -1797,9 +1797,9 @@ func TestIntegration_BasicTypes(t *testing.T) { want interface{} }{ {col: "JSON", val: msg, want: msg}, - {col: "JSON", val: msg, want: NullJSON{unmarshaledJSONstruct, true}}, + {col: "JSON", val: msg, want: NullJSON{unmarshalledJSONstruct, true}}, {col: "JSON", val: NullJSON{msg, true}, want: msg}, - {col: "JSON", val: NullJSON{msg, true}, want: NullJSON{unmarshaledJSONstruct, true}}, + {col: "JSON", val: NullJSON{msg, true}, want: NullJSON{unmarshalledJSONstruct, true}}, {col: "JSON", val: NullJSON{msg, false}}, {col: "JSON", val: nil, want: NullJSON{}}, {col: "JSONArray", val: []NullJSON(nil)}, diff --git a/spanner/value.go b/spanner/value.go index ff685aefdc3..aaaf6297665 100644 --- a/spanner/value.go +++ b/spanner/value.go @@ -460,7 +460,7 @@ func (n *NullNumeric) UnmarshalJSON(payload []byte) error { // NullJSON represents a Cloud Spanner JSON that may be NULL. type NullJSON struct { Value interface{} // Val contains the value when it is non-NULL, and nil when NULL. - Valid bool // Valid is true if Numeric is not NULL. + Valid bool // Valid is true if Json is not NULL. } // IsNull implements NullableValue.IsNull for NullJSON. @@ -475,7 +475,7 @@ func (n NullJSON) String() string { } b, err := json.Marshal(n.Value) if err != nil { - return nullString + return fmt.Sprintf("error: %v", err) } return fmt.Sprintf("%v", string(b)) } @@ -1420,7 +1420,7 @@ func decodeValue(v *proto3.Value, t *sppb.Type, ptr interface{}) error { if err != nil { return fmt.Errorf("failed to read json string: %s", err) } - // Check if it can be unmarshaled to the given type. + // Check if it can be unmarshalled to the given type. err = json.Unmarshal([]byte(x), ptr) if err != nil { return fmt.Errorf("failed to unmarshal the json string: %s", err) @@ -2942,10 +2942,8 @@ func encodeValue(v interface{}) (*proto3.Value, *sppb.Type, error) { return encodeValue(nv) } - fmt.Println("coming here - 1") // Check if the value is a variant of a base type. decodableType := getDecodableSpannerType(v, false) - fmt.Printf("%T %v\n", v, decodableType) if decodableType != spannerTypeUnknown && decodableType != spannerTypeInvalid { converted, err := convertCustomTypeValue(decodableType, v) if err != nil { @@ -2953,16 +2951,14 @@ func encodeValue(v interface{}) (*proto3.Value, *sppb.Type, error) { } return encodeValue(converted) } - fmt.Println("coming here - 2") - // Check if it can be marshaled to a json string. + // Check if it can be marshalled to a json string. b, err := json.Marshal(v) if err == nil { pb.Kind = stringKind(string(b)) pt = jsonType() return pb, pt, nil } - fmt.Println("coming here - 3") if !isStructOrArrayOfStructValue(v) { return nil, nil, errEncoderUnsupportedType(v) diff --git a/spanner/value_test.go b/spanner/value_test.go index c458d5f9bb4..061d996fec9 100644 --- a/spanner/value_test.go +++ b/spanner/value_test.go @@ -1293,8 +1293,8 @@ func TestDecodeValue(t *testing.T) { } msg := Message{"Alice", "Hello", 1294706395881547000} jsonStr := `{"Name":"Alice","Body":"Hello","Time":1294706395881547000}` - var unmarshaledJSONstruct interface{} - json.Unmarshal([]byte(jsonStr), &unmarshaledJSONstruct) + var unmarshalledJSONstruct interface{} + json.Unmarshal([]byte(jsonStr), &unmarshalledJSONstruct) invalidJsonStr := `{wrong_json_string}` // Pointer values. @@ -1416,11 +1416,11 @@ func TestDecodeValue(t *testing.T) { {desc: "decode NULL to []*big.Rat", proto: nullProto(), protoType: listType(numericType()), want: []*big.Rat(nil)}, // JSON {desc: "decode json to struct", proto: stringProto(jsonStr), protoType: jsonType(), want: msg}, - {desc: "decode json to NullJSON", proto: stringProto(jsonStr), protoType: jsonType(), want: NullJSON{unmarshaledJSONstruct, true}}, + {desc: "decode json to NullJSON", proto: stringProto(jsonStr), protoType: jsonType(), want: NullJSON{unmarshalledJSONstruct, true}}, {desc: "decode NULL to NullJSON", proto: nullProto(), protoType: jsonType(), want: NullJSON{}}, {desc: "decode an invalid json string", proto: stringProto(invalidJsonStr), protoType: jsonType(), want: msg, wantErr: true}, // JSON ARRAY with []NullJSON - {desc: "decode ARRAY to []NullJSON", proto: listProto(stringProto(jsonStr), stringProto(jsonStr), nullProto()), protoType: listType(jsonType()), want: []NullJSON{{unmarshaledJSONstruct, true}, {unmarshaledJSONstruct, true}, {}}}, + {desc: "decode ARRAY to []NullJSON", proto: listProto(stringProto(jsonStr), stringProto(jsonStr), nullProto()), protoType: listType(jsonType()), want: []NullJSON{{unmarshalledJSONstruct, true}, {unmarshalledJSONstruct, true}, {}}}, {desc: "decode NULL to []NullJSON", proto: nullProto(), protoType: listType(jsonType()), want: []NullJSON(nil)}, // TIMESTAMP {desc: "decode TIMESTAMP to time.Time", proto: timeProto(t1), protoType: timeType(), want: t1}, @@ -1632,7 +1632,7 @@ func TestDecodeValue(t *testing.T) { {desc: "decode BOOL to CustomNullBool", proto: boolProto(true), protoType: boolType(), want: CustomNullBool{true, true}}, {desc: "decode FLOAT64 to CustomNullFloat64", proto: floatProto(6.626), protoType: floatType(), want: CustomNullFloat64{6.626, true}}, {desc: "decode NUMERIC to CustomNullNumeric", proto: numericProto(numValuePtr), protoType: numericType(), want: CustomNullNumeric{*numValuePtr, true}}, - {desc: "decode JSON to CustomNullJSON", proto: stringProto(jsonStr), protoType: jsonType(), want: CustomNullJSON{unmarshaledJSONstruct, true}}, + {desc: "decode JSON to CustomNullJSON", proto: stringProto(jsonStr), protoType: jsonType(), want: CustomNullJSON{unmarshalledJSONstruct, true}}, {desc: "decode TIMESTAMP to CustomNullTime", proto: timeProto(t1), protoType: timeType(), want: CustomNullTime{t1, true}}, {desc: "decode DATE to CustomNullDate", proto: dateProto(d1), protoType: dateType(), want: CustomNullDate{d1, true}}, @@ -1680,7 +1680,7 @@ func TestDecodeValue(t *testing.T) { {desc: "decode ARRAY to []CustomNullNumeric", proto: listProto(numericProto(numValuePtr), nullProto(), numericProto(num2ValuePtr)), protoType: listType(numericType()), want: []CustomNullNumeric{{*numValuePtr, true}, {}, {*num2ValuePtr, true}}}, // JSON ARRAY {desc: "decode NULL to []CustomNullJSON", proto: nullProto(), protoType: listType(jsonType()), want: []CustomNullJSON(nil)}, - {desc: "decode ARRAY to []CustomNullJSON", proto: listProto(stringProto(jsonStr), stringProto(jsonStr), nullProto()), protoType: listType(jsonType()), want: []CustomNullJSON{{unmarshaledJSONstruct, true}, {unmarshaledJSONstruct, true}, {}}}, + {desc: "decode ARRAY to []CustomNullJSON", proto: listProto(stringProto(jsonStr), stringProto(jsonStr), nullProto()), protoType: listType(jsonType()), want: []CustomNullJSON{{unmarshalledJSONstruct, true}, {unmarshalledJSONstruct, true}, {}}}, // TIME ARRAY {desc: "decode NULL to []CustomTime", proto: nullProto(), protoType: listType(timeType()), want: []CustomTime(nil)}, {desc: "decode ARRAY with NULL values to []CustomTime", proto: listProto(timeProto(t1), nullProto(), timeProto(t2)), protoType: listType(timeType()), want: []CustomTime{}, wantErr: true}, From 665dc0521ed745b8ef3d88dcdc22ca80e2f3ffbd Mon Sep 17 00:00:00 2001 From: Hengfeng Li Date: Sat, 22 May 2021 20:18:15 +1000 Subject: [PATCH 07/14] Only use NullJSON for encoding. --- spanner/value.go | 16 ++++++---------- spanner/value_test.go | 1 - 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/spanner/value.go b/spanner/value.go index aaaf6297665..1097bd5bd12 100644 --- a/spanner/value.go +++ b/spanner/value.go @@ -2822,9 +2822,13 @@ func encodeValue(v interface{}) (*proto3.Value, *sppb.Type, error) { pt = listType(numericType()) case NullJSON: if v.Valid { - return encodeValue(v.Value) + b, err := json.Marshal(v.Value) + if err != nil { + return nil, nil, err + } + pb.Kind = stringKind(string(b)) } - pt = jsonType() + return pb, jsonType(), nil case []NullJSON: if v != nil { pb, err = encodeArray(len(v), func(i int) interface{} { return v[i] }) @@ -2952,14 +2956,6 @@ func encodeValue(v interface{}) (*proto3.Value, *sppb.Type, error) { return encodeValue(converted) } - // Check if it can be marshalled to a json string. - b, err := json.Marshal(v) - if err == nil { - pb.Kind = stringKind(string(b)) - pt = jsonType() - return pb, pt, nil - } - if !isStructOrArrayOfStructValue(v) { return nil, nil, errEncoderUnsupportedType(v) } diff --git a/spanner/value_test.go b/spanner/value_test.go index 061d996fec9..35406bf3af3 100644 --- a/spanner/value_test.go +++ b/spanner/value_test.go @@ -315,7 +315,6 @@ func TestEncodeValue(t *testing.T) { {[]*big.Rat{nil, numValuePtr}, listProto(nullProto(), numericProto(numValuePtr)), listType(tNumeric), "[]*big.Rat"}, {[]*big.Rat(nil), nullProto(), listType(tNumeric), "null []*big.Rat"}, // JSON - {msg, stringProto(jsonStr), tJSON, "json"}, {NullJSON{msg, true}, stringProto(jsonStr), tJSON, "NullJSON with value"}, {NullJSON{msg, false}, nullProto(), tJSON, "NullJSON with null"}, {[]NullJSON(nil), nullProto(), listType(tJSON), "null []NullJSON"}, From 77e4683db81739037db6d013a79566d834f21b3c Mon Sep 17 00:00:00 2001 From: Hengfeng Li Date: Sat, 22 May 2021 20:38:34 +1000 Subject: [PATCH 08/14] Update integration test. --- spanner/integration_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/spanner/integration_test.go b/spanner/integration_test.go index 4d8a42576f7..801bb86926d 100644 --- a/spanner/integration_test.go +++ b/spanner/integration_test.go @@ -1796,8 +1796,6 @@ func TestIntegration_BasicTypes(t *testing.T) { val interface{} want interface{} }{ - {col: "JSON", val: msg, want: msg}, - {col: "JSON", val: msg, want: NullJSON{unmarshalledJSONstruct, true}}, {col: "JSON", val: NullJSON{msg, true}, want: msg}, {col: "JSON", val: NullJSON{msg, true}, want: NullJSON{unmarshalledJSONstruct, true}}, {col: "JSON", val: NullJSON{msg, false}}, From 39b1ad7b8f2206faac00db14df511f1f74ffb0c1 Mon Sep 17 00:00:00 2001 From: Hengfeng Li Date: Thu, 27 May 2021 22:19:19 +1000 Subject: [PATCH 09/14] Only decode Cloud Spanner JSON to NullJSON type. --- spanner/value.go | 13 ------------- spanner/value_test.go | 9 +-------- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/spanner/value.go b/spanner/value.go index 1097bd5bd12..ee68a808609 100644 --- a/spanner/value.go +++ b/spanner/value.go @@ -1415,19 +1415,6 @@ func decodeValue(v *proto3.Value, t *sppb.Type, ptr interface{}) error { return decodableType.decodeValueToCustomType(v, t, acode, ptr) } - if code == sppb.TypeCode_JSON { - x, err := getStringValue(v) - if err != nil { - return fmt.Errorf("failed to read json string: %s", err) - } - // Check if it can be unmarshalled to the given type. - err = json.Unmarshal([]byte(x), ptr) - if err != nil { - return fmt.Errorf("failed to unmarshal the json string: %s", err) - } - break - } - // Check if the proto encoding is for an array of structs. if !(code == sppb.TypeCode_ARRAY && acode == sppb.TypeCode_STRUCT) { return errTypeMismatch(code, acode, ptr) diff --git a/spanner/value_test.go b/spanner/value_test.go index 35406bf3af3..193cae066d2 100644 --- a/spanner/value_test.go +++ b/spanner/value_test.go @@ -1285,12 +1285,6 @@ func TestDecodeValue(t *testing.T) { type CustomNullNumeric NullNumeric type CustomNullJSON NullJSON - type Message struct { - Name string - Body string - Time int64 - } - msg := Message{"Alice", "Hello", 1294706395881547000} jsonStr := `{"Name":"Alice","Body":"Hello","Time":1294706395881547000}` var unmarshalledJSONstruct interface{} json.Unmarshal([]byte(jsonStr), &unmarshalledJSONstruct) @@ -1414,10 +1408,9 @@ func TestDecodeValue(t *testing.T) { {desc: "decode ARRAY to []*big.Rat", proto: listProto(numericProto(numValuePtr), nullProto(), numericProto(num2ValuePtr)), protoType: listType(numericType()), want: []*big.Rat{numValuePtr, nil, num2ValuePtr}}, {desc: "decode NULL to []*big.Rat", proto: nullProto(), protoType: listType(numericType()), want: []*big.Rat(nil)}, // JSON - {desc: "decode json to struct", proto: stringProto(jsonStr), protoType: jsonType(), want: msg}, {desc: "decode json to NullJSON", proto: stringProto(jsonStr), protoType: jsonType(), want: NullJSON{unmarshalledJSONstruct, true}}, {desc: "decode NULL to NullJSON", proto: nullProto(), protoType: jsonType(), want: NullJSON{}}, - {desc: "decode an invalid json string", proto: stringProto(invalidJsonStr), protoType: jsonType(), want: msg, wantErr: true}, + {desc: "decode an invalid json string", proto: stringProto(invalidJsonStr), protoType: jsonType(), want: NullJSON{}, wantErr: true}, // JSON ARRAY with []NullJSON {desc: "decode ARRAY to []NullJSON", proto: listProto(stringProto(jsonStr), stringProto(jsonStr), nullProto()), protoType: listType(jsonType()), want: []NullJSON{{unmarshalledJSONstruct, true}, {unmarshalledJSONstruct, true}, {}}}, {desc: "decode NULL to []NullJSON", proto: nullProto(), protoType: listType(jsonType()), want: []NullJSON(nil)}, From b92fe20c8fd4ca24e702fda0707769907ea200a4 Mon Sep 17 00:00:00 2001 From: Hengfeng Li Date: Thu, 27 May 2021 22:53:48 +1000 Subject: [PATCH 10/14] Add more tests. --- spanner/value.go | 3 +++ spanner/value_test.go | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/spanner/value.go b/spanner/value.go index ee68a808609..81947da79c4 100644 --- a/spanner/value.go +++ b/spanner/value.go @@ -458,6 +458,9 @@ func (n *NullNumeric) UnmarshalJSON(payload []byte) error { } // NullJSON represents a Cloud Spanner JSON that may be NULL. +// +// This type must always be used when encoding values to a JSON column in Cloud +// Spanner. type NullJSON struct { Value interface{} // Val contains the value when it is non-NULL, and nil when NULL. Valid bool // Valid is true if Json is not NULL. diff --git a/spanner/value_test.go b/spanner/value_test.go index 193cae066d2..f73a07a9e16 100644 --- a/spanner/value_test.go +++ b/spanner/value_test.go @@ -221,6 +221,12 @@ func TestEncodeValue(t *testing.T) { } msg := Message{"Alice", "Hello", 1294706395881547000} jsonStr := `{"Name":"Alice","Body":"Hello","Time":1294706395881547000}` + emptyArrayJSONStr := `[]` + type PtrMessage struct { + Key *string + } + ptrMsg := PtrMessage{} + nullValueJSONStr := `{"Key":null}` sValue := "abc" var sNilPtr *string @@ -319,6 +325,8 @@ func TestEncodeValue(t *testing.T) { {NullJSON{msg, false}, nullProto(), tJSON, "NullJSON with null"}, {[]NullJSON(nil), nullProto(), listType(tJSON), "null []NullJSON"}, {[]NullJSON{{msg, true}, {msg, false}}, listProto(stringProto(jsonStr), nullProto()), listType(tJSON), "[]NullJSON"}, + {NullJSON{[]Message{}, true}, stringProto(emptyArrayJSONStr), tJSON, "a json string with empty array to NullJSON"}, + {NullJSON{ptrMsg, true}, stringProto(nullValueJSONStr), tJSON, "a json string with null value to NullJSON"}, // TIMESTAMP / TIMESTAMP ARRAY {t1, timeProto(t1), tTime, "time"}, {NullTime{t1, true}, timeProto(t1), tTime, "NullTime with value"}, @@ -1289,6 +1297,12 @@ func TestDecodeValue(t *testing.T) { var unmarshalledJSONstruct interface{} json.Unmarshal([]byte(jsonStr), &unmarshalledJSONstruct) invalidJsonStr := `{wrong_json_string}` + emptyArrayJSONStr := `[]` + var unmarshalledEmptyJSONArray interface{} + json.Unmarshal([]byte(emptyArrayJSONStr), &unmarshalledEmptyJSONArray) + nullValueJSONStr := `{"Key":null}` + var unmarshalledStructWithNull interface{} + json.Unmarshal([]byte(nullValueJSONStr), &unmarshalledStructWithNull) // Pointer values. sValue := "abc" @@ -1411,6 +1425,8 @@ func TestDecodeValue(t *testing.T) { {desc: "decode json to NullJSON", proto: stringProto(jsonStr), protoType: jsonType(), want: NullJSON{unmarshalledJSONstruct, true}}, {desc: "decode NULL to NullJSON", proto: nullProto(), protoType: jsonType(), want: NullJSON{}}, {desc: "decode an invalid json string", proto: stringProto(invalidJsonStr), protoType: jsonType(), want: NullJSON{}, wantErr: true}, + {desc: "decode a json string with empty array to a NullJSON", proto: stringProto(emptyArrayJSONStr), protoType: jsonType(), want: NullJSON{unmarshalledEmptyJSONArray, true}}, + {desc: "decode a json string with null to a NullJSON", proto: stringProto(nullValueJSONStr), protoType: jsonType(), want: NullJSON{unmarshalledStructWithNull, true}}, // JSON ARRAY with []NullJSON {desc: "decode ARRAY to []NullJSON", proto: listProto(stringProto(jsonStr), stringProto(jsonStr), nullProto()), protoType: listType(jsonType()), want: []NullJSON{{unmarshalledJSONstruct, true}, {unmarshalledJSONstruct, true}, {}}}, {desc: "decode NULL to []NullJSON", proto: nullProto(), protoType: listType(jsonType()), want: []NullJSON(nil)}, From 8029896a9d75434605290c5d3e3fa7c087d3af27 Mon Sep 17 00:00:00 2001 From: Hengfeng Li Date: Sat, 29 May 2021 23:46:04 +1000 Subject: [PATCH 11/14] Support decoding ARRAY to NullJSON. --- spanner/value.go | 81 +++++++++++++++++++++++++++++++++++++++---- spanner/value_test.go | 16 +++++---- 2 files changed, 85 insertions(+), 12 deletions(-) diff --git a/spanner/value.go b/spanner/value.go index 81947da79c4..cb10007425f 100644 --- a/spanner/value.go +++ b/spanner/value.go @@ -25,6 +25,7 @@ import ( "math/big" "reflect" "strconv" + "strings" "time" "cloud.google.com/go/civil" @@ -1090,20 +1091,52 @@ func decodeValue(v *proto3.Value, t *sppb.Type, ptr interface{}) error { if p == nil { return errNilDst(p) } - if code != sppb.TypeCode_JSON { + if code == sppb.TypeCode_ARRAY { + x, err := getListValue(v) + if err != nil { + return err + } + y, err := decodeNullJSONArrayToNullJSON(x) + if err != nil { + return err + } + *p = *y + } else { + if code != sppb.TypeCode_JSON { + return errTypeMismatch(code, acode, ptr) + } + if isNull { + *p = NullJSON{} + break + } + x := v.GetStringValue() + var y interface{} + err := json.Unmarshal([]byte(x), &y) + if err != nil { + return err + } + *p = NullJSON{y, true} + } + case *[]NullJSON: + if p == nil { + return errNilDst(p) + } + if acode != sppb.TypeCode_JSON { return errTypeMismatch(code, acode, ptr) } if isNull { - *p = NullJSON{} + *p = nil break } - x := v.GetStringValue() - var y interface{} - err := json.Unmarshal([]byte(x), &y) + x, err := getListValue(v) if err != nil { return err } - *p = NullJSON{y, true} + y, err := decodeNullJSONArray(x) + if err != nil { + return err + } + *p = y case *NullNumeric: if p == nil { return errNilDst(p) @@ -2310,6 +2343,42 @@ func decodeNullNumericArray(pb *proto3.ListValue) ([]NullNumeric, error) { return a, nil } +// decodeNullJSONArray decodes proto3.ListValue pb into a NullJSON slice. +func decodeNullJSONArray(pb *proto3.ListValue) ([]NullJSON, error) { + if pb == nil { + return nil, errNilListValue("JSON") + } + a := make([]NullJSON, len(pb.Values)) + for i, v := range pb.Values { + if err := decodeValue(v, jsonType(), &a[i]); err != nil { + return nil, errDecodeArrayElement(i, v, "JSON", err) + } + } + return a, nil +} + +// decodeNullJSONArray decodes proto3.ListValue pb into a NullJSON pointer. +func decodeNullJSONArrayToNullJSON(pb *proto3.ListValue) (*NullJSON, error) { + if pb == nil { + return nil, errNilListValue("JSON") + } + strs := []string{} + for _, v := range pb.Values { + if _, ok := v.Kind.(*proto3.Value_NullValue); ok { + strs = append(strs, "null") + } else { + strs = append(strs, v.GetStringValue()) + } + } + s := fmt.Sprintf("[%s]", strings.Join(strs, ",")) + var y interface{} + err := json.Unmarshal([]byte(s), &y) + if err != nil { + return nil, err + } + return &NullJSON{y, true}, nil +} + // decodeNumericPointerArray decodes proto3.ListValue pb into a *big.Rat slice. func decodeNumericPointerArray(pb *proto3.ListValue) ([]*big.Rat, error) { if pb == nil { diff --git a/spanner/value_test.go b/spanner/value_test.go index f73a07a9e16..1b27f0f11c7 100644 --- a/spanner/value_test.go +++ b/spanner/value_test.go @@ -1294,8 +1294,8 @@ func TestDecodeValue(t *testing.T) { type CustomNullJSON NullJSON jsonStr := `{"Name":"Alice","Body":"Hello","Time":1294706395881547000}` - var unmarshalledJSONstruct interface{} - json.Unmarshal([]byte(jsonStr), &unmarshalledJSONstruct) + var unmarshalledJSONStruct interface{} + json.Unmarshal([]byte(jsonStr), &unmarshalledJSONStruct) invalidJsonStr := `{wrong_json_string}` emptyArrayJSONStr := `[]` var unmarshalledEmptyJSONArray interface{} @@ -1303,6 +1303,9 @@ func TestDecodeValue(t *testing.T) { nullValueJSONStr := `{"Key":null}` var unmarshalledStructWithNull interface{} json.Unmarshal([]byte(nullValueJSONStr), &unmarshalledStructWithNull) + arrayJSONStr := `[{"Name":"Alice","Body":"Hello","Time":1294706395881547000},null,true]` + var unmarshalledJSONArray interface{} + json.Unmarshal([]byte(arrayJSONStr), &unmarshalledJSONArray) // Pointer values. sValue := "abc" @@ -1422,13 +1425,14 @@ func TestDecodeValue(t *testing.T) { {desc: "decode ARRAY to []*big.Rat", proto: listProto(numericProto(numValuePtr), nullProto(), numericProto(num2ValuePtr)), protoType: listType(numericType()), want: []*big.Rat{numValuePtr, nil, num2ValuePtr}}, {desc: "decode NULL to []*big.Rat", proto: nullProto(), protoType: listType(numericType()), want: []*big.Rat(nil)}, // JSON - {desc: "decode json to NullJSON", proto: stringProto(jsonStr), protoType: jsonType(), want: NullJSON{unmarshalledJSONstruct, true}}, + {desc: "decode json to NullJSON", proto: stringProto(jsonStr), protoType: jsonType(), want: NullJSON{unmarshalledJSONStruct, true}}, {desc: "decode NULL to NullJSON", proto: nullProto(), protoType: jsonType(), want: NullJSON{}}, {desc: "decode an invalid json string", proto: stringProto(invalidJsonStr), protoType: jsonType(), want: NullJSON{}, wantErr: true}, {desc: "decode a json string with empty array to a NullJSON", proto: stringProto(emptyArrayJSONStr), protoType: jsonType(), want: NullJSON{unmarshalledEmptyJSONArray, true}}, {desc: "decode a json string with null to a NullJSON", proto: stringProto(nullValueJSONStr), protoType: jsonType(), want: NullJSON{unmarshalledStructWithNull, true}}, // JSON ARRAY with []NullJSON - {desc: "decode ARRAY to []NullJSON", proto: listProto(stringProto(jsonStr), stringProto(jsonStr), nullProto()), protoType: listType(jsonType()), want: []NullJSON{{unmarshalledJSONstruct, true}, {unmarshalledJSONstruct, true}, {}}}, + {desc: "decode ARRAY to []NullJSON", proto: listProto(stringProto(jsonStr), stringProto(jsonStr), nullProto()), protoType: listType(jsonType()), want: []NullJSON{{unmarshalledJSONStruct, true}, {unmarshalledJSONStruct, true}, {}}}, + {desc: "decode ARRAY to NullJSON", proto: listProto(stringProto(jsonStr), nullProto(), stringProto("true")), protoType: listType(jsonType()), want: NullJSON{unmarshalledJSONArray, true}}, {desc: "decode NULL to []NullJSON", proto: nullProto(), protoType: listType(jsonType()), want: []NullJSON(nil)}, // TIMESTAMP {desc: "decode TIMESTAMP to time.Time", proto: timeProto(t1), protoType: timeType(), want: t1}, @@ -1640,7 +1644,7 @@ func TestDecodeValue(t *testing.T) { {desc: "decode BOOL to CustomNullBool", proto: boolProto(true), protoType: boolType(), want: CustomNullBool{true, true}}, {desc: "decode FLOAT64 to CustomNullFloat64", proto: floatProto(6.626), protoType: floatType(), want: CustomNullFloat64{6.626, true}}, {desc: "decode NUMERIC to CustomNullNumeric", proto: numericProto(numValuePtr), protoType: numericType(), want: CustomNullNumeric{*numValuePtr, true}}, - {desc: "decode JSON to CustomNullJSON", proto: stringProto(jsonStr), protoType: jsonType(), want: CustomNullJSON{unmarshalledJSONstruct, true}}, + {desc: "decode JSON to CustomNullJSON", proto: stringProto(jsonStr), protoType: jsonType(), want: CustomNullJSON{unmarshalledJSONStruct, true}}, {desc: "decode TIMESTAMP to CustomNullTime", proto: timeProto(t1), protoType: timeType(), want: CustomNullTime{t1, true}}, {desc: "decode DATE to CustomNullDate", proto: dateProto(d1), protoType: dateType(), want: CustomNullDate{d1, true}}, @@ -1688,7 +1692,7 @@ func TestDecodeValue(t *testing.T) { {desc: "decode ARRAY to []CustomNullNumeric", proto: listProto(numericProto(numValuePtr), nullProto(), numericProto(num2ValuePtr)), protoType: listType(numericType()), want: []CustomNullNumeric{{*numValuePtr, true}, {}, {*num2ValuePtr, true}}}, // JSON ARRAY {desc: "decode NULL to []CustomNullJSON", proto: nullProto(), protoType: listType(jsonType()), want: []CustomNullJSON(nil)}, - {desc: "decode ARRAY to []CustomNullJSON", proto: listProto(stringProto(jsonStr), stringProto(jsonStr), nullProto()), protoType: listType(jsonType()), want: []CustomNullJSON{{unmarshalledJSONstruct, true}, {unmarshalledJSONstruct, true}, {}}}, + {desc: "decode ARRAY to []CustomNullJSON", proto: listProto(stringProto(jsonStr), stringProto(jsonStr), nullProto()), protoType: listType(jsonType()), want: []CustomNullJSON{{unmarshalledJSONStruct, true}, {unmarshalledJSONStruct, true}, {}}}, // TIME ARRAY {desc: "decode NULL to []CustomTime", proto: nullProto(), protoType: listType(timeType()), want: []CustomTime(nil)}, {desc: "decode ARRAY with NULL values to []CustomTime", proto: listProto(timeProto(t1), nullProto(), timeProto(t2)), protoType: listType(timeType()), want: []CustomTime{}, wantErr: true}, From 0c48954c7b31654a15e73c6b3bb6262f6a7256b3 Mon Sep 17 00:00:00 2001 From: Hengfeng Li Date: Fri, 4 Jun 2021 16:48:34 +1000 Subject: [PATCH 12/14] Fix comment. --- spanner/value.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spanner/value.go b/spanner/value.go index cb10007425f..9c322b47e9a 100644 --- a/spanner/value.go +++ b/spanner/value.go @@ -1092,6 +1092,9 @@ func decodeValue(v *proto3.Value, t *sppb.Type, ptr interface{}) error { return errNilDst(p) } if code == sppb.TypeCode_ARRAY { + if acode != sppb.TypeCode_JSON { + return errTypeMismatch(code, acode, ptr) + } x, err := getListValue(v) if err != nil { return err From 6b334766399c2a197fe4a397550420c95890dfa1 Mon Sep 17 00:00:00 2001 From: Hengfeng Li Date: Thu, 12 Aug 2021 13:43:07 +1000 Subject: [PATCH 13/14] Fix the comment. --- spanner/value.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spanner/value.go b/spanner/value.go index 9c322b47e9a..4ef7e6bffdd 100644 --- a/spanner/value.go +++ b/spanner/value.go @@ -472,7 +472,7 @@ func (n NullJSON) IsNull() bool { return !n.Valid } -// String implements Stringer.String for NullJSON +// String implements Stringer.String for NullJSON. func (n NullJSON) String() string { if !n.Valid { return nullString From a29ae597d4d57819857c6324c32aadac0903653a Mon Sep 17 00:00:00 2001 From: Hengfeng Li Date: Tue, 24 Aug 2021 12:03:14 +1000 Subject: [PATCH 14/14] Fix the variable naming. --- spanner/value_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spanner/value_test.go b/spanner/value_test.go index eaebf32acfe..b6f93b259ea 100644 --- a/spanner/value_test.go +++ b/spanner/value_test.go @@ -1334,7 +1334,7 @@ func TestDecodeValue(t *testing.T) { jsonStr := `{"Name":"Alice","Body":"Hello","Time":1294706395881547000}` var unmarshalledJSONStruct interface{} json.Unmarshal([]byte(jsonStr), &unmarshalledJSONStruct) - invalidJsonStr := `{wrong_json_string}` + invalidJSONStr := `{wrong_json_string}` emptyArrayJSONStr := `[]` var unmarshalledEmptyJSONArray interface{} json.Unmarshal([]byte(emptyArrayJSONStr), &unmarshalledEmptyJSONArray) @@ -1465,7 +1465,7 @@ func TestDecodeValue(t *testing.T) { // JSON {desc: "decode json to NullJSON", proto: stringProto(jsonStr), protoType: jsonType(), want: NullJSON{unmarshalledJSONStruct, true}}, {desc: "decode NULL to NullJSON", proto: nullProto(), protoType: jsonType(), want: NullJSON{}}, - {desc: "decode an invalid json string", proto: stringProto(invalidJsonStr), protoType: jsonType(), want: NullJSON{}, wantErr: true}, + {desc: "decode an invalid json string", proto: stringProto(invalidJSONStr), protoType: jsonType(), want: NullJSON{}, wantErr: true}, {desc: "decode a json string with empty array to a NullJSON", proto: stringProto(emptyArrayJSONStr), protoType: jsonType(), want: NullJSON{unmarshalledEmptyJSONArray, true}}, {desc: "decode a json string with null to a NullJSON", proto: stringProto(nullValueJSONStr), protoType: jsonType(), want: NullJSON{unmarshalledStructWithNull, true}}, // JSON ARRAY with []NullJSON