value_test.go

// Copyright 2015 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bigquery

import (
	"encoding/base64"
	"errors"
	"fmt"
	"math"
	"math/big"
	"reflect"
	"testing"
	"time"

	"cloud.google.com/go/civil"
	"cloud.google.com/go/internal/testutil"
	"github.com/google/go-cmp/cmp"
	bq "google.golang.org/api/bigquery/v2"
)

func TestConvertBasicValues(t *testing.T) {
	schema := Schema{
		{Type: StringFieldType},
		{Type: IntegerFieldType},
		{Type: FloatFieldType},
		{Type: BooleanFieldType},
		{Type: BytesFieldType},
		{Type: NumericFieldType},
		{Type: BigNumericFieldType},
		{Type: GeographyFieldType},
		{Type: JSONFieldType},
	}
	row := &bq.TableRow{
		F: []*bq.TableCell{
			{V: "a"},
			{V: "1"},
			{V: "1.2"},
			{V: "true"},
			{V: base64.StdEncoding.EncodeToString([]byte("foo"))},
			{V: "123.123456789"},
			{V: "99999999999999999999999999999999999999.99999999999999999999999999999999999999"},
			{V: testGeography},
			{V: "{\"alpha\": \"beta\"}"},
		},
	}
	got, err := convertRow(row, schema)
	if err != nil {
		t.Fatalf("error converting: %v", err)
	}

	bigRatVal := new(big.Rat)
	bigRatVal.SetString("99999999999999999999999999999999999999.99999999999999999999999999999999999999")
	want := []Value{"a", int64(1), 1.2, true, []byte("foo"), big.NewRat(123123456789, 1e9), bigRatVal, testGeography, "{\"alpha\": \"beta\"}"}
	if !testutil.Equal(got, want) {
		t.Errorf("converting basic values: got:\n%v\nwant:\n%v", got, want)
	}
}

func TestConvertTime(t *testing.T) {
	schema := Schema{
		{Type: TimestampFieldType},
		{Type: DateFieldType},
		{Type: TimeFieldType},
		{Type: DateTimeFieldType},
	}
	ts := testTimestamp.Round(time.Millisecond)
	row := &bq.TableRow{
		F: []*bq.TableCell{
			{V: fmt.Sprintf("%.10f", float64(ts.UnixNano())/1e9)},
			{V: testDate.String()},
			{V: testTime.String()},
			{V: testDateTime.String()},
		},
	}
	got, err := convertRow(row, schema)
	if err != nil {
		t.Fatalf("error converting: %v", err)
	}
	want := []Value{ts, testDate, testTime, testDateTime}
	for i, g := range got {
		w := want[i]
		if !testutil.Equal(g, w) {
			t.Errorf("#%d: got:\n%v\nwant:\n%v", i, g, w)
		}
	}
	if got[0].(time.Time).Location() != time.UTC {
		t.Errorf("expected time zone UTC: got:\n%v", got)
	}
}

func TestConvertSmallTimes(t *testing.T) {
	for _, year := range []int{1600, 1066, 1} {
		want := time.Date(year, time.January, 1, 0, 0, 0, 0, time.UTC)
		s := fmt.Sprintf("%.10f", float64(want.Unix()))
		got, err := convertBasicType(s, TimestampFieldType)
		if err != nil {
			t.Fatal(err)
		}
		if !got.(time.Time).Equal(want) {
			t.Errorf("got %v, want %v", got, want)
		}
	}
}

func TestConvertTimePrecision(t *testing.T) {
	tcs := []struct {
		// Internally, BigQuery stores timestamps as microsecond-precision
		// floats.
		bq   float64
		want time.Time
	}{
		{
			bq:   1555593697.154358,
			want: time.Unix(1555593697, 154358*1000),
		},
		{
			bq:   1555593697.154359,
			want: time.Unix(1555593697, 154359*1000),
		},
		{
			bq:   1555593697.154360,
			want: time.Unix(1555593697, 154360*1000),
		},
	}
	for _, tc := range tcs {
		bqS := fmt.Sprintf("%.6f", tc.bq)
		t.Run(bqS, func(t *testing.T) {
			got, err := convertBasicType(bqS, TimestampFieldType)
			if err != nil {
				t.Fatalf("convertBasicType failed: %v", err)
			}
			gotT, ok := got.(time.Time)
			if !ok {
				t.Fatalf("got a %T from convertBasicType, want a time.Time; got = %v", got, got)
			}
			if !gotT.Equal(tc.want) {
				t.Errorf("got %v from convertBasicType, want %v", gotT, tc.want)
			}
		})
	}
}

func TestConvertNullValues(t *testing.T) {
	schema := Schema{{Type: StringFieldType}}
	row := &bq.TableRow{
		F: []*bq.TableCell{
			{V: nil},
		},
	}
	got, err := convertRow(row, schema)
	if err != nil {
		t.Fatalf("error converting: %v", err)
	}
	want := []Value{nil}
	if !testutil.Equal(got, want) {
		t.Errorf("converting null values: got:\n%v\nwant:\n%v", got, want)
	}
}

func TestBasicRepetition(t *testing.T) {
	schema := Schema{
		{Type: IntegerFieldType, Repeated: true},
	}
	row := &bq.TableRow{
		F: []*bq.TableCell{
			{
				V: []interface{}{
					map[string]interface{}{
						"v": "1",
					},
					map[string]interface{}{
						"v": "2",
					},
					map[string]interface{}{
						"v": "3",
					},
				},
			},
		},
	}
	got, err := convertRow(row, schema)
	if err != nil {
		t.Fatalf("error converting: %v", err)
	}
	want := []Value{[]Value{int64(1), int64(2), int64(3)}}
	if !testutil.Equal(got, want) {
		t.Errorf("converting basic repeated values: got:\n%v\nwant:\n%v", got, want)
	}
}

func TestNestedRecordContainingRepetition(t *testing.T) {
	schema := Schema{
		{
			Type: RecordFieldType,
			Schema: Schema{
				{Type: IntegerFieldType, Repeated: true},
			},
		},
	}
	row := &bq.TableRow{
		F: []*bq.TableCell{
			{
				V: map[string]interface{}{
					"f": []interface{}{
						map[string]interface{}{
							"v": []interface{}{
								map[string]interface{}{"v": "1"},
								map[string]interface{}{"v": "2"},
								map[string]interface{}{"v": "3"},
							},
						},
					},
				},
			},
		},
	}

	got, err := convertRow(row, schema)
	if err != nil {
		t.Fatalf("error converting: %v", err)
	}
	want := []Value{[]Value{[]Value{int64(1), int64(2), int64(3)}}}
	if !testutil.Equal(got, want) {
		t.Errorf("converting basic repeated values: got:\n%v\nwant:\n%v", got, want)
	}
}

func TestRepeatedRecordContainingRepetition(t *testing.T) {
	schema := Schema{
		{
			Type:     RecordFieldType,
			Repeated: true,
			Schema: Schema{
				{Type: IntegerFieldType, Repeated: true},
			},
		},
	}
	row := &bq.TableRow{F: []*bq.TableCell{
		{
			V: []interface{}{ // repeated records.
				map[string]interface{}{ // first record.
					"v": map[string]interface{}{ // pointless single-key-map wrapper.
						"f": []interface{}{ // list of record fields.
							map[string]interface{}{ // only record (repeated ints)
								"v": []interface{}{ // pointless wrapper.
									map[string]interface{}{
										"v": "1",
									},
									map[string]interface{}{
										"v": "2",
									},
									map[string]interface{}{
										"v": "3",
									},
								},
							},
						},
					},
				},
				map[string]interface{}{ // second record.
					"v": map[string]interface{}{
						"f": []interface{}{
							map[string]interface{}{
								"v": []interface{}{
									map[string]interface{}{
										"v": "4",
									},
									map[string]interface{}{
										"v": "5",
									},
									map[string]interface{}{
										"v": "6",
									},
								},
							},
						},
					},
				},
			},
		},
	}}

	got, err := convertRow(row, schema)
	if err != nil {
		t.Fatalf("error converting: %v", err)
	}
	want := []Value{ // the row is a list of length 1, containing an entry for the repeated record.
		[]Value{ // the repeated record is a list of length 2, containing an entry for each repetition.
			[]Value{ // the record is a list of length 1, containing an entry for the repeated integer field.
				[]Value{int64(1), int64(2), int64(3)}, // the repeated integer field is a list of length 3.
			},
			[]Value{ // second record
				[]Value{int64(4), int64(5), int64(6)},
			},
		},
	}
	if !testutil.Equal(got, want) {
		t.Errorf("converting repeated records with repeated values: got:\n%v\nwant:\n%v", got, want)
	}
}

func TestRepeatedRecordContainingRecord(t *testing.T) {
	schema := Schema{
		{
			Type:     RecordFieldType,
			Repeated: true,
			Schema: Schema{
				{
					Type: StringFieldType,
				},
				{
					Type: RecordFieldType,
					Schema: Schema{
						{Type: IntegerFieldType},
						{Type: StringFieldType},
					},
				},
			},
		},
	}
	row := &bq.TableRow{F: []*bq.TableCell{
		{
			V: []interface{}{ // repeated records.
				map[string]interface{}{ // first record.
					"v": map[string]interface{}{ // pointless single-key-map wrapper.
						"f": []interface{}{ // list of record fields.
							map[string]interface{}{ // first record field (name)
								"v": "first repeated record",
							},
							map[string]interface{}{ // second record field (nested record).
								"v": map[string]interface{}{ // pointless single-key-map wrapper.
									"f": []interface{}{ // nested record fields
										map[string]interface{}{
											"v": "1",
										},
										map[string]interface{}{
											"v": "two",
										},
									},
								},
							},
						},
					},
				},
				map[string]interface{}{ // second record.
					"v": map[string]interface{}{
						"f": []interface{}{
							map[string]interface{}{
								"v": "second repeated record",
							},
							map[string]interface{}{
								"v": map[string]interface{}{
									"f": []interface{}{
										map[string]interface{}{
											"v": "3",
										},
										map[string]interface{}{
											"v": "four",
										},
									},
								},
							},
						},
					},
				},
			},
		},
	}}

	got, err := convertRow(row, schema)
	if err != nil {
		t.Fatalf("error converting: %v", err)
	}
	// TODO: test with flattenresults.
	want := []Value{ // the row is a list of length 1, containing an entry for the repeated record.
		[]Value{ // the repeated record is a list of length 2, containing an entry for each repetition.
			[]Value{ // record contains a string followed by a nested record.
				"first repeated record",
				[]Value{
					int64(1),
					"two",
				},
			},
			[]Value{ // second record.
				"second repeated record",
				[]Value{
					int64(3),
					"four",
				},
			},
		},
	}
	if !testutil.Equal(got, want) {
		t.Errorf("converting repeated records containing record : got:\n%v\nwant:\n%v", got, want)
	}
}

func TestConvertRowErrors(t *testing.T) {
	// mismatched lengths
	if _, err := convertRow(&bq.TableRow{F: []*bq.TableCell{{V: ""}}}, Schema{}); err == nil {
		t.Error("got nil, want error")
	}
	v3 := map[string]interface{}{"v": 3}
	for _, test := range []struct {
		value interface{}
		fs    FieldSchema
	}{
		{3, FieldSchema{Type: IntegerFieldType}}, // not a string
		{[]interface{}{v3}, // not a string, repeated
			FieldSchema{Type: IntegerFieldType, Repeated: true}},
		{map[string]interface{}{"f": []interface{}{v3}}, // not a string, nested
			FieldSchema{Type: RecordFieldType, Schema: Schema{{Type: IntegerFieldType}}}},
		{map[string]interface{}{"f": []interface{}{v3}}, // wrong length, nested
			FieldSchema{Type: RecordFieldType, Schema: Schema{}}},
	} {
		_, err := convertRow(
			&bq.TableRow{F: []*bq.TableCell{{V: test.value}}},
			Schema{&test.fs})
		if err == nil {
			t.Errorf("value %v, fs %v: got nil, want error", test.value, test.fs)
		}
	}

	// bad field type
	if _, err := convertBasicType("", FieldType("BAD")); err == nil {
		t.Error("got nil, want error")
	}
}

func TestValuesSaverConvertsToMap(t *testing.T) {
	testCases := []struct {
		vs           ValuesSaver
		wantInsertID string
		wantRow      map[string]Value
	}{
		{
			vs: ValuesSaver{
				Schema: Schema{
					{Name: "intField", Type: IntegerFieldType},
					{Name: "strField", Type: StringFieldType},
					{Name: "dtField", Type: DateTimeFieldType},
					{Name: "nField", Type: NumericFieldType},
					{Name: "bigNumField", Type: BigNumericFieldType},
					{Name: "geoField", Type: GeographyFieldType},
				},
				InsertID: "iid",
				Row: []Value{1, "a",
					civil.DateTime{
						Date: civil.Date{Year: 1, Month: 2, Day: 3},
						Time: civil.Time{Hour: 4, Minute: 5, Second: 6, Nanosecond: 7000}},
					big.NewRat(123456789000, 1e9),
					big.NewRat(1, 3),
					testGeography,
				},
			},
			wantInsertID: "iid",
			wantRow: map[string]Value{
				"intField":    1,
				"strField":    "a",
				"dtField":     "0001-02-03 04:05:06.000007",
				"nField":      "123.456789000",
				"bigNumField": "0.33333333333333333333333333333333333333",
				"geoField":    testGeography,
			},
		},
		{
			vs: ValuesSaver{
				Schema: Schema{
					{Name: "intField", Type: IntegerFieldType},
					{
						Name: "recordField",
						Type: RecordFieldType,
						Schema: Schema{
							{Name: "nestedInt", Type: IntegerFieldType, Repeated: true},
						},
					},
				},
				InsertID: "iid",
				Row:      []Value{1, []Value{[]Value{2, 3}}},
			},
			wantInsertID: "iid",
			wantRow: map[string]Value{
				"intField": 1,
				"recordField": map[string]Value{
					"nestedInt": []Value{2, 3},
				},
			},
		},
		{ // repeated nested field
			vs: ValuesSaver{
				Schema: Schema{
					{
						Name: "records",
						Type: RecordFieldType,
						Schema: Schema{
							{Name: "x", Type: IntegerFieldType},
							{Name: "y", Type: IntegerFieldType},
						},
						Repeated: true,
					},
				},
				InsertID: "iid",
				Row: []Value{ // a row is a []Value
					[]Value{ // repeated field's value is a []Value
						[]Value{1, 2}, // first record of the repeated field
						[]Value{3, 4}, // second record
					},
				},
			},
			wantInsertID: "iid",
			wantRow: map[string]Value{
				"records": []Value{
					map[string]Value{"x": 1, "y": 2},
					map[string]Value{"x": 3, "y": 4},
				},
			},
		},
		{ // zero-length repeated nested field
			vs: ValuesSaver{
				Schema: Schema{
					{
						Name: "records",
						Type: RecordFieldType,
						Schema: Schema{
							{Name: "x", Type: IntegerFieldType},
							{Name: "y", Type: IntegerFieldType},
						},
						Repeated: true,
					},
				},
				InsertID: "iid",
				Row: []Value{ // a row is a []Value
					[]Value{}, // repeated field's value is a []Value, and non-nil
				},
			},
			wantInsertID: "iid",
			wantRow: map[string]Value{
				"records": []Value{},
			},
		},
	}
	for _, tc := range testCases {
		gotRow, gotInsertID, err := tc.vs.Save()
		if err != nil {
			t.Errorf("Expected successful save; got: %v", err)
			continue
		}
		if !testutil.Equal(gotRow, tc.wantRow) {
			t.Errorf("%v row:\ngot:\n%+v\nwant:\n%+v", tc.vs, gotRow, tc.wantRow)
		}
		if !testutil.Equal(gotInsertID, tc.wantInsertID) {
			t.Errorf("%v ID:\ngot:\n%+v\nwant:\n%+v", tc.vs, gotInsertID, tc.wantInsertID)
		}
	}
}

func TestValuesToMapErrors(t *testing.T) {
	for _, test := range []struct {
		values []Value
		schema Schema
	}{
		{ // mismatched length
			[]Value{1},
			Schema{},
		},
		{ // nested record not a slice
			[]Value{1},
			Schema{{Type: RecordFieldType}},
		},
		{ // nested record mismatched length
			[]Value{[]Value{1}},
			Schema{{Type: RecordFieldType}},
		},
		{ // nested repeated record not a slice
			[]Value{[]Value{1}},
			Schema{{Type: RecordFieldType, Repeated: true}},
		},
		{ // nested repeated record mismatched length
			[]Value{[]Value{[]Value{1}}},
			Schema{{Type: RecordFieldType, Repeated: true}},
		},
	} {
		_, err := valuesToMap(test.values, test.schema)
		if err == nil {
			t.Errorf("%v, %v: got nil, want error", test.values, test.schema)
		}
	}
}

func TestStructSaver(t *testing.T) {
	schema := Schema{
		{Name: "s", Type: StringFieldType},
		{Name: "r", Type: IntegerFieldType, Repeated: true},
		{Name: "t", Type: TimeFieldType},
		{Name: "tr", Type: TimeFieldType, Repeated: true},
		{Name: "nested", Type: RecordFieldType, Schema: Schema{
			{Name: "b", Type: BooleanFieldType},
		}},
		{Name: "rnested", Type: RecordFieldType, Repeated: true, Schema: Schema{
			{Name: "b", Type: BooleanFieldType},
		}},
		{Name: "p", Type: IntegerFieldType, Required: false},
		{Name: "n", Type: NumericFieldType, Required: false},
		{Name: "nr", Type: NumericFieldType, Repeated: true},
		{Name: "bn", Type: BigNumericFieldType, Required: false},
		{Name: "bnr", Type: BigNumericFieldType, Repeated: true},
		{Name: "g", Type: GeographyFieldType, Required: false},
		{Name: "gr", Type: GeographyFieldType, Repeated: true},
	}

	type (
		N struct{ B bool }
		T struct {
			S       string
			R       []int
			T       civil.Time
			TR      []civil.Time
			Nested  *N
			Rnested []*N
			P       NullInt64
			N       *big.Rat
			NR      []*big.Rat
			BN      *big.Rat
			BNR     []*big.Rat
			G       NullGeography
			GR      []string // Repeated Geography
		}
	)

	check := func(msg string, in interface{}, want map[string]Value) {
		ss := StructSaver{
			Schema:   schema,
			InsertID: "iid",
			Struct:   in,
		}
		got, gotIID, err := ss.Save()
		if err != nil {
			t.Fatalf("%s: %v", msg, err)
		}
		if wantIID := "iid"; gotIID != wantIID {
			t.Errorf("%s: InsertID: got %q, want %q", msg, gotIID, wantIID)
		}
		if diff := testutil.Diff(got, want); diff != "" {
			t.Errorf("%s: %s", msg, diff)
		}
	}

	ct1 := civil.Time{Hour: 1, Minute: 2, Second: 3, Nanosecond: 4000}
	ct2 := civil.Time{Hour: 5, Minute: 6, Second: 7, Nanosecond: 8000}
	in := T{
		S:       "x",
		R:       []int{1, 2},
		T:       ct1,
		TR:      []civil.Time{ct1, ct2},
		Nested:  &N{B: true},
		Rnested: []*N{{true}, {false}},
		P:       NullInt64{Valid: true, Int64: 17},
		N:       big.NewRat(123456, 1000),
		NR:      []*big.Rat{big.NewRat(3, 1), big.NewRat(56789, 1e5)},
		BN:      big.NewRat(1, 3),
		BNR:     []*big.Rat{big.NewRat(1, 3), big.NewRat(1, 2)},
		G:       NullGeography{Valid: true, GeographyVal: "POINT(-122.350220 47.649154)"},
		GR:      []string{"POINT(-122.350220 47.649154)", "POINT(-122.198939 47.669865)"},
	}
	want := map[string]Value{
		"s":       "x",
		"r":       []int{1, 2},
		"t":       "01:02:03.000004",
		"tr":      []string{"01:02:03.000004", "05:06:07.000008"},
		"nested":  map[string]Value{"b": true},
		"rnested": []Value{map[string]Value{"b": true}, map[string]Value{"b": false}},
		"p":       NullInt64{Valid: true, Int64: 17},
		"n":       "123.456000000",
		"nr":      []string{"3.000000000", "0.567890000"},
		"bn":      "0.33333333333333333333333333333333333333",
		"bnr":     []string{"0.33333333333333333333333333333333333333", "0.50000000000000000000000000000000000000"},
		"g":       NullGeography{Valid: true, GeographyVal: "POINT(-122.350220 47.649154)"},
		"gr":      []string{"POINT(-122.350220 47.649154)", "POINT(-122.198939 47.669865)"},
	}
	check("all values", in, want)
	check("all values, ptr", &in, want)
	check("empty struct", T{}, map[string]Value{"s": "", "t": "00:00:00", "p": NullInt64{}, "g": NullGeography{}})

	// Missing and extra fields ignored.
	type T2 struct {
		S string
		// missing R, Nested, RNested
		Extra int
	}
	check("missing and extra", T2{S: "x"}, map[string]Value{"s": "x"})

	check("nils in slice", T{Rnested: []*N{{true}, nil, {false}}},
		map[string]Value{
			"s":       "",
			"t":       "00:00:00",
			"p":       NullInt64{},
			"g":       NullGeography{},
			"rnested": []Value{map[string]Value{"b": true}, map[string]Value(nil), map[string]Value{"b": false}},
		})

	check("zero-length repeated", T{Rnested: []*N{}},
		map[string]Value{
			"rnested": []Value{},
			"s":       "",
			"t":       "00:00:00",
			"p":       NullInt64{},
			"g":       NullGeography{},
		})
}

func TestStructSaverErrors(t *testing.T) {
	type (
		badField struct {
			I int `bigquery:"@"`
		}
		badR  struct{ R int }
		badRN struct{ R []int }
	)

	for i, test := range []struct {
		inputStruct interface{}
		schema      Schema
	}{
		{0, nil},           // not a struct
		{&badField{}, nil}, // bad field name
		{&badR{}, Schema{{Name: "r", Repeated: true}}},        // repeated field has bad type
		{&badR{}, Schema{{Name: "r", Type: RecordFieldType}}}, // nested field has bad type
		{&badRN{[]int{0}}, // nested repeated field has bad type
			Schema{{Name: "r", Type: RecordFieldType, Repeated: true}}},
	} {
		ss := &StructSaver{Struct: test.inputStruct, Schema: test.schema}
		_, _, err := ss.Save()
		if err == nil {
			t.Errorf("#%d, %v, %v: got nil, want error", i, test.inputStruct, test.schema)
		}
	}
}

func TestNumericStrings(t *testing.T) {
	for _, test := range []struct {
		description    string
		in             *big.Rat
		wantNumeric    string
		wantBigNumeric string
	}{
		{"repeating with rounding", big.NewRat(2, 3), "0.666666667", "0.66666666666666666666666666666666666667"},
		{"all zero padding", big.NewRat(1, 2), "0.500000000", "0.50000000000000000000000000000000000000"},
		{"zero pad with digit", big.NewRat(1, 2*1e8), "0.000000005", "0.00000000500000000000000000000000000000"},
		{"smaller rounding case 1", big.NewRat(5, 1e10), "0.000000001", "0.00000000050000000000000000000000000000"},
		{"smaller rounding case 2", big.NewRat(-5, 1e10), "-0.000000001", "-0.00000000050000000000000000000000000000"},
	} {
		if got := NumericString(test.in); got != test.wantNumeric {
			t.Errorf("case %q, val %v as numeric: got %q, want %q", test.description, test.in, got, test.wantNumeric)
		}
		if got := BigNumericString(test.in); got != test.wantBigNumeric {
			t.Errorf("case %q, val %v as bignumeric: got %q, want %q", test.description, test.in, got, test.wantBigNumeric)
		}
	}
}

func TestConvertRows(t *testing.T) {
	schema := Schema{
		{Type: StringFieldType},
		{Type: IntegerFieldType},
		{Type: FloatFieldType},
		{Type: BooleanFieldType},
		{Type: GeographyFieldType},
	}
	rows := []*bq.TableRow{
		{F: []*bq.TableCell{
			{V: "a"},
			{V: "1"},
			{V: "1.2"},
			{V: "true"},
			{V: "POINT(-122.350220 47.649154)"},
		}},
		{F: []*bq.TableCell{
			{V: "b"},
			{V: "2"},
			{V: "2.2"},
			{V: "false"},
			{V: "POINT(-122.198939 47.669865)"},
		}},
	}
	want := [][]Value{
		{"a", int64(1), 1.2, true, "POINT(-122.350220 47.649154)"},
		{"b", int64(2), 2.2, false, "POINT(-122.198939 47.669865)"},
	}
	got, err := convertRows(rows, schema)
	if err != nil {
		t.Fatalf("got %v, want nil", err)
	}
	if !testutil.Equal(got, want) {
		t.Errorf("\ngot  %v\nwant %v", got, want)
	}

	rows[0].F[0].V = 1
	_, err = convertRows(rows, schema)
	if err == nil {
		t.Error("got nil, want error")
	}
}

func TestValueList(t *testing.T) {
	schema := Schema{
		{Name: "s", Type: StringFieldType},
		{Name: "i", Type: IntegerFieldType},
		{Name: "f", Type: FloatFieldType},
		{Name: "b", Type: BooleanFieldType},
	}
	want := []Value{"x", 7, 3.14, true}
	var got []Value
	vl := (*valueList)(&got)
	if err := vl.Load(want, schema); err != nil {
		t.Fatal(err)
	}

	if !testutil.Equal(got, want) {
		t.Errorf("got %+v, want %+v", got, want)
	}

	// Load truncates, not appends.
	// https://github.com/googleapis/google-cloud-go/issues/437
	if err := vl.Load(want, schema); err != nil {
		t.Fatal(err)
	}
	if !testutil.Equal(got, want) {
		t.Errorf("got %+v, want %+v", got, want)
	}
}

func TestValueMap(t *testing.T) {
	ns := Schema{
		{Name: "x", Type: IntegerFieldType},
		{Name: "y", Type: IntegerFieldType},
	}
	schema := Schema{
		{Name: "s", Type: StringFieldType},
		{Name: "i", Type: IntegerFieldType},
		{Name: "f", Type: FloatFieldType},
		{Name: "b", Type: BooleanFieldType},
		{Name: "n", Type: RecordFieldType, Schema: ns},
		{Name: "rn", Type: RecordFieldType, Schema: ns, Repeated: true},
	}
	in := []Value{"x", 7, 3.14, true,
		[]Value{1, 2},
		[]Value{[]Value{3, 4}, []Value{5, 6}},
	}
	var vm valueMap
	if err := vm.Load(in, schema); err != nil {
		t.Fatal(err)
	}
	want := map[string]Value{
		"s": "x",
		"i": 7,
		"f": 3.14,
		"b": true,
		"n": map[string]Value{"x": 1, "y": 2},
		"rn": []Value{
			map[string]Value{"x": 3, "y": 4},
			map[string]Value{"x": 5, "y": 6},
		},
	}
	if !testutil.Equal(vm, valueMap(want)) {
		t.Errorf("got\n%+v\nwant\n%+v", vm, want)
	}

	in = make([]Value, len(schema))
	want = map[string]Value{
		"s":  nil,
		"i":  nil,
		"f":  nil,
		"b":  nil,
		"n":  nil,
		"rn": nil,
	}
	var vm2 valueMap
	if err := vm2.Load(in, schema); err != nil {
		t.Fatal(err)
	}
	if !testutil.Equal(vm2, valueMap(want)) {
		t.Errorf("got\n%+v\nwant\n%+v", vm2, want)
	}
}

var (
	// For testing StructLoader
	schema2 = Schema{
		{Name: "s", Type: StringFieldType},
		{Name: "s2", Type: StringFieldType},
		{Name: "by", Type: BytesFieldType},
		{Name: "I", Type: IntegerFieldType},
		{Name: "U", Type: IntegerFieldType},
		{Name: "F", Type: FloatFieldType},
		{Name: "B", Type: BooleanFieldType},
		{Name: "TS", Type: TimestampFieldType},
		{Name: "D", Type: DateFieldType},
		{Name: "T", Type: TimeFieldType},
		{Name: "DT", Type: DateTimeFieldType},
		{Name: "N", Type: NumericFieldType},
		{Name: "BN", Type: BigNumericFieldType},
		{Name: "G", Type: GeographyFieldType},
		{Name: "nested", Type: RecordFieldType, Schema: Schema{
			{Name: "nestS", Type: StringFieldType},
			{Name: "nestI", Type: IntegerFieldType},
		}},
		{Name: "t", Type: StringFieldType},
	}

	testTimestamp  = time.Date(2016, 11, 5, 7, 50, 22, 8, time.UTC)
	testDate       = civil.Date{Year: 2016, Month: 11, Day: 5}
	testTime       = civil.Time{Hour: 7, Minute: 50, Second: 22, Nanosecond: 8}
	testDateTime   = civil.DateTime{Date: testDate, Time: testTime}
	testNumeric    = big.NewRat(123, 456)
	testBigNumeric = big.NewRat(456, 789)
	// testGeography is a WKT string representing a single point.
	testGeography = "POINT(-122.350220 47.649154)"

	testValues = []Value{"x", "y", []byte{1, 2, 3}, int64(7), int64(8), 3.14, true,
		testTimestamp, testDate, testTime, testDateTime, testNumeric, testBigNumeric, testGeography,
		[]Value{"nested", int64(17)}, "z"}
)

type testStruct1 struct {
	B bool
	I int
	U uint16
	times
	S      string
	S2     String
	By     []byte
	F      float64
	N      *big.Rat
	BN     *big.Rat
	G      string
	Nested nested
	Tagged string `bigquery:"t"`
}

type String string

type nested struct {
	NestS string
	NestI int
}

type times struct {
	TS time.Time
	T  civil.Time
	D  civil.Date
	DT civil.DateTime
}

func TestStructLoader(t *testing.T) {
	var ts1 testStruct1
	mustLoad(t, &ts1, schema2, testValues)
	// Note: the schema field named "s" gets matched to the exported struct
	// field "S", not the unexported "s".
	want := &testStruct1{
		B:      true,
		I:      7,
		U:      8,
		F:      3.14,
		times:  times{TS: testTimestamp, T: testTime, D: testDate, DT: testDateTime},
		S:      "x",
		S2:     "y",
		By:     []byte{1, 2, 3},
		N:      big.NewRat(123, 456),
		BN:     big.NewRat(456, 789),
		G:      testGeography,
		Nested: nested{NestS: "nested", NestI: 17},
		Tagged: "z",
	}
	if diff := testutil.Diff(&ts1, want, cmp.AllowUnexported(testStruct1{})); diff != "" {
		t.Error(diff)
	}

	// Test pointers to nested structs.
	type nestedPtr struct{ Nested *nested }
	var np nestedPtr
	mustLoad(t, &np, schema2, testValues)
	want2 := &nestedPtr{Nested: &nested{NestS: "nested", NestI: 17}}
	if diff := testutil.Diff(&np, want2); diff != "" {
		t.Error(diff)
	}

	// Existing values should be reused.
	nst := &nested{NestS: "x", NestI: -10}
	np = nestedPtr{Nested: nst}
	mustLoad(t, &np, schema2, testValues)
	if diff := testutil.Diff(&np, want2); diff != "" {
		t.Error(diff)
	}
	if np.Nested != nst {
		t.Error("nested struct pointers not equal")
	}
}

type repStruct struct {
	Nums      []int
	ShortNums [2]int // to test truncation
	LongNums  [5]int // to test padding with zeroes
	Nested    []*nested
}

var (
	repSchema = Schema{
		{Name: "nums", Type: IntegerFieldType, Repeated: true},
		{Name: "shortNums", Type: IntegerFieldType, Repeated: true},
		{Name: "longNums", Type: IntegerFieldType, Repeated: true},
		{Name: "nested", Type: RecordFieldType, Repeated: true, Schema: Schema{
			{Name: "nestS", Type: StringFieldType},
			{Name: "nestI", Type: IntegerFieldType},
		}},
	}
	v123      = []Value{int64(1), int64(2), int64(3)}
	repValues = []Value{v123, v123, v123,
		[]Value{
			[]Value{"x", int64(1)},
			[]Value{"y", int64(2)},
		},
	}
)

func TestStructLoaderRepeated(t *testing.T) {
	var r1 repStruct
	mustLoad(t, &r1, repSchema, repValues)
	want := repStruct{
		Nums:      []int{1, 2, 3},
		ShortNums: [...]int{1, 2}, // extra values discarded
		LongNums:  [...]int{1, 2, 3, 0, 0},
		Nested:    []*nested{{"x", 1}, {"y", 2}},
	}
	if diff := testutil.Diff(r1, want); diff != "" {
		t.Error(diff)
	}
	r2 := repStruct{
		Nums:     []int{-1, -2, -3, -4, -5},    // truncated to zero and appended to
		LongNums: [...]int{-1, -2, -3, -4, -5}, // unset elements are zeroed
	}
	mustLoad(t, &r2, repSchema, repValues)
	if diff := testutil.Diff(r2, want); diff != "" {
		t.Error(diff)
	}
	if got, want := cap(r2.Nums), 5; got != want {
		t.Errorf("cap(r2.Nums) = %d, want %d", got, want)
	}

	// Short slice case.
	r3 := repStruct{Nums: []int{-1}}
	mustLoad(t, &r3, repSchema, repValues)
	if diff := testutil.Diff(r3, want); diff != "" {
		t.Error(diff)
	}
	if got, want := cap(r3.Nums), 3; got != want {
		t.Errorf("cap(r3.Nums) = %d, want %d", got, want)
	}
}

type testStructNullable struct {
	String     NullString
	Bytes      []byte
	Integer    NullInt64
	Float      NullFloat64
	Boolean    NullBool
	Timestamp  NullTimestamp
	Date       NullDate
	Time       NullTime
	DateTime   NullDateTime
	Numeric    *big.Rat
	BigNumeric *big.Rat
	Geography  NullGeography
	Record     *subNullable
}

type subNullable struct {
	X NullInt64
}

var testStructNullableSchema = Schema{
	{Name: "String", Type: StringFieldType, Required: false},
	{Name: "Bytes", Type: BytesFieldType, Required: false},
	{Name: "Integer", Type: IntegerFieldType, Required: false},
	{Name: "Float", Type: FloatFieldType, Required: false},
	{Name: "Boolean", Type: BooleanFieldType, Required: false},
	{Name: "Timestamp", Type: TimestampFieldType, Required: false},
	{Name: "Date", Type: DateFieldType, Required: false},
	{Name: "Time", Type: TimeFieldType, Required: false},
	{Name: "DateTime", Type: DateTimeFieldType, Required: false},
	{Name: "Numeric", Type: NumericFieldType, Required: false},
	{Name: "BigNumeric", Type: BigNumericFieldType, Required: false},
	{Name: "Geography", Type: GeographyFieldType, Required: false},
	{Name: "Record", Type: RecordFieldType, Required: false, Schema: Schema{
		{Name: "X", Type: IntegerFieldType, Required: false},
	}},
}

func TestStructLoaderNullable(t *testing.T) {
	var ts testStructNullable
	nilVals := make([]Value, len(testStructNullableSchema))
	mustLoad(t, &ts, testStructNullableSchema, nilVals)
	want := testStructNullable{}
	if diff := testutil.Diff(ts, want); diff != "" {
		t.Error(diff)
	}

	nonnilVals := []Value{"x", []byte{1, 2, 3}, int64(1), 2.3, true, testTimestamp, testDate, testTime,
		testDateTime, big.NewRat(1, 2), big.NewRat(3, 4), testGeography, []Value{int64(4)}}

	// All ts fields are nil. Loading non-nil values will cause them all to
	// be allocated.
	mustLoad(t, &ts, testStructNullableSchema, nonnilVals)
	want = testStructNullable{
		String:     NullString{StringVal: "x", Valid: true},
		Bytes:      []byte{1, 2, 3},
		Integer:    NullInt64{Int64: 1, Valid: true},
		Float:      NullFloat64{Float64: 2.3, Valid: true},
		Boolean:    NullBool{Bool: true, Valid: true},
		Timestamp:  NullTimestamp{Timestamp: testTimestamp, Valid: true},
		Date:       NullDate{Date: testDate, Valid: true},
		Time:       NullTime{Time: testTime, Valid: true},
		DateTime:   NullDateTime{DateTime: testDateTime, Valid: true},
		Numeric:    big.NewRat(1, 2),
		BigNumeric: big.NewRat(3, 4),
		Geography:  NullGeography{GeographyVal: testGeography, Valid: true},
		Record:     &subNullable{X: NullInt64{Int64: 4, Valid: true}},
	}
	if diff := testutil.Diff(ts, want); diff != "" {
		t.Error(diff)
	}

	// Struct pointers are reused, byte slices are not.
	want = ts
	want.Bytes = []byte{17}
	vals2 := []Value{nil, []byte{17}, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, []Value{int64(7)}}
	mustLoad(t, &ts, testStructNullableSchema, vals2)
	if ts.Record != want.Record {
		t.Error("record pointers not identical")
	}
}

func TestStructLoaderOverflow(t *testing.T) {
	type S struct {
		I int16
		U uint16
		F float32
	}
	schema := Schema{
		{Name: "I", Type: IntegerFieldType},
		{Name: "U", Type: IntegerFieldType},
		{Name: "F", Type: FloatFieldType},
	}
	var s S
	z64 := int64(0)
	for _, vals := range [][]Value{
		{int64(math.MaxInt16 + 1), z64, 0},
		{z64, int64(math.MaxInt32), 0},
		{z64, int64(-1), 0},
		{z64, z64, math.MaxFloat32 * 2},
	} {
		if err := load(&s, schema, vals); err == nil {
			t.Errorf("%+v: got nil, want error", vals)
		}
	}
}

func TestStructLoaderFieldOverlap(t *testing.T) {
	// It's OK if the struct has fields that the schema does not, and vice versa.
	type S1 struct {
		I int
		X [][]int // not in the schema; does not even correspond to a valid BigQuery type
		// many schema fields missing
	}
	var s1 S1
	if err := load(&s1, schema2, testValues); err != nil {
		t.Fatal(err)
	}
	want1 := S1{I: 7}
	if diff := testutil.Diff(s1, want1); diff != "" {
		t.Error(diff)
	}

	// It's even valid to have no overlapping fields at all.
	type S2 struct{ Z int }

	var s2 S2
	mustLoad(t, &s2, schema2, testValues)
	want2 := S2{}
	if diff := testutil.Diff(s2, want2); diff != "" {
		t.Error(diff)
	}
}

func TestStructLoaderErrors(t *testing.T) {
	check := func(sp interface{}) {
		var sl structLoader
		err := sl.set(sp, schema2)
		if err == nil {
			t.Errorf("%T: got nil, want error", sp)
		}
	}

	type bad1 struct{ F int32 } // wrong type for FLOAT column
	check(&bad1{})

	type bad2 struct{ I uint } // unsupported integer type
	check(&bad2{})

	type bad3 struct {
		I int `bigquery:"@"`
	} // bad field name
	check(&bad3{})

	type bad4 struct{ Nested int } // non-struct for nested field
	check(&bad4{})

	type bad5 struct{ Nested struct{ NestS int } } // bad nested struct
	check(&bad5{})

	bad6 := &struct{ Nums int }{} // non-slice for repeated field
	sl := structLoader{}
	err := sl.set(bad6, repSchema)
	if err == nil {
		t.Errorf("%T: got nil, want error", bad6)
	}

	// sl.set's error is sticky, even with good input.
	err2 := sl.set(&repStruct{}, repSchema)
	if err2 != err {
		t.Errorf("%v != %v, expected equal", err2, err)
	}
	// sl.Load is similarly sticky
	err2 = sl.Load(nil, nil)
	if err2 != err {
		t.Errorf("%v != %v, expected equal", err2, err)
	}

	// Null values.
	schema := Schema{
		{Name: "i", Type: IntegerFieldType},
		{Name: "f", Type: FloatFieldType},
		{Name: "b", Type: BooleanFieldType},
		{Name: "s", Type: StringFieldType},
		{Name: "d", Type: DateFieldType},
		{Name: "r", Type: RecordFieldType, Schema: Schema{{Name: "X", Type: IntegerFieldType}}},
	}
	type s struct {
		I int
		F float64
		B bool
		S string
		D civil.Date
	}
	vstruct := reflect.ValueOf(s{}).Type()
	fieldNames := []string{"I", "F", "B", "S", "D"}
	vals := []Value{int64(0), 0.0, false, "", testDate}
	mustLoad(t, &s{}, schema, vals)
	for i, e := range vals {
		vals[i] = nil
		got := load(&s{}, schema, vals)
		if errors.Is(got, errNoNulls) {
			t.Errorf("#%d: got %v, want %v", i, got, errNoNulls)
		}
		f, _ := vstruct.FieldByName(fieldNames[i])
		expectedError := fmt.Sprintf("bigquery: NULL cannot be assigned to field `%s` of type %s", f.Name, f.Type.Name())
		if got.Error() != expectedError {
			t.Errorf("#%d: got %v, want %v", i, got, expectedError)
		}
		vals[i] = e
	}

	// Using more than one struct type with the same structLoader.
	type different struct {
		B bool
		I int
		times
		S    string
		Nums []int
	}

	sl = structLoader{}
	if err := sl.set(&testStruct1{}, schema2); err != nil {
		t.Fatal(err)
	}
	err = sl.set(&different{}, schema2)
	if err == nil {
		t.Error("different struct types: got nil, want error")
	}
}

func mustLoad(t *testing.T, pval interface{}, schema Schema, vals []Value) {
	if err := load(pval, schema, vals); err != nil {
		t.Fatalf("loading: %v", err)
	}
}

func load(pval interface{}, schema Schema, vals []Value) error {
	var sl structLoader
	if err := sl.set(pval, schema); err != nil {
		return err
	}
	return sl.Load(vals, nil)
}

func BenchmarkStructLoader_NoCompile(b *testing.B) {
	benchmarkStructLoader(b, false)
}

func BenchmarkStructLoader_Compile(b *testing.B) {
	benchmarkStructLoader(b, true)
}

func benchmarkStructLoader(b *testing.B, compile bool) {
	var ts1 testStruct1
	for i := 0; i < b.N; i++ {
		var sl structLoader
		for j := 0; j < 10; j++ {
			if err := load(&ts1, schema2, testValues); err != nil {
				b.Fatal(err)
			}
			if !compile {
				sl.typ = nil
			}
		}
	}
}