feat(bigquery/storage/managedwriter): enable field name indirection

bigquery/storage/managedwriter/adapt/protoconversion.go

@@ -18,7 +18,9 @@ import (
 	"encoding/base64"
 	"fmt"
 	"strings"
+	"unicode"
 
+	"cloud.google.com/go/bigquery/storage/managedwriter/internal/annotations"
 	storagepb "google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/reflect/protodesc"
@@ -286,46 +288,71 @@ func storageSchemaToDescriptorInternal(inSchema *storagepb.TableSchema, scope st
 //
 // Messages are always nullable, and repeated fields are as well.
 func tableFieldSchemaToFieldDescriptorProto(field *storagepb.TableFieldSchema, idx int32, scope string, useProto3 bool) (*descriptorpb.FieldDescriptorProto, error) {
+
 	name := strings.ToLower(field.GetName())
+	var fdp *descriptorpb.FieldDescriptorProto
+
 	if field.GetType() == storagepb.TableFieldSchema_STRUCT {
-		return &descriptorpb.FieldDescriptorProto{
+		fdp = &descriptorpb.FieldDescriptorProto{
 			Name:     proto.String(name),
 			Number:   proto.Int32(idx),
 			TypeName: proto.String(scope),
 			Label:    convertModeToLabel(field.GetMode(), useProto3),
-		}, nil
-	}
-
-	// For (REQUIRED||REPEATED) fields for proto3, or all cases for proto2, we can use the expected scalar types.
-	if field.GetMode() != storagepb.TableFieldSchema_NULLABLE || !useProto3 {
-		outType := bqTypeToFieldTypeMap[field.GetType()]
-		fdp := &descriptorpb.FieldDescriptorProto{
-			Name:   proto.String(name),
-			Number: proto.Int32(idx),
-			Type:   outType.Enum(),
-			Label:  convertModeToLabel(field.GetMode(), useProto3),
 		}
-		// Special case: proto2 repeated fields may benefit from using packed annotation.
-		if field.GetMode() == storagepb.TableFieldSchema_REPEATED && !useProto3 {
-			for _, v := range packedTypes {
-				if outType == v {
-					fdp.Options = &descriptorpb.FieldOptions{
-						Packed: proto.Bool(true),
+	} else {
+		// For (REQUIRED||REPEATED) fields for proto3, or all cases for proto2, we can use the expected scalar types.
+		if field.GetMode() != storagepb.TableFieldSchema_NULLABLE || !useProto3 {
+			outType := bqTypeToFieldTypeMap[field.GetType()]
+			fdp = &descriptorpb.FieldDescriptorProto{
+				Name:   proto.String(name),
+				Number: proto.Int32(idx),
+				Type:   outType.Enum(),
+				Label:  convertModeToLabel(field.GetMode(), useProto3),
+			}
+
+			// Special case: proto2 repeated fields may benefit from using packed annotation.
+			if field.GetMode() == storagepb.TableFieldSchema_REPEATED && !useProto3 {
+				for _, v := range packedTypes {
+					if outType == v {
+						fdp.Options = &descriptorpb.FieldOptions{
+							Packed: proto.Bool(true),
+						}
+						break
 					}
-					break
 				}
 			}
+		} else {
+			// For NULLABLE proto3 fields, use a wrapper type.
+			fdp = &descriptorpb.FieldDescriptorProto{
+				Name:     proto.String(name),
+				Number:   proto.Int32(idx),
+				Type:     descriptorpb.FieldDescriptorProto_TYPE_MESSAGE.Enum(),
+				TypeName: proto.String(bqTypeToWrapperMap[field.GetType()]),
+				Label:    descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(),
+			}
 		}
-		return fdp, nil
 	}
-	// For NULLABLE proto3 fields, use a wrapper type.
-	return &descriptorpb.FieldDescriptorProto{
-		Name:     proto.String(name),
-		Number:   proto.Int32(idx),
-		Type:     descriptorpb.FieldDescriptorProto_TYPE_MESSAGE.Enum(),
-		TypeName: proto.String(bqTypeToWrapperMap[field.GetType()]),
-		Label:    descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(),
-	}, nil
+	if nameRequiresAnnotation(name) {
+		// TODO: need agreement across implementations on how to normalize?
+		// Possibly use golang.org/x/text/unicode/norm for ascii-fying?
+		fdp.Name = proto.String(fmt.Sprintf("unicode_field_%d", fdp.GetNumber()))
+		opts := fdp.GetOptions()
+		if opts == nil {
+			fdp.Options = &descriptorpb.FieldOptions{}
+		}
+		proto.SetExtension(fdp.Options, annotations.E_ColumnName, name)
+	}
+	return fdp, nil
+}
+
+// nameRequiresAnnotation determines whether a field name requires unicode-annotation.
+func nameRequiresAnnotation(in string) bool {
+	for i := 0; i < len(in); i++ {
+		if in[i] > unicode.MaxASCII {
+			return true
+		}
+	}
+	return false
 }
 
 // NormalizeDescriptor builds a self-contained DescriptorProto suitable for communicating schema

bigquery/storage/managedwriter/adapt/protoconversion_test.go

@@ -19,6 +19,7 @@ import (
 	"reflect"
 	"testing"
 
+	"cloud.google.com/go/bigquery/storage/managedwriter/internal/annotations"
 	"cloud.google.com/go/bigquery/storage/managedwriter/testdata"
 	"github.com/google/go-cmp/cmp"
 	storagepb "google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1"
@@ -413,6 +414,43 @@ func TestSchemaToProtoConversion(t *testing.T) {
 				},
 			},
 		},
+		{
+			description: "unicode",
+			bq: &storagepb.TableSchema{
+				Fields: []*storagepb.TableFieldSchema{
+					{Name: "foo", Type: storagepb.TableFieldSchema_STRING, Mode: storagepb.TableFieldSchema_NULLABLE},
+					{Name: "💩", Type: storagepb.TableFieldSchema_INT64, Mode: storagepb.TableFieldSchema_REQUIRED},
+					{Name: "☕_addict", Type: storagepb.TableFieldSchema_BYTES, Mode: storagepb.TableFieldSchema_REPEATED},
+				}},
+			wantProto2: func() *descriptorpb.DescriptorProto {
+				dp := &descriptorpb.DescriptorProto{
+					Name: proto.String("root"),
+					Field: []*descriptorpb.FieldDescriptorProto{
+						{
+							Name:   proto.String("foo"),
+							Number: proto.Int32(1),
+							Type:   descriptorpb.FieldDescriptorProto_TYPE_STRING.Enum(),
+							Label:  descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum()},
+						{
+							Name:    proto.String("unicode_field_2"),
+							Number:  proto.Int32(2),
+							Type:    descriptorpb.FieldDescriptorProto_TYPE_INT64.Enum(),
+							Options: &descriptorpb.FieldOptions{},
+							Label:   descriptorpb.FieldDescriptorProto_LABEL_REQUIRED.Enum()},
+						{
+							Name:    proto.String("unicode_field_3"),
+							Number:  proto.Int32(3),
+							Type:    descriptorpb.FieldDescriptorProto_TYPE_BYTES.Enum(),
+							Options: &descriptorpb.FieldOptions{},
+							Label:   descriptorpb.FieldDescriptorProto_LABEL_REPEATED.Enum(),
+						},
+					},
+				}
+				proto.SetExtension(dp.Field[1].Options, annotations.E_ColumnName, "💩")
+				proto.SetExtension(dp.Field[2].Options, annotations.E_ColumnName, "☕_addict")
+				return dp
+			}(),
+		},
 	}
 	for _, tc := range testCases {
 		// Proto2

bigquery/storage/managedwriter/internal/annotations/annotations.proto

@@ -0,0 +1,40 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package google.cloud.bigquery.storage.v1;
+
+import "google/protobuf/descriptor.proto";
+
+option go_package = "cloud.google.com/go/bigquery/storage/managedwriter/internal/annotations;annotations";
+option java_package = "com.google.cloud.bigquery.storage.v1";
+option java_multiple_files = true;
+option java_outer_classname = "AnnotationsProto";
+
+extend google.protobuf.FieldOptions {
+  // Setting the column_name extension allows users to reference
+  // bigquery column independently of the field name in the protocol buffer
+  // message.
+  //
+  // The intended use of this annotation is to reference a destination column
+  // named using characters unavailable for protobuf field names (e.g. unicode
+  // characters).
+  //
+  // More details about BigQuery naming limitations can be found here:
+  // https://cloud.google.com/bigquery/docs/schemas#column_names
+  //
+  // This extension is currently experimental.
+  optional string column_name = 454943157;
+}

bigquery/storage/managedwriter/testdata/schemas.go

@@ -257,4 +257,19 @@ var (
 			Repeated: true,
 		},
 	}
+
+	ValidationColumnAnnotations bigquery.Schema = bigquery.Schema{
+		{
+			Name: "first",
+			Type: bigquery.StringFieldType,
+		},
+		{
+			Name: "second",
+			Type: bigquery.StringFieldType,
+		},
+		{
+			Name: "特別コラム",
+			Type: bigquery.StringFieldType,
+		},
+	}
 )