diff --git a/bigquery/external.go b/bigquery/external.go index c5337a00216..3d3442b7fa3 100644 --- a/bigquery/external.go +++ b/bigquery/external.go @@ -230,17 +230,22 @@ type CSVOptions struct { // An optional custom string that will represent a NULL // value in CSV import data. NullMarker string + + // Preserves the embedded ASCII control characters (the first 32 characters in the ASCII-table, + // from '\\x00' to '\\x1F') when loading from CSV. Only applicable to CSV, ignored for other formats. + PreserveASCIIControlCharacters bool } func (o *CSVOptions) populateExternalDataConfig(c *bq.ExternalDataConfiguration) { c.CsvOptions = &bq.CsvOptions{ - AllowJaggedRows: o.AllowJaggedRows, - AllowQuotedNewlines: o.AllowQuotedNewlines, - Encoding: string(o.Encoding), - FieldDelimiter: o.FieldDelimiter, - Quote: o.quote(), - SkipLeadingRows: o.SkipLeadingRows, - NullMarker: o.NullMarker, + AllowJaggedRows: o.AllowJaggedRows, + AllowQuotedNewlines: o.AllowQuotedNewlines, + Encoding: string(o.Encoding), + FieldDelimiter: o.FieldDelimiter, + Quote: o.quote(), + SkipLeadingRows: o.SkipLeadingRows, + NullMarker: o.NullMarker, + PreserveAsciiControlCharacters: o.PreserveASCIIControlCharacters, } } @@ -267,12 +272,13 @@ func (o *CSVOptions) setQuote(ps *string) { func bqToCSVOptions(q *bq.CsvOptions) *CSVOptions { o := &CSVOptions{ - AllowJaggedRows: q.AllowJaggedRows, - AllowQuotedNewlines: q.AllowQuotedNewlines, - Encoding: Encoding(q.Encoding), - FieldDelimiter: q.FieldDelimiter, - SkipLeadingRows: q.SkipLeadingRows, - NullMarker: q.NullMarker, + AllowJaggedRows: q.AllowJaggedRows, + AllowQuotedNewlines: q.AllowQuotedNewlines, + Encoding: Encoding(q.Encoding), + FieldDelimiter: q.FieldDelimiter, + SkipLeadingRows: q.SkipLeadingRows, + NullMarker: q.NullMarker, + PreserveASCIIControlCharacters: q.PreserveAsciiControlCharacters, } o.setQuote(q.Quote) return o diff --git a/bigquery/file.go b/bigquery/file.go index 44eab5dc070..9afdbb53599 100644 --- a/bigquery/file.go +++ b/bigquery/file.go @@ -93,6 +93,7 @@ func (fc *FileConfig) populateLoadConfig(conf *bq.JobConfigurationLoad) { conf.IgnoreUnknownValues = fc.IgnoreUnknownValues conf.MaxBadRecords = fc.MaxBadRecords conf.NullMarker = fc.NullMarker + conf.PreserveAsciiControlCharacters = fc.PreserveASCIIControlCharacters if fc.Schema != nil { conf.Schema = fc.Schema.toBQ() } @@ -120,6 +121,7 @@ func bqPopulateFileConfig(conf *bq.JobConfigurationLoad, fc *FileConfig) { fc.Encoding = Encoding(conf.Encoding) fc.FieldDelimiter = conf.FieldDelimiter fc.CSVOptions.NullMarker = conf.NullMarker + fc.CSVOptions.PreserveASCIIControlCharacters = conf.PreserveAsciiControlCharacters fc.CSVOptions.setQuote(conf.Quote) } diff --git a/bigquery/file_test.go b/bigquery/file_test.go index e57633f3635..77ba8b2610b 100644 --- a/bigquery/file_test.go +++ b/bigquery/file_test.go @@ -33,13 +33,14 @@ var ( nestedFieldSchema(), }, CSVOptions: CSVOptions{ - Quote: hyphen, - FieldDelimiter: "\t", - SkipLeadingRows: 8, - AllowJaggedRows: true, - AllowQuotedNewlines: true, - Encoding: UTF_8, - NullMarker: "marker", + Quote: hyphen, + FieldDelimiter: "\t", + SkipLeadingRows: 8, + AllowJaggedRows: true, + AllowQuotedNewlines: true, + Encoding: UTF_8, + NullMarker: "marker", + PreserveASCIIControlCharacters: true, }, } ) @@ -63,16 +64,17 @@ func TestFileConfigPopulateLoadConfig(t *testing.T) { description: "csv", fileConfig: &fc, want: &bq.JobConfigurationLoad{ - SourceFormat: "CSV", - FieldDelimiter: "\t", - SkipLeadingRows: 8, - AllowJaggedRows: true, - AllowQuotedNewlines: true, - Autodetect: true, - Encoding: "UTF-8", - MaxBadRecords: 7, - IgnoreUnknownValues: true, - NullMarker: "marker", + SourceFormat: "CSV", + FieldDelimiter: "\t", + SkipLeadingRows: 8, + AllowJaggedRows: true, + AllowQuotedNewlines: true, + Autodetect: true, + Encoding: "UTF-8", + MaxBadRecords: 7, + IgnoreUnknownValues: true, + NullMarker: "marker", + PreserveAsciiControlCharacters: true, Schema: &bq.TableSchema{ Fields: []*bq.TableFieldSchema{ bqStringFieldSchema(), @@ -150,13 +152,14 @@ func TestFileConfigPopulateExternalDataConfig(t *testing.T) { bqNestedFieldSchema(), }}, CsvOptions: &bq.CsvOptions{ - AllowJaggedRows: true, - AllowQuotedNewlines: true, - Encoding: "UTF-8", - FieldDelimiter: "\t", - Quote: &hyphen, - SkipLeadingRows: 8, - NullMarker: "marker", + AllowJaggedRows: true, + AllowQuotedNewlines: true, + Encoding: "UTF-8", + FieldDelimiter: "\t", + Quote: &hyphen, + SkipLeadingRows: 8, + NullMarker: "marker", + PreserveAsciiControlCharacters: true, }, }, },