Skip to content

Commit

Permalink
Merge branch 'main' into regen_gocloud
Browse files Browse the repository at this point in the history
  • Loading branch information
@codyoss @yoshi-automation
codyoss authored and yoshi-automation committed Dec 2, 2021
2 parents 8302dd3 + b2b5476 commit 93fc2c8
Show file tree
Hide file tree
Showing 34 changed files with 1,223 additions and 203 deletions.
20 changes: 20 additions & 0 deletions bigquery/dataset.go
View file
Expand Up @@ -18,6 +18,7 @@ import (
"context"
"errors"
"fmt"
"strings"
"time"

"cloud.google.com/go/internal/optional"
Expand Down Expand Up @@ -88,6 +89,25 @@ func (c *Client) DatasetInProject(projectID, datasetID string) *Dataset {
}
}

// Identifier returns the ID of the dataset in the requested format.
//
// For Standard SQL format, the identifier will be quoted if the
// ProjectID contains dash (-) characters.
func (d *Dataset) Identifier(f IdentifierFormat) (string, error) {
switch f {
case LegacySQLID:
return fmt.Sprintf("%s:%s", d.ProjectID, d.DatasetID), nil
case StandardSQLID:
// Quote project identifiers if they have a dash character.
if strings.Contains(d.ProjectID, "-") {
return fmt.Sprintf("`%s`.%s", d.ProjectID, d.DatasetID), nil
}
return fmt.Sprintf("%s.%s", d.ProjectID, d.DatasetID), nil
default:
return "", ErrUnknownIdentifierFormat
}
}

// Create creates a dataset in the BigQuery service. An error will be returned if the
// dataset already exists. Pass in a DatasetMetadata value to configure the dataset.
func (d *Dataset) Create(ctx context.Context, md *DatasetMetadata) (err error) {
Expand Down
60 changes: 60 additions & 0 deletions bigquery/dataset_test.go
View file
Expand Up @@ -476,3 +476,63 @@ func TestConvertAccessEntry(t *testing.T) {
t.Error("got nil, want error")
}
}

func TestDatasetIdentifiers(t *testing.T) {
testDataset := &Dataset{
ProjectID: "p",
DatasetID: "d",
c: nil,
}
for _, tc := range []struct {
description string
in *Dataset
format IdentifierFormat
want string
wantErr bool
}{
{
description: "empty format string",
in: testDataset,
format: "",
wantErr: true,
},
{
description: "legacy",
in: testDataset,
format: LegacySQLID,
want: "p:d",
},
{
description: "standard unquoted",
in: testDataset,
format: StandardSQLID,
want: "p.d",
},
{
description: "standard w/quoting",
in: &Dataset{ProjectID: "p-p", DatasetID: "d"},
format: StandardSQLID,
want: "`p-p`.d",
},
{
description: "api resource",
in: testDataset,
format: StorageAPIResourceID,
wantErr: true,
},
} {
got, err := tc.in.Identifier(tc.format)
if tc.wantErr && err == nil {
t.Errorf("case %q: wanted err, was success", tc.description)
}
if !tc.wantErr {
if err != nil {
t.Errorf("case %q: wanted success, got err: %v", tc.description, err)
} else {
if got != tc.want {
t.Errorf("case %q: got %s, want %s", tc.description, got, tc.want)
}
}
}
}
}
4 changes: 2 additions & 2 deletions bigquery/go.mod
View file
Expand Up @@ -12,8 +12,8 @@ require (
go.opencensus.io v0.23.0
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
google.golang.org/api v0.60.0
google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1
google.golang.org/api v0.61.0
google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12
google.golang.org/grpc v1.40.0
google.golang.org/protobuf v1.27.1
)
16 changes: 8 additions & 8 deletions bigquery/go.sum
View file
Expand Up @@ -289,8 +289,8 @@ golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ
golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 h1:B333XXssMuKQeBwiNODx4TupZy7bf4sxFZnN2ZOcvUE=
golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 h1:RerP+noqYHUQ8CMRcPlC2nvTa4dcBIjegkuWdcUDuqg=
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
Expand Down Expand Up @@ -348,8 +348,8 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 h1:2B5p2L5IfGiD7+b9BOoRMC6DgObAVZV+Fsp050NqXik=
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 h1:TyHqChC80pFkXWraUUf6RuB5IqFdQieMLwwCJokV2pc=
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand Down Expand Up @@ -448,8 +448,8 @@ google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6
google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI=
google.golang.org/api v0.60.0 h1:eq/zs5WPH4J9undYM9IP1O7dSr7Yh8Y0GtSCpzGzIUk=
google.golang.org/api v0.60.0/go.mod h1:d7rl65NZAkEQ90JFzqBjcRq1TVeG5ZoGV3sSpEnnVb4=
google.golang.org/api v0.61.0 h1:TXXKS1slM3b2bZNJwD5DV/Tp6/M2cLzLOLh9PjDhrw8=
google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
Expand Down Expand Up @@ -515,9 +515,9 @@ google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEc
google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20211021150943-2b146023228c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1 h1:b9mVrqYfq3P4bCdaLg1qtBnPzUYgglsIdjZkL/fQVOE=
google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12 h1:DN5b3HU13J4sMd/QjDx34U6afpaexKTDdop+26pdjdk=
google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
Expand Down
38 changes: 22 additions & 16 deletions bigquery/integration_test.go
View file
Expand Up @@ -371,12 +371,12 @@ func TestIntegration_TableCreateView(t *testing.T) {
}
ctx := context.Background()
table := newTable(t, schema)
tableIdentifier, _ := table.Identifier(StandardSQLID)
defer table.Delete(ctx)

// Test that standard SQL views work.
view := dataset.Table("t_view_standardsql")
query := fmt.Sprintf("SELECT APPROX_COUNT_DISTINCT(name) FROM `%s.%s.%s`",
dataset.ProjectID, dataset.DatasetID, table.TableID)
query := fmt.Sprintf("SELECT APPROX_COUNT_DISTINCT(name) FROM %s", tableIdentifier)
err := view.Create(context.Background(), &TableMetadata{
ViewQuery: query,
UseStandardSQL: true,
Expand Down Expand Up @@ -936,10 +936,11 @@ func TestIntegration_DatasetUpdateAccess(t *testing.T) {
// Create a sample UDF so we can verify adding authorized UDFs
routineID := routineIDs.New()
routine := dataset.Routine(routineID)
routineSQLID, _ := routine.Identifier(StandardSQLID)

sql := fmt.Sprintf(`
CREATE FUNCTION `+"`%s`"+`(x INT64) AS (x * 3);`,
routine.FullyQualifiedName())
CREATE FUNCTION %s(x INT64) AS (x * 3);`,
routineSQLID)
if _, _, err := runQuerySQL(ctx, sql); err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -1348,13 +1349,14 @@ func TestIntegration_RoutineStoredProcedure(t *testing.T) {
// Define a simple stored procedure via DDL.
routineID := routineIDs.New()
routine := dataset.Routine(routineID)
routineSQLID, _ := routine.Identifier(StandardSQLID)
sql := fmt.Sprintf(`
CREATE OR REPLACE PROCEDURE `+"`%s`"+`(val INT64)
CREATE OR REPLACE PROCEDURE %s(val INT64)
BEGIN
SELECT CURRENT_TIMESTAMP() as ts;
SELECT val * 2 as f2;
END`,
routine.FullyQualifiedName())
routineSQLID)

if _, _, err := runQuerySQL(ctx, sql); err != nil {
t.Fatal(err)
Expand All @@ -1363,8 +1365,8 @@ func TestIntegration_RoutineStoredProcedure(t *testing.T) {

// Invoke the stored procedure.
sql = fmt.Sprintf(`
CALL `+"`%s`"+`(5)`,
routine.FullyQualifiedName())
CALL %s(5)`,
routineSQLID)

q := client.Query(sql)
it, err := q.Read(ctx)
Expand Down Expand Up @@ -2354,8 +2356,10 @@ func TestIntegration_QueryExternalHivePartitioning(t *testing.T) {
}
defer customTable.Delete(ctx)

customTableSQLID, _ := customTable.Identifier(StandardSQLID)

// Issue a test query that prunes based on the custom hive partitioning key, and verify the result is as expected.
sql := fmt.Sprintf("SELECT COUNT(*) as ct FROM `%s`.%s.%s WHERE pkey=\"foo\"", customTable.ProjectID, customTable.DatasetID, customTable.TableID)
sql := fmt.Sprintf("SELECT COUNT(*) as ct FROM %s WHERE pkey=\"foo\"", customTableSQLID)
q := client.Query(sql)
it, err := q.Read(ctx)
if err != nil {
Expand Down Expand Up @@ -3227,10 +3231,10 @@ func TestIntegration_ModelLifecycle(t *testing.T) {
// Create a model via a CREATE MODEL query
modelID := modelIDs.New()
model := dataset.Model(modelID)
modelRef := fmt.Sprintf("%s.%s.%s", dataset.ProjectID, dataset.DatasetID, modelID)
modelSQLID, _ := model.Identifier(StandardSQLID)

sql := fmt.Sprintf(`
CREATE MODEL `+"`%s`"+`
CREATE MODEL %s
OPTIONS (
model_type='linear_reg',
max_iteration=1,
Expand All @@ -3240,7 +3244,7 @@ func TestIntegration_ModelLifecycle(t *testing.T) {
SELECT 'a' AS f1, 2.0 AS label
UNION ALL
SELECT 'b' AS f1, 3.8 AS label
)`, modelRef)
)`, modelSQLID)
if _, _, err := runQuerySQL(ctx, sql); err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -3417,13 +3421,14 @@ func TestIntegration_RoutineComplexTypes(t *testing.T) {

routineID := routineIDs.New()
routine := dataset.Routine(routineID)
routineSQLID, _ := routine.Identifier(StandardSQLID)
sql := fmt.Sprintf(`
CREATE FUNCTION `+"`%s`("+`
CREATE FUNCTION %s(
arr ARRAY<STRUCT<name STRING, val INT64>>
) AS (
(SELECT SUM(IF(elem.name = "foo",elem.val,null)) FROM UNNEST(arr) AS elem)
)`,
routine.FullyQualifiedName())
routineSQLID)
if _, _, err := runQuerySQL(ctx, sql); err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -3480,10 +3485,11 @@ func TestIntegration_RoutineLifecycle(t *testing.T) {
// Create a scalar UDF routine via a CREATE FUNCTION query
routineID := routineIDs.New()
routine := dataset.Routine(routineID)
routineSQLID, _ := routine.Identifier(StandardSQLID)

sql := fmt.Sprintf(`
CREATE FUNCTION `+"`%s`"+`(x INT64) AS (x * 3);`,
routine.FullyQualifiedName())
CREATE FUNCTION %s(x INT64) AS (x * 3);`,
routineSQLID)
if _, _, err := runQuerySQL(ctx, sql); err != nil {
t.Fatal(err)
}
Expand Down
25 changes: 24 additions & 1 deletion bigquery/model.go
View file
Expand Up @@ -17,6 +17,7 @@ package bigquery
import (
"context"
"fmt"
"strings"
"time"

"cloud.google.com/go/internal/optional"
Expand All @@ -41,9 +42,31 @@ type Model struct {
c *Client
}

// Identifier returns the ID of the model in the requested format.
//
// For Standard SQL format, the identifier will be quoted if the
// ProjectID contains dash (-) characters.
func (m *Model) Identifier(f IdentifierFormat) (string, error) {
switch f {
case LegacySQLID:
return fmt.Sprintf("%s:%s.%s", m.ProjectID, m.DatasetID, m.ModelID), nil
case StandardSQLID:
// Per https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-create#model_name
// we quote the entire identifier.
out := fmt.Sprintf("%s.%s.%s", m.ProjectID, m.DatasetID, m.ModelID)
if strings.Contains(out, "-") {
out = fmt.Sprintf("`%s`", out)
}
return out, nil
default:
return "", ErrUnknownIdentifierFormat
}
}

// FullyQualifiedName returns the ID of the model in projectID:datasetID.modelid format.
func (m *Model) FullyQualifiedName() string {
return fmt.Sprintf("%s:%s.%s", m.ProjectID, m.DatasetID, m.ModelID)
s, _ := m.Identifier(LegacySQLID)
return s
}

func (m *Model) toBQ() *bq.ModelReference {
Expand Down
61 changes: 61 additions & 0 deletions bigquery/model_test.go
View file
Expand Up @@ -120,3 +120,64 @@ func TestModelMetadataUpdateToBQ(t *testing.T) {
}
}
}

func TestModelIdentifiers(t *testing.T) {
testModel := &Model{
ProjectID: "p",
DatasetID: "d",
ModelID: "m",
c: nil,
}
for _, tc := range []struct {
description string
in *Model
format IdentifierFormat
want string
wantErr bool
}{
{
description: "empty format string",
in: testModel,
format: "",
wantErr: true,
},
{
description: "legacy",
in: testModel,
format: LegacySQLID,
want: "p:d.m",
},
{
description: "standard unquoted",
in: testModel,
format: StandardSQLID,
want: "p.d.m",
},
{
description: "standard w/dash",
in: &Model{ProjectID: "p-p", DatasetID: "d", ModelID: "m"},
format: StandardSQLID,
want: "`p-p.d.m`",
},
{
description: "api resource",
in: testModel,
format: StorageAPIResourceID,
wantErr: true,
},
} {
got, err := tc.in.Identifier(tc.format)
if tc.wantErr && err == nil {
t.Errorf("case %q: wanted err, was success", tc.description)
}
if !tc.wantErr {
if err != nil {
t.Errorf("case %q: wanted success, got err: %v", tc.description, err)
} else {
if got != tc.want {
t.Errorf("case %q: got %s, want %s", tc.description, got, tc.want)
}
}
}
}
}

0 comments on commit 93fc2c8

Please sign in to comment.