From 06d28164d89bc990065c9a292cfc1765ae37138d Mon Sep 17 00:00:00 2001 From: Cody Oss Date: Wed, 7 Sep 2022 09:07:12 -0500 Subject: [PATCH 1/3] chore(internal/gapicgen): add more tooling for generating aliases - Add new genbot local mode flag for generating genproto aliases and adding a shim dependency on genproto. - Work aliasgen into local mode --- internal/gapicgen/cmd/genbot/local.go | 2 + internal/gapicgen/cmd/genbot/main.go | 2 + internal/gapicgen/generator/gapics.go | 41 +++++++++++++++++++++ internal/gapicgen/generator/generator.go | 1 + internal/gapicgen/generator/genproto.go | 47 ++++++++++++++++++++---- internal/gapicgen/go.mod | 3 ++ internal/gapicgen/go.sum | 1 + 7 files changed, 89 insertions(+), 8 deletions(-) diff --git a/internal/gapicgen/cmd/genbot/local.go b/internal/gapicgen/cmd/genbot/local.go index 328c52f14ec..a25290b2a06 100644 --- a/internal/gapicgen/cmd/genbot/local.go +++ b/internal/gapicgen/cmd/genbot/local.go @@ -39,6 +39,7 @@ type localConfig struct { regenOnly bool forceAll bool genModule bool + genAlias bool } func genLocal(ctx context.Context, c localConfig) error { @@ -75,6 +76,7 @@ func genLocal(ctx context.Context, c localConfig) error { RegenOnly: c.regenOnly, ForceAll: c.forceAll, GenModule: c.genModule, + GenAlias: c.genAlias, } if _, err := generator.Generate(ctx, conf); err != nil { log.Printf("Generator ran (and failed) in %s\n", tmpDir) diff --git a/internal/gapicgen/cmd/genbot/main.go b/internal/gapicgen/cmd/genbot/main.go index dbf68fca11e..461149d2aa9 100644 --- a/internal/gapicgen/cmd/genbot/main.go +++ b/internal/gapicgen/cmd/genbot/main.go @@ -53,6 +53,7 @@ func main() { onlyGapics := flag.Bool("only-gapics", strToBool(os.Getenv("ONLY_GAPICS")), "Enabling stops regenerating genproto.") regenOnly := flag.Bool("regen-only", strToBool(os.Getenv("REGEN_ONLY")), "Enabling means no vetting, manifest updates, or compilation.") genModule := flag.Bool("generate-module", strToBool(os.Getenv("GENERATE_MODULE")), "Enabling means a new module will be generated for API being generated.") + genAlias := flag.Bool("generate-alias", strToBool(os.Getenv("GENERATE_ALIAS")), "Enabling means alias files will be generated.") flag.Parse() @@ -67,6 +68,7 @@ func main() { regenOnly: *regenOnly, forceAll: *forceAll, genModule: *genModule, + genAlias: *genAlias, }); err != nil { log.Fatal(err) } diff --git a/internal/gapicgen/generator/gapics.go b/internal/gapicgen/generator/gapics.go index 0d96dba0e38..8bcd97ff3a3 100644 --- a/internal/gapicgen/generator/gapics.go +++ b/internal/gapicgen/generator/gapics.go @@ -53,6 +53,7 @@ type GapicGenerator struct { genModule bool modifiedPkgs []string forceAll bool + genAlias bool } // NewGapicGenerator creates a GapicGenerator. @@ -68,6 +69,7 @@ func NewGapicGenerator(c *Config, modifiedPkgs []string) *GapicGenerator { genModule: c.GenModule, modifiedPkgs: modifiedPkgs, forceAll: c.ForceAll, + genAlias: c.GenAlias, } } @@ -104,6 +106,11 @@ func (g *GapicGenerator) Regen(ctx context.Context) error { if err := g.genVersionFile(c); err != nil { return err } + if g.genAlias { + if err := g.genAliasShim(modPath); err != nil { + return err + } + } if g.genModule { if err := gocmd.ModTidy(modPath); err != nil { return nil @@ -620,6 +627,40 @@ func (g *GapicGenerator) copyMicrogenFiles() error { return c.Run() } +func (g *GapicGenerator) genAliasShim(modPath string) error { + aliasshimBody := `// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by gapicgen. DO NOT EDIT. + +//go:build aliasshim +// +build aliasshim + +// Package aliasshim is used to keep the dependency on go-genproto during our +// go-genproto to google-cloud-go stubs migration window. +package aliasshim +` + os.MkdirAll(filepath.Join(modPath, "aliasshim"), os.ModePerm) + f, err := os.Create(filepath.Join(modPath, "aliasshim", "aliasshim.go")) + if err != nil { + return err + } + defer f.Close() + _, err = fmt.Fprint(f, aliasshimBody) + return err +} + func ParseAPIShortnames(googleapisDir string, confs []*MicrogenConfig, manualEntries []ManifestEntry) (map[string]string, error) { shortnames := map[string]string{} for _, conf := range confs { diff --git a/internal/gapicgen/generator/generator.go b/internal/gapicgen/generator/generator.go index 97c406bfa41..e967b7604e3 100644 --- a/internal/gapicgen/generator/generator.go +++ b/internal/gapicgen/generator/generator.go @@ -40,6 +40,7 @@ type Config struct { RegenOnly bool ForceAll bool GenModule bool + GenAlias bool } // Generate generates genproto and gapics. diff --git a/internal/gapicgen/generator/genproto.go b/internal/gapicgen/generator/genproto.go index 5db3b905947..e56d5091a13 100644 --- a/internal/gapicgen/generator/genproto.go +++ b/internal/gapicgen/generator/genproto.go @@ -26,6 +26,8 @@ import ( "strconv" "strings" + "cloud.google.com/go/internal/aliasfix" + "cloud.google.com/go/internal/aliasgen" "cloud.google.com/go/internal/gapicgen/execv" "cloud.google.com/go/internal/gapicgen/execv/gocmd" "cloud.google.com/go/internal/gapicgen/git" @@ -52,19 +54,25 @@ var noGRPC = map[string]bool{ // GenprotoGenerator is used to generate code for googleapis/go-genproto. type GenprotoGenerator struct { - genprotoDir string - googleapisDir string - protoSrcDir string - forceAll bool + genprotoDir string + googleapisDir string + protoSrcDir string + googleCloudDir string + gapicToGenerate string + forceAll bool + genAlias bool } // NewGenprotoGenerator creates a new GenprotoGenerator. func NewGenprotoGenerator(c *Config) *GenprotoGenerator { return &GenprotoGenerator{ - genprotoDir: c.GenprotoDir, - googleapisDir: c.GoogleapisDir, - protoSrcDir: filepath.Join(c.ProtoDir, "/src"), - forceAll: c.ForceAll, + genprotoDir: c.GenprotoDir, + googleapisDir: c.GoogleapisDir, + protoSrcDir: filepath.Join(c.ProtoDir, "/src"), + googleCloudDir: c.GapicDir, + gapicToGenerate: c.GapicToGenerate, + forceAll: c.ForceAll, + genAlias: c.GenAlias, } } @@ -99,6 +107,10 @@ func hasPrefix(s string, prefixes []string) bool { func (g *GenprotoGenerator) Regen(ctx context.Context) error { log.Println("regenerating genproto") + if g.genAlias { + return g.generateAliases() + } + // Create space to put generated .pb.go's. c := execv.Command("mkdir", "-p", "generated") c.Dir = g.genprotoDir @@ -279,3 +291,22 @@ func (g *GenprotoGenerator) moveAndCleanupGeneratedSrc() error { return nil } + +func (g *GenprotoGenerator) generateAliases() error { + for genprotoImport, newPkg := range aliasfix.GenprotoPkgMigration { + if !isMigrated(genprotoImport) || g.gapicToGenerate == "" { + continue + } + // remove the stubs dir segment from path + gapicImport := newPkg.ImportPath[:strings.LastIndex(newPkg.ImportPath, "/")] + if !strings.Contains(g.gapicToGenerate, gapicImport) { + continue + } + srdDir := filepath.Join(g.googleCloudDir, strings.TrimPrefix(newPkg.ImportPath, "cloud.google.com/go/")) + destDir := filepath.Join(g.genprotoDir, "googleapis", strings.TrimPrefix(genprotoImport, "google.golang.org/genproto/googleapis/")) + if err := aliasgen.Run(srdDir, destDir); err != nil { + return err + } + } + return nil +} diff --git a/internal/gapicgen/go.mod b/internal/gapicgen/go.mod index 7ddc080e0e4..5082edb8989 100644 --- a/internal/gapicgen/go.mod +++ b/internal/gapicgen/go.mod @@ -5,6 +5,7 @@ go 1.19 require ( cloud.google.com/go v0.102.1 cloud.google.com/go/internal/aliasfix v0.0.0 + cloud.google.com/go/internal/aliasgen v0.0.0-20220902151655-a6004e762f78 cloud.google.com/go/internal/godocfx v0.0.0-20220625055333-3f8d1627b9c2 github.com/google/go-github/v35 v35.3.0 github.com/shurcooL/githubv4 v0.0.0-20220520033151-0b4e3294ff00 @@ -39,3 +40,5 @@ require ( ) replace cloud.google.com/go/internal/aliasfix => ../aliasfix + +replace cloud.google.com/go/internal/aliasgen => ../aliasgen diff --git a/internal/gapicgen/go.sum b/internal/gapicgen/go.sum index c18d3caee30..51c91155347 100644 --- a/internal/gapicgen/go.sum +++ b/internal/gapicgen/go.sum @@ -678,6 +678,7 @@ google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11 google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.48.0 h1:rQOsyJ/8+ufEDJd/Gdsz7HG220Mh9HAhFHRGnIjda0w= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From 79a515bd27ce03b62a89c56078d5864762f4645b Mon Sep 17 00:00:00 2001 From: Cody Oss Date: Wed, 7 Sep 2022 09:18:35 -0500 Subject: [PATCH 2/3] some docs --- internal/gapicgen/cmd/genbot/README.md | 27 ++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/internal/gapicgen/cmd/genbot/README.md b/internal/gapicgen/cmd/genbot/README.md index 08268e70866..5b029a724e7 100644 --- a/internal/gapicgen/cmd/genbot/README.md +++ b/internal/gapicgen/cmd/genbot/README.md @@ -103,6 +103,33 @@ This will speed up the build a bit: -v `go env GOMODCACHE`:/root/go/pkg/mod ``` +### Generating new stubs + +Flip status in aliasfix for gapics being migrated to in progress. + +```shell +cd /path/to/internal/gapicgen +go run cloud.google.com/go/internal/gapicgen/cmd/genbot \ + -local \ + -only-gapics + -gocloud-dir=/path/to/google-cloud-go \ + -gapic=cloud.google.com/go/foo/apiv1 +``` + +### Generating type aliases + +Flip status in aliasfix for gapics being migrated to migrated. + +```shell +cd /path/to/internal/gapicgen +go run cloud.google.com/go/internal/gapicgen/cmd/genbot \ + -local \ + -generate-alias + -gocloud-dir=/path/to/google-cloud-go \ + -genproto-dir=/path/to/go-genproto \ + -gapic=cloud.google.com/go/foo/apiv1 +``` + ## FAQ ### How to bump to a later version of the microgenerator From fc375c9421a7962c6d917a920c5934f9c3770aeb Mon Sep 17 00:00:00 2001 From: Cody Oss Date: Wed, 7 Sep 2022 14:30:29 -0500 Subject: [PATCH 3/3] fixup shim --- internal/gapicgen/generator/gapics.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/gapicgen/generator/gapics.go b/internal/gapicgen/generator/gapics.go index 8bcd97ff3a3..6be09b72cb1 100644 --- a/internal/gapicgen/generator/gapics.go +++ b/internal/gapicgen/generator/gapics.go @@ -650,6 +650,8 @@ func (g *GapicGenerator) genAliasShim(modPath string) error { // Package aliasshim is used to keep the dependency on go-genproto during our // go-genproto to google-cloud-go stubs migration window. package aliasshim + +import _ "google.golang.org/genproto/protobuf/api" ` os.MkdirAll(filepath.Join(modPath, "aliasshim"), os.ModePerm) f, err := os.Create(filepath.Join(modPath, "aliasshim", "aliasshim.go"))