From e557d3edf3537b332f406ec329a85a05cb8e865a Mon Sep 17 00:00:00 2001 From: pytimer Date: Thu, 20 Jan 2022 12:55:11 +0800 Subject: [PATCH] Imporve performance when generating spec with external dependencies --- cmd/swag/main.go | 7 +++++ gen/gen.go | 4 +++ golist.go | 75 ++++++++++++++++++++++++++++++++++++++++++++++++ packages.go | 6 ++++ parser.go | 47 +++++++++++++++++++++--------- parser_test.go | 14 +++++++++ 6 files changed, 139 insertions(+), 14 deletions(-) create mode 100644 golist.go diff --git a/cmd/swag/main.go b/cmd/swag/main.go index 0a18900f0..53366ca6b 100644 --- a/cmd/swag/main.go +++ b/cmd/swag/main.go @@ -29,6 +29,7 @@ const ( parseDepthFlag = "parseDepth" instanceNameFlag = "instanceName" overridesFileFlag = "overridesFile" + parseGoListFlag = "parseGoList" ) var initFlags = []cli.Flag{ @@ -110,6 +111,11 @@ var initFlags = []cli.Flag{ Value: gen.DefaultOverridesFile, Usage: "File to read global type overrides from.", }, + &cli.BoolFlag{ + Name: parseGoListFlag, + Value: true, + Usage: "Parse dependency via 'go list'", + }, } func initAction(c *cli.Context) error { @@ -142,6 +148,7 @@ func initAction(c *cli.Context) error { ParseDepth: c.Int(parseDepthFlag), InstanceName: c.String(instanceNameFlag), OverridesFile: c.String(overridesFileFlag), + ParseGoList: c.Bool(parseGoListFlag), }) } diff --git a/gen/gen.go b/gen/gen.go index 105b89c6f..29b3f094a 100644 --- a/gen/gen.go +++ b/gen/gen.go @@ -107,6 +107,9 @@ type Config struct { // OverridesFile defines global type overrides. OverridesFile string + + // ParseGoList whether swag use go list to parse dependency + ParseGoList bool } // Build builds swagger json file for given searchDir and mainAPIFile. Returns json @@ -151,6 +154,7 @@ func (g *Gen) Build(config *Config) error { p.ParseVendor = config.ParseVendor p.ParseDependency = config.ParseDependency p.ParseInternal = config.ParseInternal + p.ParseGoList = config.ParseGoList if err := p.ParseAPIMultiSearchDir(searchDirs, config.MainAPIFile, config.ParseDepth); err != nil { return err diff --git a/golist.go b/golist.go new file mode 100644 index 000000000..d9f9b4a9f --- /dev/null +++ b/golist.go @@ -0,0 +1,75 @@ +package swag + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "go/build" + "os/exec" + "path/filepath" +) + +func listPackages(ctx context.Context, dir string, env []string, args ...string) (pkgs []*build.Package, finalErr error) { + goArgs := append([]string{"list", "-json", "-e"}, args...) + cmd := exec.CommandContext(ctx, "go", goArgs...) + cmd.Env = env + cmd.Dir = dir + + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + var stderrBuf bytes.Buffer + cmd.Stderr = &stderrBuf + defer func() { + if stderrBuf.Len() > 0 { + finalErr = fmt.Errorf("%v\n%s", finalErr, stderrBuf.Bytes()) + } + }() + + if err := cmd.Start(); err != nil { + return nil, err + } + dec := json.NewDecoder(stdout) + for dec.More() { + var pkg build.Package + if err := dec.Decode(&pkg); err != nil { + return nil, err + } + pkgs = append(pkgs, &pkg) + } + if err := cmd.Wait(); err != nil { + return nil, err + } + return pkgs, nil +} + +func (parser *Parser) getAllGoFileInfoFromDepsByList(pkg *build.Package) error { + ignoreInternal := pkg.Goroot && !parser.ParseInternal + if ignoreInternal { // ignored internal + return nil + } + + // Skip cgo + if pkg.Name == "C" { + return nil + } + + srcDir := pkg.Dir + for i := range pkg.GoFiles { + path := filepath.Join(srcDir, pkg.GoFiles[i]) + if err := parser.parseFile(pkg.ImportPath, path, nil); err != nil { + return err + } + } + + for i := range pkg.CFiles { + path := filepath.Join(srcDir, pkg.CFiles[i]) + if err := parser.parseFile(pkg.ImportPath, path, nil); err != nil { + return err + } + } + + return nil +} diff --git a/packages.go b/packages.go index c454f0652..f165168c8 100644 --- a/packages.go +++ b/packages.go @@ -6,6 +6,7 @@ import ( "go/token" "os" "path/filepath" + "runtime" "sort" "strings" @@ -77,6 +78,11 @@ func (pkgs *PackagesDefinitions) CollectAstFile(packageDir, path string, astFile func rangeFiles(files map[*ast.File]*AstFileInfo, handle func(filename string, file *ast.File) error) error { sortedFiles := make([]*AstFileInfo, 0, len(files)) for _, info := range files { + // ignore package path prefix with 'vendor' or $GOROOT, + // because the router info of api will not be included these files. + if strings.HasPrefix(info.PackagePath, "vendor") || strings.HasPrefix(info.Path, runtime.GOROOT()) { + continue + } sortedFiles = append(sortedFiles, info) } diff --git a/parser.go b/parser.go index f4195feba..dae4f8f84 100644 --- a/parser.go +++ b/parser.go @@ -1,6 +1,7 @@ package swag import ( + "context" "encoding/json" "errors" "fmt" @@ -140,6 +141,9 @@ type Parser struct { // Overrides allows global replacements of types. A blank replacement will be skipped. Overrides map[string]string + + // ParseGoList whether swag use go list to parse dependency + ParseGoList bool } // FieldParserFactory create FieldParser @@ -283,26 +287,41 @@ func (parser *Parser) ParseAPIMultiSearchDir(searchDirs []string, mainAPIFile st return err } + // Use 'go list' command instead of depth.Resolve() if parser.ParseDependency { - var t depth.Tree - t.ResolveInternal = true - t.MaxDepth = parseDepth - - pkgName, err := getPkgName(filepath.Dir(absMainAPIFilePath)) - if err != nil { - return err - } + if parser.ParseGoList { + pkgs, err := listPackages(context.Background(), filepath.Dir(absMainAPIFilePath), nil, "-deps") + if err != nil { + return fmt.Errorf("pkg %s cannot find all dependencies, %s", filepath.Dir(absMainAPIFilePath), err) + } - err = t.Resolve(pkgName) - if err != nil { - return fmt.Errorf("pkg %s cannot find all dependencies, %s", pkgName, err) - } + length := len(pkgs) + for i := 0; i < length; i++ { + err := parser.getAllGoFileInfoFromDepsByList(pkgs[i]) + if err != nil { + return err + } + } + } else { + var t depth.Tree + t.ResolveInternal = true + t.MaxDepth = parseDepth - for i := 0; i < len(t.Root.Deps); i++ { - err := parser.getAllGoFileInfoFromDeps(&t.Root.Deps[i]) + pkgName, err := getPkgName(filepath.Dir(absMainAPIFilePath)) if err != nil { return err } + + err = t.Resolve(pkgName) + if err != nil { + return fmt.Errorf("pkg %s cannot find all dependencies, %s", pkgName, err) + } + for i := 0; i < len(t.Root.Deps); i++ { + err := parser.getAllGoFileInfoFromDeps(&t.Root.Deps[i]) + if err != nil { + return err + } + } } } diff --git a/parser_test.go b/parser_test.go index 1506acb19..ff0cbcf28 100644 --- a/parser_test.go +++ b/parser_test.go @@ -2127,6 +2127,20 @@ func TestParseExternalModels(t *testing.T) { assert.Equal(t, string(expected), string(b)) } +func TestParseGoList(t *testing.T) { + searchDir := "testdata/external_models/main" + mainAPIFile := "main.go" + p := New() + p.ParseDependency = true + p.ParseGoList = true + err := p.ParseAPI(searchDir, mainAPIFile, defaultParseDepth) + assert.NoError(t, err) + b, _ := json.MarshalIndent(p.swagger, "", " ") + expected, err := ioutil.ReadFile(filepath.Join(searchDir, "expected.json")) + assert.NoError(t, err) + assert.Equal(t, string(expected), string(b)) +} + func TestParser_ParseStructArrayObject(t *testing.T) { t.Parallel()