Imporve performance when generating spec with external dependencies

cmd/swag/main.go

@@ -29,6 +29,7 @@ const (
 	parseDepthFlag       = "parseDepth"
 	instanceNameFlag     = "instanceName"
 	overridesFileFlag    = "overridesFile"
+	parseGoListFlag      = "parseGoList"
 )
 
 var initFlags = []cli.Flag{
@@ -110,6 +111,11 @@ var initFlags = []cli.Flag{
 		Value: gen.DefaultOverridesFile,
 		Usage: "File to read global type overrides from.",
 	},
+	&cli.BoolFlag{
+		Name:  parseGoListFlag,
+		Value: true,
+		Usage: "Parse dependency via 'go list'",
+	},
 }
 
 func initAction(c *cli.Context) error {
@@ -142,6 +148,7 @@ func initAction(c *cli.Context) error {
 		ParseDepth:          c.Int(parseDepthFlag),
 		InstanceName:        c.String(instanceNameFlag),
 		OverridesFile:       c.String(overridesFileFlag),
+		ParseGoList:         c.Bool(parseGoListFlag),
 	})
 }
 

gen/gen.go

@@ -107,6 +107,9 @@ type Config struct {
 
 	// OverridesFile defines global type overrides.
 	OverridesFile string
+
+	// ParseGoList whether swag use go list to parse dependency
+	ParseGoList bool
 }
 
 // Build builds swagger json file  for given searchDir and mainAPIFile. Returns json
@@ -151,6 +154,7 @@ func (g *Gen) Build(config *Config) error {
 	p.ParseVendor = config.ParseVendor
 	p.ParseDependency = config.ParseDependency
 	p.ParseInternal = config.ParseInternal
+	p.ParseGoList = config.ParseGoList
 
 	if err := p.ParseAPIMultiSearchDir(searchDirs, config.MainAPIFile, config.ParseDepth); err != nil {
 		return err

golist.go

@@ -0,0 +1,75 @@
+package swag
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"go/build"
+	"os/exec"
+	"path/filepath"
+)
+
+func listPackages(ctx context.Context, dir string, env []string, args ...string) (pkgs []*build.Package, finalErr error) {
+	goArgs := append([]string{"list", "-json", "-e"}, args...)
+	cmd := exec.CommandContext(ctx, "go", goArgs...)
+	cmd.Env = env
+	cmd.Dir = dir
+
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, err
+	}
+	var stderrBuf bytes.Buffer
+	cmd.Stderr = &stderrBuf
+	defer func() {
+		if stderrBuf.Len() > 0 {
+			finalErr = fmt.Errorf("%v\n%s", finalErr, stderrBuf.Bytes())
+		}
+	}()
+
+	if err := cmd.Start(); err != nil {
+		return nil, err
+	}
+	dec := json.NewDecoder(stdout)
+	for dec.More() {
+		var pkg build.Package
+		if err := dec.Decode(&pkg); err != nil {
+			return nil, err
+		}
+		pkgs = append(pkgs, &pkg)
+	}
+	if err := cmd.Wait(); err != nil {
+		return nil, err
+	}
+	return pkgs, nil
+}
+
+func (parser *Parser) getAllGoFileInfoFromDepsByList(pkg *build.Package) error {
+	ignoreInternal := pkg.Goroot && !parser.ParseInternal
+	if ignoreInternal { // ignored internal
+		return nil
+	}
+
+	// Skip cgo
+	if pkg.Name == "C" {
+		return nil
+	}
+
+	srcDir := pkg.Dir
+	for i := range pkg.GoFiles {
+		path := filepath.Join(srcDir, pkg.GoFiles[i])
+		if err := parser.parseFile(pkg.ImportPath, path, nil); err != nil {
+			return err
+		}
+	}
+
+	for i := range pkg.CFiles {
+		path := filepath.Join(srcDir, pkg.CFiles[i])
+		if err := parser.parseFile(pkg.ImportPath, path, nil); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

packages.go

@@ -6,6 +6,7 @@ import (
 	"go/token"
 	"os"
 	"path/filepath"
+	"runtime"
 	"sort"
 	"strings"
 
@@ -77,6 +78,11 @@ func (pkgs *PackagesDefinitions) CollectAstFile(packageDir, path string, astFile
 func rangeFiles(files map[*ast.File]*AstFileInfo, handle func(filename string, file *ast.File) error) error {
 	sortedFiles := make([]*AstFileInfo, 0, len(files))
 	for _, info := range files {
+		// ignore package path prefix with 'vendor' or $GOROOT,
+		// because the router info of api will not be included these files.
+		if strings.HasPrefix(info.PackagePath, "vendor") || strings.HasPrefix(info.Path, runtime.GOROOT()) {
+			continue
+		}
 		sortedFiles = append(sortedFiles, info)
 	}
 

parser.go

@@ -1,6 +1,7 @@
 package swag
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -140,6 +141,9 @@ type Parser struct {
 
 	// Overrides allows global replacements of types. A blank replacement will be skipped.
 	Overrides map[string]string
+
+	// ParseGoList whether swag use go list to parse dependency
+	ParseGoList bool
 }
 
 // FieldParserFactory create FieldParser
@@ -283,26 +287,41 @@ func (parser *Parser) ParseAPIMultiSearchDir(searchDirs []string, mainAPIFile st
 		return err
 	}
 
+	// Use 'go list' command instead of depth.Resolve()
 	if parser.ParseDependency {
-		var t depth.Tree
-		t.ResolveInternal = true
-		t.MaxDepth = parseDepth
-
-		pkgName, err := getPkgName(filepath.Dir(absMainAPIFilePath))
-		if err != nil {
-			return err
-		}
+		if parser.ParseGoList {
+			pkgs, err := listPackages(context.Background(), filepath.Dir(absMainAPIFilePath), nil, "-deps")
+			if err != nil {
+				return fmt.Errorf("pkg %s cannot find all dependencies, %s", filepath.Dir(absMainAPIFilePath), err)
+			}
 
-		err = t.Resolve(pkgName)
-		if err != nil {
-			return fmt.Errorf("pkg %s cannot find all dependencies, %s", pkgName, err)
-		}
+			length := len(pkgs)
+			for i := 0; i < length; i++ {
+				err := parser.getAllGoFileInfoFromDepsByList(pkgs[i])
+				if err != nil {
+					return err
+				}
+			}
+		} else {
+			var t depth.Tree
+			t.ResolveInternal = true
+			t.MaxDepth = parseDepth
 
-		for i := 0; i < len(t.Root.Deps); i++ {
-			err := parser.getAllGoFileInfoFromDeps(&t.Root.Deps[i])
+			pkgName, err := getPkgName(filepath.Dir(absMainAPIFilePath))
 			if err != nil {
 				return err
 			}
+
+			err = t.Resolve(pkgName)
+			if err != nil {
+				return fmt.Errorf("pkg %s cannot find all dependencies, %s", pkgName, err)
+			}
+			for i := 0; i < len(t.Root.Deps); i++ {
+				err := parser.getAllGoFileInfoFromDeps(&t.Root.Deps[i])
+				if err != nil {
+					return err
+				}
+			}
 		}
 	}
 

parser_test.go

@@ -2127,6 +2127,20 @@ func TestParseExternalModels(t *testing.T) {
 	assert.Equal(t, string(expected), string(b))
 }
 
+func TestParseGoList(t *testing.T) {
+	searchDir := "testdata/external_models/main"
+	mainAPIFile := "main.go"
+	p := New()
+	p.ParseDependency = true
+	p.ParseGoList = true
+	err := p.ParseAPI(searchDir, mainAPIFile, defaultParseDepth)
+	assert.NoError(t, err)
+	b, _ := json.MarshalIndent(p.swagger, "", "    ")
+	expected, err := ioutil.ReadFile(filepath.Join(searchDir, "expected.json"))
+	assert.NoError(t, err)
+	assert.Equal(t, string(expected), string(b))
+}
+
 func TestParser_ParseStructArrayObject(t *testing.T) {
 	t.Parallel()