-
Notifications
You must be signed in to change notification settings - Fork 399
/
loader.go
625 lines (556 loc) · 18.8 KB
/
loader.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
/*
Copyright 2019-2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package loader
import (
"fmt"
"go/ast"
"go/parser"
"go/scanner"
"go/token"
"go/types"
"io/ioutil"
"os"
"path/filepath"
"regexp"
"sync"
"golang.org/x/tools/go/packages"
"k8s.io/apimachinery/pkg/util/sets"
)
// Much of this is strongly inspired by the contents of go/packages,
// except that it allows for lazy loading of syntax and type-checking
// information to speed up cases where full traversal isn't needed.
// PrintErrors print errors associated with all packages
// in the given package graph, starting at the given root
// packages and traversing through all imports. It will skip
// any errors of the kinds specified in filterKinds. It will
// return true if any errors were printed.
func PrintErrors(pkgs []*Package, filterKinds ...packages.ErrorKind) bool {
pkgsRaw := make([]*packages.Package, len(pkgs))
for i, pkg := range pkgs {
pkgsRaw[i] = pkg.Package
}
toSkip := make(map[packages.ErrorKind]struct{})
for _, errKind := range filterKinds {
toSkip[errKind] = struct{}{}
}
hadErrors := false
packages.Visit(pkgsRaw, nil, func(pkgRaw *packages.Package) {
for _, err := range pkgRaw.Errors {
if _, skip := toSkip[err.Kind]; skip {
continue
}
hadErrors = true
fmt.Fprintln(os.Stderr, err)
}
})
return hadErrors
}
// Package is a single, unique Go package that can be
// lazily parsed and type-checked. Packages should not
// be constructed directly -- instead, use LoadRoots.
// For a given call to LoadRoots, only a single instance
// of each package exists, and thus they may be used as keys
// and for comparison.
type Package struct {
*packages.Package
imports map[string]*Package
loader *loader
sync.Mutex
}
// Imports returns the imports for the given package, indexed by
// package path (*not* name in any particular file).
func (p *Package) Imports() map[string]*Package {
if p.imports == nil {
p.imports = p.loader.packagesFor(p.Package.Imports)
}
return p.imports
}
// NeedTypesInfo indicates that type-checking information is needed for this package.
// Actual type-checking information can be accessed via the Types and TypesInfo fields.
func (p *Package) NeedTypesInfo() {
if p.TypesInfo != nil {
return
}
p.NeedSyntax()
p.loader.typeCheck(p)
}
// NeedSyntax indicates that a parsed AST is needed for this package.
// Actual ASTs can be accessed via the Syntax field.
func (p *Package) NeedSyntax() {
if p.Syntax != nil {
return
}
out := make([]*ast.File, len(p.CompiledGoFiles))
var wg sync.WaitGroup
wg.Add(len(p.CompiledGoFiles))
for i, filename := range p.CompiledGoFiles {
go func(i int, filename string) {
defer wg.Done()
src, err := ioutil.ReadFile(filename)
if err != nil {
p.AddError(err)
return
}
out[i], err = p.loader.parseFile(filename, src)
if err != nil {
p.AddError(err)
return
}
}(i, filename)
}
wg.Wait()
for _, file := range out {
if file == nil {
return
}
}
p.Syntax = out
}
// AddError adds an error to the errors associated with the given package.
func (p *Package) AddError(err error) {
switch typedErr := err.(type) {
case *os.PathError:
// file-reading errors
p.Errors = append(p.Errors, packages.Error{
Pos: typedErr.Path + ":1",
Msg: typedErr.Err.Error(),
Kind: packages.ParseError,
})
case scanner.ErrorList:
// parsing/scanning errors
for _, subErr := range typedErr {
p.Errors = append(p.Errors, packages.Error{
Pos: subErr.Pos.String(),
Msg: subErr.Msg,
Kind: packages.ParseError,
})
}
case types.Error:
// type-checking errors
p.Errors = append(p.Errors, packages.Error{
Pos: typedErr.Fset.Position(typedErr.Pos).String(),
Msg: typedErr.Msg,
Kind: packages.TypeError,
})
case ErrList:
for _, subErr := range typedErr {
p.AddError(subErr)
}
case PositionedError:
p.Errors = append(p.Errors, packages.Error{
Pos: p.loader.cfg.Fset.Position(typedErr.Pos).String(),
Msg: typedErr.Error(),
Kind: packages.UnknownError,
})
default:
// should only happen for external errors, like ref checking
p.Errors = append(p.Errors, packages.Error{
Pos: p.ID + ":-",
Msg: err.Error(),
Kind: packages.UnknownError,
})
}
}
// loader loads packages and their imports. Loaded packages will have
// type size, imports, and exports file information populated. Additional
// information, like ASTs and type-checking information, can be accessed
// via methods on individual packages.
type loader struct {
// Roots are the loaded "root" packages in the package graph loaded via
// LoadRoots.
Roots []*Package
// cfg contains the package loading config (initialized on demand)
cfg *packages.Config
// packages contains the cache of Packages indexed by the underlying
// package.Package, so that we don't ever produce two Packages with
// the same underlying packages.Package.
packages map[*packages.Package]*Package
packagesMu sync.Mutex
}
// packageFor returns a wrapped Package for the given packages.Package,
// ensuring that there's a one-to-one mapping between the two.
// It's *not* threadsafe -- use packagesFor for that.
func (l *loader) packageFor(pkgRaw *packages.Package) *Package {
if l.packages[pkgRaw] == nil {
l.packages[pkgRaw] = &Package{
Package: pkgRaw,
loader: l,
}
}
return l.packages[pkgRaw]
}
// packagesFor returns a map of Package objects for each packages.Package in the input
// map, ensuring that there's a one-to-one mapping between package.Package and Package
// (as per packageFor).
func (l *loader) packagesFor(pkgsRaw map[string]*packages.Package) map[string]*Package {
l.packagesMu.Lock()
defer l.packagesMu.Unlock()
out := make(map[string]*Package, len(pkgsRaw))
for name, rawPkg := range pkgsRaw {
out[name] = l.packageFor(rawPkg)
}
return out
}
// typeCheck type-checks the given package.
func (l *loader) typeCheck(pkg *Package) {
// don't conflict with typeCheckFromExportData
pkg.TypesInfo = &types.Info{
Types: make(map[ast.Expr]types.TypeAndValue),
Defs: make(map[*ast.Ident]types.Object),
Uses: make(map[*ast.Ident]types.Object),
Implicits: make(map[ast.Node]types.Object),
Scopes: make(map[ast.Node]*types.Scope),
Selections: make(map[*ast.SelectorExpr]*types.Selection),
}
pkg.Fset = l.cfg.Fset
pkg.Types = types.NewPackage(pkg.PkgPath, pkg.Name)
importer := importerFunc(func(path string) (*types.Package, error) {
if path == "unsafe" {
return types.Unsafe, nil
}
// The imports map is keyed by import path.
importedPkg := pkg.Imports()[path]
if importedPkg == nil {
return nil, fmt.Errorf("package %q possibly creates an import loop", path)
}
// it's possible to have a call to check in parallel to a call to this
// if one package in the package graph gets its dependency filtered out,
// but another doesn't (so one wants a "placeholder" package here, and another
// wants the full check).
//
// Thus, we need to lock here (at least for the time being) to avoid
// races between the above write to `pkg.Types` and this checking of
// importedPkg.Types.
importedPkg.Lock()
defer importedPkg.Unlock()
if importedPkg.Types != nil && importedPkg.Types.Complete() {
return importedPkg.Types, nil
}
// if we haven't already loaded typecheck data, we don't care about this package's types
return types.NewPackage(importedPkg.PkgPath, importedPkg.Name), nil
})
var errs []error
// type-check
checkConfig := &types.Config{
Importer: importer,
IgnoreFuncBodies: true, // we only need decl-level info
Error: func(err error) {
errs = append(errs, err)
},
Sizes: pkg.TypesSizes,
}
if err := types.NewChecker(checkConfig, l.cfg.Fset, pkg.Types, pkg.TypesInfo).Files(pkg.Syntax); err != nil {
errs = append(errs, err)
}
// make sure that if a given sub-import is ill-typed, we mark this package as ill-typed as well.
illTyped := len(errs) > 0
if !illTyped {
for _, importedPkg := range pkg.Imports() {
if importedPkg.IllTyped {
illTyped = true
break
}
}
}
pkg.IllTyped = illTyped
// publish errors to the package error list.
for _, err := range errs {
pkg.AddError(err)
}
}
// parseFile parses the given file, including comments.
func (l *loader) parseFile(filename string, src []byte) (*ast.File, error) {
// skip function bodies
file, err := parser.ParseFile(l.cfg.Fset, filename, src, parser.AllErrors|parser.ParseComments)
if err != nil {
return nil, err
}
return file, nil
}
// LoadRoots loads the given "root" packages by path, transitively loading
// and all imports as well.
//
// Loaded packages will have type size, imports, and exports file information
// populated. Additional information, like ASTs and type-checking information,
// can be accessed via methods on individual packages.
func LoadRoots(roots ...string) ([]*Package, error) {
return LoadRootsWithConfig(&packages.Config{}, roots...)
}
// LoadRootsWithConfig functions like LoadRoots, except that it allows passing
// a custom loading config. The config will be modified to suit the needs of
// the loader.
//
// This is generally only useful for use in testing when you need to modify
// loading settings to load from a fake location.
//
// This function will traverse Go module boundaries for roots that are file-
// system paths and end with "...". Please note this feature currently only
// supports roots that are filesystem paths. For more information, please
// refer to the high-level outline of this function's logic:
//
// 1. If no roots are provided then load the working directory and return
// early.
//
// 2. Otherwise sort the provided roots into two, distinct buckets:
//
// a. package/module names
// b. filesystem paths
//
// A filesystem path is distinguished from a Go package/module name by
// the same rules as followed by the "go" command. At a high level, a
// root is a filesystem path IFF it meets ANY of the following criteria:
//
// * is absolute
// * begins with .
// * begins with ..
//
// For more information please refer to the output of the command
// "go help packages".
//
// 3. Load the package/module roots as a single call to packages.Load. If
// there are no filesystem path roots then return early.
//
// 4. For filesystem path roots ending with "...", check to see if its
// descendants include any nested, Go modules. If so, add the directory
// that contains the nested Go module to the filesystem path roots.
//
// 5. Load the filesystem path roots and return the load packages for the
// package/module roots AND the filesystem path roots.
func LoadRootsWithConfig(cfg *packages.Config, roots ...string) ([]*Package, error) {
l := &loader{
cfg: cfg,
packages: make(map[*packages.Package]*Package),
}
l.cfg.Mode |= packages.LoadImports | packages.NeedTypesSizes
if l.cfg.Fset == nil {
l.cfg.Fset = token.NewFileSet()
}
// put our build flags first so that callers can override them
l.cfg.BuildFlags = append([]string{"-tags", "ignore_autogenerated"}, l.cfg.BuildFlags...)
// Visit the import graphs of the loaded, root packages. If an imported
// package refers to another loaded, root package, then replace the
// instance of the imported package with a reference to the loaded, root
// package. This is required to make kubebuilder markers work correctly
// when multiple root paths are loaded and types from one path reference
// types from another root path.
defer func() {
for i := range l.Roots {
visitImports(l.Roots, l.Roots[i], nil)
}
}()
// uniquePkgIDs is used to keep track of the discovered packages to be nice
// and try and prevent packages from showing up twice when nested module
// support is enabled. there is not harm that comes from this per se, but
// it makes testing easier when a known number of modules can be asserted
uniquePkgIDs := sets.String{}
// loadPackages returns the Go packages for the provided roots
//
// if validatePkgFn is nil, a package will be returned in the slice,
// otherwise the package is only returned if the result of
// validatePkgFn(pkg.ID) is truthy
loadPackages := func(roots ...string) ([]*Package, error) {
rawPkgs, err := packages.Load(l.cfg, roots...)
if err != nil {
return nil, err
}
var pkgs []*Package
for _, rp := range rawPkgs {
p := l.packageFor(rp)
if !uniquePkgIDs.Has(p.ID) {
pkgs = append(pkgs, p)
uniquePkgIDs.Insert(p.ID)
}
}
return pkgs, nil
}
// if no roots were provided then load the current package and return early
if len(roots) == 0 {
pkgs, err := loadPackages()
if err != nil {
return nil, err
}
l.Roots = append(l.Roots, pkgs...)
return l.Roots, nil
}
// pkgRoots is a slice of roots that are package/modules and fspRoots
// is a slice of roots that are local filesystem paths.
//
// please refer to this function's godoc comments for more information on
// how these two types of roots are distinguished from one another
var (
pkgRoots []string
fspRoots []string
fspRootRx = regexp.MustCompile(`^\.{1,2}`)
)
for _, r := range roots {
if filepath.IsAbs(r) || fspRootRx.MatchString(r) {
fspRoots = append(fspRoots, r)
} else {
pkgRoots = append(pkgRoots, r)
}
}
// handle the package roots by sending them into the packages.Load function
// all at once. this is more efficient, but cannot be used for the file-
// system path roots due to them needing a custom, calculated value for the
// cfg.Dir field
if len(pkgRoots) > 0 {
pkgs, err := loadPackages(pkgRoots...)
if err != nil {
return nil, err
}
l.Roots = append(l.Roots, pkgs...)
}
// if there are no filesystem path roots then go ahead and return early
if len(fspRoots) == 0 {
return l.Roots, nil
}
//
// at this point we are handling filesystem path roots
//
// ensure the cfg.Dir field is reset to its original value upon
// returning from this function. it should honestly be fine if it is
// not given most callers will not send in the cfg parameter directly,
// as it's largely for testing, but still, let's be good stewards.
defer func(d string) {
cfg.Dir = d
}(cfg.Dir)
// store the value of cfg.Dir so we can use it later if it is non-empty.
// we need to store it now as the value of cfg.Dir will be updated by
// a loop below
cfgDir := cfg.Dir
// addNestedGoModulesToRoots is given to filepath.WalkDir and adds the
// directory part of p to the list of filesystem path roots IFF p is the
// path to a file named "go.mod"
addNestedGoModulesToRoots := func(
p string,
d os.DirEntry,
e error) error {
if e != nil {
return e
}
if !d.IsDir() && filepath.Base(p) == "go.mod" {
fspRoots = append(fspRoots, filepath.Join(filepath.Dir(p), "..."))
}
return nil
}
// in the first pass over the filesystem path roots we:
//
// 1. make the root into an absolute path
//
// 2. check to see if a root uses the nested path syntax, ex. ...
//
// 3. if so, walk the root's descendants, searching for any nested Go
// modules
//
// 4. if found then the directory containing the Go module is added to
// the list of the filesystem path roots
for i := range fspRoots {
r := fspRoots[i]
// clean up the root
r = filepath.Clean(r)
// get the absolute path of the root
if !filepath.IsAbs(r) {
// if the initial value of cfg.Dir was non-empty then use it when
// building the absolute path to this root. otherwise use the
// filepath.Abs function to get the absolute path of the root based
// on the working directory
if cfgDir != "" {
r = filepath.Join(cfgDir, r)
} else {
ar, err := filepath.Abs(r)
if err != nil {
return nil, err
}
r = ar
}
}
// update the root to be an absolute path
fspRoots[i] = r
b, d := filepath.Base(r), filepath.Dir(r)
// if the base element is "..." then it means nested traversal is
// activated. this can be passed directly to the loader. however, if
// specified we also want to traverse the path manually to determine if
// there are any nested Go modules we want to add to the list of file-
// system path roots to process
if b == "..." {
if err := filepath.WalkDir(
d,
addNestedGoModulesToRoots); err != nil {
return nil, err
}
}
}
// in the second pass over the filesystem path roots we:
//
// 1. determine the directory from which to execute the loader
//
// 2. update the loader config's Dir property to be the directory from
// step one
//
// 3. determine whether the root passed to the loader should be "./."
// or "./..."
//
// 4. execute the loader with the value from step three
for _, r := range fspRoots {
b, d := filepath.Base(r), filepath.Dir(r)
// we want the base part of the path to be either "..." or ".", except
// Go's filepath utilities clean paths during manipulation, removing the
// ".". thus, if not "...", let's update the path components so that:
//
// d = r
// b = "."
if b != "..." {
d = r
b = "."
}
// update the loader configuration's Dir field to the directory part of
// the root
l.cfg.Dir = d
// update the root to be "./..." or "./."
// (with OS-specific filepath separator). please note filepath.Join
// would clean up the trailing "." character that we want preserved,
// hence the more manual path concatenation logic
r = fmt.Sprintf(".%s%s", string(filepath.Separator), b)
// load the packages from the roots
pkgs, err := loadPackages(r)
if err != nil {
return nil, err
}
l.Roots = append(l.Roots, pkgs...)
}
return l.Roots, nil
}
// visitImports walks a dependency graph, replacing imported package
// references with those from the rootPkgs list. This ensures the
// kubebuilder marker generation is handled correctly. For more info,
// please see issue 680.
func visitImports(rootPkgs []*Package, pkg *Package, seen sets.String) {
if seen == nil {
seen = sets.String{}
}
for importedPkgID, importedPkg := range pkg.Imports() {
for i := range rootPkgs {
if importedPkgID == rootPkgs[i].ID {
pkg.imports[importedPkgID] = rootPkgs[i]
}
}
if !seen.Has(importedPkgID) {
seen.Insert(importedPkgID)
visitImports(rootPkgs, importedPkg, seen)
}
}
}
// importFunc is an implementation of the single-method
// types.Importer interface based on a function value.
type importerFunc func(path string) (*types.Package, error)
func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) }