From bfb9a1c91005896e94c03a32a560b7a606403a4f Mon Sep 17 00:00:00 2001 From: Henning Surmeier Date: Thu, 8 Oct 2020 15:41:31 +0200 Subject: [PATCH 1/4] re-extract (2020-10-08) --- _internal_/cfg/cfg.go | 64 + _internal_/goroot/gc.go | 15 +- _internal_/lazyregexp/lazyre.go | 78 + _internal_/lazytemplate/lazytemplate.go | 52 + _internal_/xcoff/file.go | 12 +- cmd/_internal_/objabi/autotype.go | 2 +- cmd/_internal_/objabi/doc.go | 6 +- cmd/_internal_/objabi/flag.go | 10 +- cmd/_internal_/objabi/funcdata.go | 38 +- cmd/_internal_/objabi/funcid.go | 15 + cmd/_internal_/objabi/head.go | 9 +- cmd/_internal_/objabi/line.go | 54 +- cmd/_internal_/objabi/reloctype.go | 72 +- cmd/_internal_/objabi/reloctype_string.go | 4 +- cmd/_internal_/objabi/stack.go | 2 +- cmd/_internal_/objabi/symkind.go | 6 +- cmd/_internal_/objabi/symkind_string.go | 24 +- cmd/_internal_/objabi/typekind.go | 1 - cmd/_internal_/objabi/util.go | 62 +- cmd/_internal_/objabi/zbootstrap.go | 6 +- cmd/_internal_/sys/arch.go | 24 +- cmd/_internal_/sys/supported.go | 87 ++ cmd/go/_internal_/auth/auth.go | 25 + cmd/go/_internal_/auth/netrc.go | 111 ++ cmd/go/_internal_/base/base.go | 34 +- cmd/go/_internal_/base/env.go | 38 +- cmd/go/_internal_/base/flag.go | 5 + cmd/go/_internal_/base/goflags.go | 15 +- cmd/go/_internal_/base/signal_unix.go | 2 +- cmd/go/_internal_/cache/cache.go | 108 +- cmd/go/_internal_/cache/default.go | 3 +- cmd/go/_internal_/cfg/cfg.go | 235 ++- cmd/go/_internal_/cfg/zdefaultcc.go | 4 +- cmd/go/_internal_/cfg/zosarch.go | 15 +- cmd/go/_internal_/dirhash/hash.go | 103 -- cmd/go/_internal_/get/discovery.go | 16 +- cmd/go/_internal_/get/get.go | 55 +- cmd/go/_internal_/get/path.go | 6 +- cmd/go/_internal_/get/vcs.go | 250 ++-- cmd/go/_internal_/imports/build.go | 67 +- cmd/go/_internal_/imports/scan.go | 2 +- cmd/go/_internal_/imports/tags.go | 15 + cmd/go/_internal_/load/path.go | 24 - cmd/go/_internal_/load/pkg.go | 1303 +++++++++++------ cmd/go/_internal_/load/test.go | 148 +- cmd/go/_internal_/lockedfile/lockedfile.go | 65 + cmd/go/_internal_/lockedfile/mutex.go | 11 +- cmd/go/_internal_/modconv/convert.go | 39 +- cmd/go/_internal_/modconv/dep.go | 72 +- cmd/go/_internal_/modconv/glide.go | 7 +- cmd/go/_internal_/modconv/glock.go | 7 +- cmd/go/_internal_/modconv/godeps.go | 4 +- cmd/go/_internal_/modconv/modconv.go | 2 +- cmd/go/_internal_/modconv/tsv.go | 7 +- cmd/go/_internal_/modconv/vconf.go | 7 +- cmd/go/_internal_/modconv/vjson.go | 4 +- cmd/go/_internal_/modconv/vmanifest.go | 4 +- cmd/go/_internal_/modconv/vyml.go | 7 +- cmd/go/_internal_/modfetch/cache.go | 235 ++- .../_internal_/modfetch/codehost/codehost.go | 63 +- cmd/go/_internal_/modfetch/codehost/git.go | 290 ++-- cmd/go/_internal_/modfetch/codehost/svn.go | 154 ++ cmd/go/_internal_/modfetch/codehost/vcs.go | 149 +- cmd/go/_internal_/modfetch/coderepo.go | 761 +++++++--- cmd/go/_internal_/modfetch/fetch.go | 607 ++++++-- cmd/go/_internal_/modfetch/insecure.go | 16 + cmd/go/_internal_/modfetch/key.go | 9 + cmd/go/_internal_/modfetch/proxy.go | 384 ++++- cmd/go/_internal_/modfetch/pseudo.go | 161 +- cmd/go/_internal_/modfetch/repo.go | 115 +- cmd/go/_internal_/modfetch/sumdb.go | 280 ++++ cmd/go/_internal_/modfetch/unzip.go | 173 --- cmd/go/_internal_/modfetch/web.go | 31 - cmd/go/_internal_/modfile/gopkgin.go | 47 - cmd/go/_internal_/modfile/print.go | 164 --- cmd/go/_internal_/modfile/read.go | 869 ----------- cmd/go/_internal_/modfile/rule.go | 740 ---------- cmd/go/_internal_/modinfo/info.go | 2 +- cmd/go/_internal_/modload/build.go | 174 ++- cmd/go/_internal_/modload/help.go | 177 ++- cmd/go/_internal_/modload/import.go | 289 +++- cmd/go/_internal_/modload/init.go | 598 +++++--- cmd/go/_internal_/modload/list.go | 77 +- cmd/go/_internal_/modload/load.go | 788 +++++----- cmd/go/_internal_/modload/modfile.go | 164 +++ cmd/go/_internal_/modload/mvs.go | 259 ++++ cmd/go/_internal_/modload/query.go | 628 +++++++- cmd/go/_internal_/modload/search.go | 127 +- cmd/go/_internal_/modload/stat_unix.go | 31 + cmd/go/_internal_/modload/vendor.go | 217 +++ cmd/go/_internal_/module/module.go | 540 ------- cmd/go/_internal_/mvs/mvs.go | 251 +++- cmd/go/_internal_/par/work.go | 41 + cmd/go/_internal_/renameio/renameio.go | 44 +- cmd/go/_internal_/robustio/robustio.go | 53 + cmd/go/_internal_/robustio/robustio_other.go | 28 + cmd/go/_internal_/search/search.go | 300 ++-- cmd/go/_internal_/semver/semver.go | 388 ----- cmd/go/_internal_/str/path.go | 47 +- cmd/go/_internal_/web/api.go | 237 +++ cmd/go/_internal_/web/http.go | 246 +++- cmd/go/_internal_/web/security.go | 16 - cmd/go/_internal_/web/url.go | 95 ++ cmd/go/_internal_/web/url_other.go | 21 + cmd/go/_internal_/web2/web.go | 299 ---- cmd/go/_internal_/work/action.go | 142 +- cmd/go/_internal_/work/build.go | 153 +- cmd/go/_internal_/work/buildid.go | 48 +- cmd/go/_internal_/work/exec.go | 500 ++++--- cmd/go/_internal_/work/gc.go | 138 +- cmd/go/_internal_/work/gccgo.go | 70 +- cmd/go/_internal_/work/init.go | 150 +- cmd/go/_internal_/work/security.go | 42 +- extract/go.mod | 2 + extract/main.go | 2 +- 115 files changed, 9427 insertions(+), 6668 deletions(-) create mode 100644 _internal_/cfg/cfg.go create mode 100644 _internal_/lazyregexp/lazyre.go create mode 100644 _internal_/lazytemplate/lazytemplate.go create mode 100644 cmd/go/_internal_/auth/auth.go create mode 100644 cmd/go/_internal_/auth/netrc.go delete mode 100644 cmd/go/_internal_/dirhash/hash.go create mode 100644 cmd/go/_internal_/modfetch/codehost/svn.go create mode 100644 cmd/go/_internal_/modfetch/insecure.go create mode 100644 cmd/go/_internal_/modfetch/key.go create mode 100644 cmd/go/_internal_/modfetch/sumdb.go delete mode 100644 cmd/go/_internal_/modfetch/unzip.go delete mode 100644 cmd/go/_internal_/modfetch/web.go delete mode 100644 cmd/go/_internal_/modfile/gopkgin.go delete mode 100644 cmd/go/_internal_/modfile/print.go delete mode 100644 cmd/go/_internal_/modfile/read.go delete mode 100644 cmd/go/_internal_/modfile/rule.go create mode 100644 cmd/go/_internal_/modload/modfile.go create mode 100644 cmd/go/_internal_/modload/mvs.go create mode 100644 cmd/go/_internal_/modload/stat_unix.go create mode 100644 cmd/go/_internal_/modload/vendor.go delete mode 100644 cmd/go/_internal_/module/module.go create mode 100644 cmd/go/_internal_/robustio/robustio.go create mode 100644 cmd/go/_internal_/robustio/robustio_other.go delete mode 100644 cmd/go/_internal_/semver/semver.go create mode 100644 cmd/go/_internal_/web/api.go delete mode 100644 cmd/go/_internal_/web/security.go create mode 100644 cmd/go/_internal_/web/url.go create mode 100644 cmd/go/_internal_/web/url_other.go delete mode 100644 cmd/go/_internal_/web2/web.go diff --git a/_internal_/cfg/cfg.go b/_internal_/cfg/cfg.go new file mode 100644 index 0000000..bdbe9df --- /dev/null +++ b/_internal_/cfg/cfg.go @@ -0,0 +1,64 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cfg holds configuration shared by the Go command and internal/testenv. +// Definitions that don't need to be exposed outside of cmd/go should be in +// cmd/go/internal/cfg instead of this package. +package cfg + +// KnownEnv is a list of environment variables that affect the operation +// of the Go command. +const KnownEnv = ` + AR + CC + CGO_CFLAGS + CGO_CFLAGS_ALLOW + CGO_CFLAGS_DISALLOW + CGO_CPPFLAGS + CGO_CPPFLAGS_ALLOW + CGO_CPPFLAGS_DISALLOW + CGO_CXXFLAGS + CGO_CXXFLAGS_ALLOW + CGO_CXXFLAGS_DISALLOW + CGO_ENABLED + CGO_FFLAGS + CGO_FFLAGS_ALLOW + CGO_FFLAGS_DISALLOW + CGO_LDFLAGS + CGO_LDFLAGS_ALLOW + CGO_LDFLAGS_DISALLOW + CXX + FC + GCCGO + GO111MODULE + GO386 + GOARCH + GOARM + GOBIN + GOCACHE + GOENV + GOEXE + GOFLAGS + GOGCCFLAGS + GOHOSTARCH + GOHOSTOS + GOINSECURE + GOMIPS + GOMIPS64 + GOMODCACHE + GONOPROXY + GONOSUMDB + GOOS + GOPATH + GOPPC64 + GOPRIVATE + GOPROXY + GOROOT + GOSUMDB + GOTMPDIR + GOTOOLDIR + GOWASM + GO_EXTLINK_ENABLED + PKG_CONFIG +` diff --git a/_internal_/goroot/gc.go b/_internal_/goroot/gc.go index e064026..e51fb66 100644 --- a/_internal_/goroot/gc.go +++ b/_internal_/goroot/gc.go @@ -121,18 +121,9 @@ func (gd *gccgoDirs) isStandard(path string) bool { } for _, dir := range gd.dirs { - full := filepath.Join(dir, path) - pkgdir, pkg := filepath.Split(full) - for _, p := range [...]string{ - full, - full + ".gox", - pkgdir + "lib" + pkg + ".so", - pkgdir + "lib" + pkg + ".a", - full + ".o", - } { - if fi, err := os.Stat(p); err == nil && !fi.IsDir() { - return true - } + full := filepath.Join(dir, path) + ".gox" + if fi, err := os.Stat(full); err == nil && !fi.IsDir() { + return true } } diff --git a/_internal_/lazyregexp/lazyre.go b/_internal_/lazyregexp/lazyre.go new file mode 100644 index 0000000..7619e38 --- /dev/null +++ b/_internal_/lazyregexp/lazyre.go @@ -0,0 +1,78 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package lazyregexp is a thin wrapper over regexp, allowing the use of global +// regexp variables without forcing them to be compiled at init. +package lazyregexp + +import ( + "os" + "regexp" + "strings" + "sync" +) + +// Regexp is a wrapper around regexp.Regexp, where the underlying regexp will be +// compiled the first time it is needed. +type Regexp struct { + str string + once sync.Once + rx *regexp.Regexp +} + +func (r *Regexp) re() *regexp.Regexp { + r.once.Do(r.build) + return r.rx +} + +func (r *Regexp) build() { + r.rx = regexp.MustCompile(r.str) + r.str = "" +} + +func (r *Regexp) FindSubmatch(s []byte) [][]byte { + return r.re().FindSubmatch(s) +} + +func (r *Regexp) FindStringSubmatch(s string) []string { + return r.re().FindStringSubmatch(s) +} + +func (r *Regexp) FindStringSubmatchIndex(s string) []int { + return r.re().FindStringSubmatchIndex(s) +} + +func (r *Regexp) ReplaceAllString(src, repl string) string { + return r.re().ReplaceAllString(src, repl) +} + +func (r *Regexp) FindString(s string) string { + return r.re().FindString(s) +} + +func (r *Regexp) FindAllString(s string, n int) []string { + return r.re().FindAllString(s, n) +} + +func (r *Regexp) MatchString(s string) bool { + return r.re().MatchString(s) +} + +func (r *Regexp) SubexpNames() []string { + return r.re().SubexpNames() +} + +var inTest = len(os.Args) > 0 && strings.HasSuffix(strings.TrimSuffix(os.Args[0], ".exe"), ".test") + +// New creates a new lazy regexp, delaying the compiling work until it is first +// needed. If the code is being run as part of tests, the regexp compiling will +// happen immediately. +func New(str string) *Regexp { + lr := &Regexp{str: str} + if inTest { + // In tests, always compile the regexps early. + lr.re() + } + return lr +} diff --git a/_internal_/lazytemplate/lazytemplate.go b/_internal_/lazytemplate/lazytemplate.go new file mode 100644 index 0000000..3d79e60 --- /dev/null +++ b/_internal_/lazytemplate/lazytemplate.go @@ -0,0 +1,52 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package lazytemplate is a thin wrapper over text/template, allowing the use +// of global template variables without forcing them to be parsed at init. +package lazytemplate + +import ( + "io" + "os" + "strings" + "sync" + "text/template" +) + +// Template is a wrapper around text/template.Template, where the underlying +// template will be parsed the first time it is needed. +type Template struct { + name, text string + + once sync.Once + tmpl *template.Template +} + +func (r *Template) tp() *template.Template { + r.once.Do(r.build) + return r.tmpl +} + +func (r *Template) build() { + r.tmpl = template.Must(template.New(r.name).Parse(r.text)) + r.name, r.text = "", "" +} + +func (r *Template) Execute(w io.Writer, data interface{}) error { + return r.tp().Execute(w, data) +} + +var inTest = len(os.Args) > 0 && strings.HasSuffix(strings.TrimSuffix(os.Args[0], ".exe"), ".test") + +// New creates a new lazy template, delaying the parsing work until it is first +// needed. If the code is being run as part of tests, the template parsing will +// happen immediately. +func New(name, text string) *Template { + lt := &Template{name: name, text: text} + if inTest { + // In tests, always parse the templates early. + lt.tp() + } + return lt +} diff --git a/_internal_/xcoff/file.go b/_internal_/xcoff/file.go index 9a21f9e..fed4950 100644 --- a/_internal_/xcoff/file.go +++ b/_internal_/xcoff/file.go @@ -334,8 +334,8 @@ func NewFile(r io.ReaderAt) (*File, error) { // If this symbol is a function, it must retrieve its size from // its AUX_FCN entry. - // It can happend that a function symbol doesn't have any AUX_FCN. - // In this case, needAuxFcn is false and their size will be set to 0 + // It can happen that a function symbol doesn't have any AUX_FCN. + // In this case, needAuxFcn is false and their size will be set to 0. if needAuxFcn { switch f.TargetMachine { case U802TOCMAGIC: @@ -412,10 +412,10 @@ func NewFile(r io.ReaderAt) (*File, error) { sect.Relocs[i].Type = rel.Rtype sect.Relocs[i].Length = rel.Rsize&0x3F + 1 - if rel.Rsize&0x80 == 1 { + if rel.Rsize&0x80 != 0 { sect.Relocs[i].Signed = true } - if rel.Rsize&0x40 == 1 { + if rel.Rsize&0x40 != 0 { sect.Relocs[i].InstructionFixed = true } @@ -428,10 +428,10 @@ func NewFile(r io.ReaderAt) (*File, error) { sect.Relocs[i].Symbol = idxToSym[int(rel.Rsymndx)] sect.Relocs[i].Type = rel.Rtype sect.Relocs[i].Length = rel.Rsize&0x3F + 1 - if rel.Rsize&0x80 == 1 { + if rel.Rsize&0x80 != 0 { sect.Relocs[i].Signed = true } - if rel.Rsize&0x40 == 1 { + if rel.Rsize&0x40 != 0 { sect.Relocs[i].InstructionFixed = true } } diff --git a/cmd/_internal_/objabi/autotype.go b/cmd/_internal_/objabi/autotype.go index 9621062..8df1eee 100644 --- a/cmd/_internal_/objabi/autotype.go +++ b/cmd/_internal_/objabi/autotype.go @@ -1,5 +1,5 @@ // Derived from Inferno utils/6l/l.h and related files. -// https://bitbucket.org/inferno-os/inferno-os/src/default/utils/6l/l.h +// https://bitbucket.org/inferno-os/inferno-os/src/master/utils/6l/l.h // // Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. // Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) diff --git a/cmd/_internal_/objabi/doc.go b/cmd/_internal_/objabi/doc.go index 03dc9fb..08e922b 100644 --- a/cmd/_internal_/objabi/doc.go +++ b/cmd/_internal_/objabi/doc.go @@ -22,10 +22,12 @@ // // The file format is: // -// - magic header: "\x00go112ld" +// - magic header: "\x00go114ld" // - byte 1 - version number // - sequence of strings giving dependencies (imported packages) // - empty string (marks end of sequence) +// - number of entries in the following sequence +// - sequence of filename strings to generate debug information // - sequence of symbol references used by the defined symbols // - byte 0xff (marks end of sequence) // - sequence of integer lengths: @@ -38,7 +40,7 @@ // - data, the content of the defined symbols // - sequence of defined symbols // - byte 0xff (marks end of sequence) -// - magic footer: "\xffgo112ld" +// - magic footer: "\xffgo114ld" // // All integers are stored in a zigzag varint format. // See golang.org/s/go12symtab for a definition. diff --git a/cmd/_internal_/objabi/flag.go b/cmd/_internal_/objabi/flag.go index a9876ed..6870812 100644 --- a/cmd/_internal_/objabi/flag.go +++ b/cmd/_internal_/objabi/flag.go @@ -86,6 +86,10 @@ func (versionFlag) Set(s string) error { name = name[strings.LastIndex(name, `/`)+1:] name = name[strings.LastIndex(name, `\`)+1:] name = strings.TrimSuffix(name, ".exe") + + // If there's an active experiment, include that, + // to distinguish go1.10.2 with an experiment + // from go1.10.2 without an experiment. p := Expstring() if p == DefaultExpstring() { p = "" @@ -101,12 +105,6 @@ func (versionFlag) Set(s string) error { // build ID of the binary, so that if the compiler is changed and // rebuilt, we notice and rebuild all packages. if s == "full" { - // If there's an active experiment, include that, - // to distinguish go1.10.2 with an experiment - // from go1.10.2 without an experiment. - if x := Expstring(); x != "" { - p += " " + x - } if strings.HasPrefix(Version, "devel") { p += " buildID=" + buildID } diff --git a/cmd/_internal_/objabi/funcdata.go b/cmd/_internal_/objabi/funcdata.go index ba951e5..4a3d5cd 100644 --- a/cmd/_internal_/objabi/funcdata.go +++ b/cmd/_internal_/objabi/funcdata.go @@ -11,14 +11,17 @@ package objabi // ../../../runtime/symtab.go. const ( - PCDATA_StackMapIndex = 0 - PCDATA_InlTreeIndex = 1 - PCDATA_RegMapIndex = 2 + PCDATA_RegMapIndex = 0 // if !go115ReduceLiveness + PCDATA_UnsafePoint = 0 // if go115ReduceLiveness + PCDATA_StackMapIndex = 1 + PCDATA_InlTreeIndex = 2 + FUNCDATA_ArgsPointerMaps = 0 FUNCDATA_LocalsPointerMaps = 1 - FUNCDATA_InlTree = 2 - FUNCDATA_RegPointerMaps = 3 - FUNCDATA_StackObjects = 4 + FUNCDATA_RegPointerMaps = 2 // if !go115ReduceLiveness + FUNCDATA_StackObjects = 3 + FUNCDATA_InlTree = 4 + FUNCDATA_OpenCodedDeferInfo = 5 // ArgsSizeUnknown is set in Func.argsize to mark all functions // whose argument size is unknown (C vararg functions, and @@ -26,3 +29,26 @@ const ( // This value is generated by the compiler, assembler, or linker. ArgsSizeUnknown = -0x80000000 ) + +// Special PCDATA values. +const ( + // PCDATA_RegMapIndex values. + // + // Only if !go115ReduceLiveness. + PCDATA_RegMapUnsafe = -2 // Unsafe for async preemption + + // PCDATA_UnsafePoint values. + PCDATA_UnsafePointSafe = -1 // Safe for async preemption + PCDATA_UnsafePointUnsafe = -2 // Unsafe for async preemption + + // PCDATA_Restart1(2) apply on a sequence of instructions, within + // which if an async preemption happens, we should back off the PC + // to the start of the sequence when resuming. + // We need two so we can distinguish the start/end of the sequence + // in case that two sequences are next to each other. + PCDATA_Restart1 = -3 + PCDATA_Restart2 = -4 + + // Like PCDATA_Restart1, but back to function entry if async preempted. + PCDATA_RestartAtEntry = -5 +) diff --git a/cmd/_internal_/objabi/funcid.go b/cmd/_internal_/objabi/funcid.go index 43ddcfe..a0a75d5 100644 --- a/cmd/_internal_/objabi/funcid.go +++ b/cmd/_internal_/objabi/funcid.go @@ -37,6 +37,8 @@ const ( FuncID_debugCallV1 FuncID_gopanic FuncID_panicwrap + FuncID_handleAsyncEvent + FuncID_asyncPreempt FuncID_wrapper // any autogenerated code (hash/eq algorithms, method wrappers, etc.) ) @@ -82,6 +84,19 @@ func GetFuncID(name, file string) FuncID { return FuncID_gopanic case "runtime.panicwrap": return FuncID_panicwrap + case "runtime.handleAsyncEvent": + return FuncID_handleAsyncEvent + case "runtime.asyncPreempt": + return FuncID_asyncPreempt + case "runtime.deferreturn": + // Don't show in the call stack (used when invoking defer functions) + return FuncID_wrapper + case "runtime.runOpenDeferFrame": + // Don't show in the call stack (used when invoking defer functions) + return FuncID_wrapper + case "runtime.reflectcallSave": + // Don't show in the call stack (used when invoking defer functions) + return FuncID_wrapper } if file == "" { return FuncID_wrapper diff --git a/cmd/_internal_/objabi/head.go b/cmd/_internal_/objabi/head.go index 989a301..a6c73e1 100644 --- a/cmd/_internal_/objabi/head.go +++ b/cmd/_internal_/objabi/head.go @@ -1,5 +1,5 @@ // Derived from Inferno utils/6l/l.h and related files. -// https://bitbucket.org/inferno-os/inferno-os/src/default/utils/6l/l.h +// https://bitbucket.org/inferno-os/inferno-os/src/master/utils/6l/l.h // // Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. // Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) @@ -42,7 +42,6 @@ const ( Hfreebsd Hjs Hlinux - Hnacl Hnetbsd Hopenbsd Hplan9 @@ -65,15 +64,13 @@ func (h *HeadType) Set(s string) error { *h = Hjs case "linux", "android": *h = Hlinux - case "nacl": - *h = Hnacl case "netbsd": *h = Hnetbsd case "openbsd": *h = Hopenbsd case "plan9": *h = Hplan9 - case "solaris": + case "illumos", "solaris": *h = Hsolaris case "windows": *h = Hwindows @@ -97,8 +94,6 @@ func (h *HeadType) String() string { return "js" case Hlinux: return "linux" - case Hnacl: - return "nacl" case Hnetbsd: return "netbsd" case Hopenbsd: diff --git a/cmd/_internal_/objabi/line.go b/cmd/_internal_/objabi/line.go index 1c671b2..178c836 100644 --- a/cmd/_internal_/objabi/line.go +++ b/cmd/_internal_/objabi/line.go @@ -7,6 +7,7 @@ package objabi import ( "os" "path/filepath" + "strings" ) // WorkingDir returns the current working directory @@ -21,32 +22,63 @@ func WorkingDir() string { return filepath.ToSlash(path) } -// AbsFile returns the absolute filename for file in the given directory. -// It also removes a leading pathPrefix, or else rewrites a leading $GOROOT -// prefix to the literal "$GOROOT". +// AbsFile returns the absolute filename for file in the given directory, +// as rewritten by the rewrites argument. +// For unrewritten paths, AbsFile rewrites a leading $GOROOT prefix to the literal "$GOROOT". // If the resulting path is the empty string, the result is "??". -func AbsFile(dir, file, pathPrefix string) string { +// +// The rewrites argument is a ;-separated list of rewrites. +// Each rewrite is of the form "prefix" or "prefix=>replace", +// where prefix must match a leading sequence of path elements +// and is either removed entirely or replaced by the replacement. +func AbsFile(dir, file, rewrites string) string { abs := file if dir != "" && !filepath.IsAbs(file) { abs = filepath.Join(dir, file) } - if pathPrefix != "" && hasPathPrefix(abs, pathPrefix) { - if abs == pathPrefix { - abs = "" - } else { - abs = abs[len(pathPrefix)+1:] + start := 0 + for i := 0; i <= len(rewrites); i++ { + if i == len(rewrites) || rewrites[i] == ';' { + if new, ok := applyRewrite(abs, rewrites[start:i]); ok { + abs = new + goto Rewritten + } + start = i + 1 } - } else if hasPathPrefix(abs, GOROOT) { + } + if hasPathPrefix(abs, GOROOT) { abs = "$GOROOT" + abs[len(GOROOT):] } + +Rewritten: if abs == "" { abs = "??" } - return abs } +// applyRewrite applies the rewrite to the path, +// returning the rewritten path and a boolean +// indicating whether the rewrite applied at all. +func applyRewrite(path, rewrite string) (string, bool) { + prefix, replace := rewrite, "" + if j := strings.LastIndex(rewrite, "=>"); j >= 0 { + prefix, replace = rewrite[:j], rewrite[j+len("=>"):] + } + + if prefix == "" || !hasPathPrefix(path, prefix) { + return path, false + } + if len(path) == len(prefix) { + return replace, true + } + if replace == "" { + return path[len(prefix)+1:], true + } + return replace + path[len(prefix):], true +} + // Does s have t as a path prefix? // That is, does s == t or does s begin with t followed by a slash? // For portability, we allow ASCII case folding, so that hasPathPrefix("a/b/c", "A/B") is true. diff --git a/cmd/_internal_/objabi/reloctype.go b/cmd/_internal_/objabi/reloctype.go index 9313a6d..980d3b3 100644 --- a/cmd/_internal_/objabi/reloctype.go +++ b/cmd/_internal_/objabi/reloctype.go @@ -1,5 +1,5 @@ // Derived from Inferno utils/6l/l.h and related files. -// https://bitbucket.org/inferno-os/inferno-os/src/default/utils/6l/l.h +// https://bitbucket.org/inferno-os/inferno-os/src/master/utils/6l/l.h // // Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. // Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) @@ -30,7 +30,7 @@ package objabi -type RelocType int32 +type RelocType int16 //go:generate stringer -type=RelocType const ( @@ -64,6 +64,8 @@ const ( // R_CALLMIPS (only used on mips64) resolves to non-PC-relative target address // of a CALL (JAL) instruction, by encoding the address into the instruction. R_CALLMIPS + // R_CALLRISCV marks RISC-V CALLs for stack checking. + R_CALLRISCV R_CONST R_PCREL // R_TLS_LE, used on 386, amd64, and ARM, resolves to the offset of the @@ -132,6 +134,26 @@ const ( // slot of the referenced symbol. R_ARM64_GOTPCREL + // R_ARM64_GOT resolves a GOT-relative instruction sequence, usually an adrp + // followed by another ld instruction. + R_ARM64_GOT + + // R_ARM64_PCREL resolves a PC-relative addresses instruction sequence, usually an + // adrp followed by another add instruction. + R_ARM64_PCREL + + // R_ARM64_LDST8 sets a LD/ST immediate value to bits [11:0] of a local address. + R_ARM64_LDST8 + + // R_ARM64_LDST32 sets a LD/ST immediate value to bits [11:2] of a local address. + R_ARM64_LDST32 + + // R_ARM64_LDST64 sets a LD/ST immediate value to bits [11:3] of a local address. + R_ARM64_LDST64 + + // R_ARM64_LDST128 sets a LD/ST immediate value to bits [11:4] of a local address. + R_ARM64_LDST128 + // PPC64. // R_POWER_TLS_LE is used to implement the "local exec" model for tls @@ -180,6 +202,16 @@ const ( // relocated symbol rather than the symbol's address. R_ADDRPOWER_TOCREL_DS + // RISC-V. + + // R_RISCV_PCREL_ITYPE resolves a 32-bit PC-relative address using an + // AUIPC + I-type instruction pair. + R_RISCV_PCREL_ITYPE + + // R_RISCV_PCREL_STYPE resolves a 32-bit PC-relative address using an + // AUIPC + S-type instruction pair. + R_RISCV_PCREL_STYPE + // R_PCRELDBL relocates s390x 2-byte aligned PC-relative addresses. // TODO(mundaym): remove once variants can be serialized - see issue 14218. R_PCRELDBL @@ -190,24 +222,48 @@ const ( // R_ADDRMIPSTLS (only used on mips64) resolves to the low 16 bits of a TLS // address (offset from thread pointer), by encoding it into the instruction. R_ADDRMIPSTLS + // R_ADDRCUOFF resolves to a pointer-sized offset from the start of the // symbol's DWARF compile unit. R_ADDRCUOFF // R_WASMIMPORT resolves to the index of the WebAssembly function import. R_WASMIMPORT + + // R_XCOFFREF (only used on aix/ppc64) prevents garbage collection by ld + // of a symbol. This isn't a real relocation, it can be placed in anywhere + // in a symbol and target any symbols. + R_XCOFFREF ) +// IsDirectCall reports whether r is a relocation for a direct call. +// A direct call is a CALL instruction that takes the target address +// as an immediate. The address is embedded into the instruction, possibly +// with limited width. An indirect call is a CALL instruction that takes +// the target address in register or memory. +func (r RelocType) IsDirectCall() bool { + switch r { + case R_CALL, R_CALLARM, R_CALLARM64, R_CALLMIPS, R_CALLPOWER, R_CALLRISCV: + return true + } + return false +} + // IsDirectJump reports whether r is a relocation for a direct jump. -// A direct jump is a CALL or JMP instruction that takes the target address -// as immediate. The address is embedded into the instruction, possibly -// with limited width. -// An indirect jump is a CALL or JMP instruction that takes the target address -// in register or memory. +// A direct jump is a JMP instruction that takes the target address +// as an immediate. The address is embedded into the instruction, possibly +// with limited width. An indirect jump is a JMP instruction that takes +// the target address in register or memory. func (r RelocType) IsDirectJump() bool { switch r { - case R_CALL, R_CALLARM, R_CALLARM64, R_CALLPOWER, R_CALLMIPS, R_JMPMIPS: + case R_JMPMIPS: return true } return false } + +// IsDirectCallOrJump reports whether r is a relocation for a direct +// call or a direct jump. +func (r RelocType) IsDirectCallOrJump() bool { + return r.IsDirectCall() || r.IsDirectJump() +} diff --git a/cmd/_internal_/objabi/reloctype_string.go b/cmd/_internal_/objabi/reloctype_string.go index 2cd3a94..83dfe71 100644 --- a/cmd/_internal_/objabi/reloctype_string.go +++ b/cmd/_internal_/objabi/reloctype_string.go @@ -4,9 +4,9 @@ package objabi import "strconv" -const _RelocType_name = "R_ADDRR_ADDRPOWERR_ADDRARM64R_ADDRMIPSR_ADDROFFR_WEAKADDROFFR_SIZER_CALLR_CALLARMR_CALLARM64R_CALLINDR_CALLPOWERR_CALLMIPSR_CONSTR_PCRELR_TLS_LER_TLS_IER_GOTOFFR_PLT0R_PLT1R_PLT2R_USEFIELDR_USETYPER_METHODOFFR_POWER_TOCR_GOTPCRELR_JMPMIPSR_DWARFSECREFR_DWARFFILEREFR_ARM64_TLS_LER_ARM64_TLS_IER_ARM64_GOTPCRELR_POWER_TLS_LER_POWER_TLS_IER_POWER_TLSR_ADDRPOWER_DSR_ADDRPOWER_GOTR_ADDRPOWER_PCRELR_ADDRPOWER_TOCRELR_ADDRPOWER_TOCREL_DSR_PCRELDBLR_ADDRMIPSUR_ADDRMIPSTLSR_ADDRCUOFFR_WASMIMPORT" +const _RelocType_name = "R_ADDRR_ADDRPOWERR_ADDRARM64R_ADDRMIPSR_ADDROFFR_WEAKADDROFFR_SIZER_CALLR_CALLARMR_CALLARM64R_CALLINDR_CALLPOWERR_CALLMIPSR_CALLRISCVR_CONSTR_PCRELR_TLS_LER_TLS_IER_GOTOFFR_PLT0R_PLT1R_PLT2R_USEFIELDR_USETYPER_METHODOFFR_POWER_TOCR_GOTPCRELR_JMPMIPSR_DWARFSECREFR_DWARFFILEREFR_ARM64_TLS_LER_ARM64_TLS_IER_ARM64_GOTPCRELR_ARM64_GOTR_ARM64_PCRELR_ARM64_LDST8R_ARM64_LDST32R_ARM64_LDST64R_ARM64_LDST128R_POWER_TLS_LER_POWER_TLS_IER_POWER_TLSR_ADDRPOWER_DSR_ADDRPOWER_GOTR_ADDRPOWER_PCRELR_ADDRPOWER_TOCRELR_ADDRPOWER_TOCREL_DSR_RISCV_PCREL_ITYPER_RISCV_PCREL_STYPER_PCRELDBLR_ADDRMIPSUR_ADDRMIPSTLSR_ADDRCUOFFR_WASMIMPORTR_XCOFFREF" -var _RelocType_index = [...]uint16{0, 6, 17, 28, 38, 47, 60, 66, 72, 81, 92, 101, 112, 122, 129, 136, 144, 152, 160, 166, 172, 178, 188, 197, 208, 219, 229, 238, 251, 265, 279, 293, 309, 323, 337, 348, 362, 377, 394, 412, 433, 443, 454, 467, 478, 490} +var _RelocType_index = [...]uint16{0, 6, 17, 28, 38, 47, 60, 66, 72, 81, 92, 101, 112, 122, 133, 140, 147, 155, 163, 171, 177, 183, 189, 199, 208, 219, 230, 240, 249, 262, 276, 290, 304, 320, 331, 344, 357, 371, 385, 400, 414, 428, 439, 453, 468, 485, 503, 524, 543, 562, 572, 583, 596, 607, 619, 629} func (i RelocType) String() string { i -= 1 diff --git a/cmd/_internal_/objabi/stack.go b/cmd/_internal_/objabi/stack.go index 71f4f95..1552a32 100644 --- a/cmd/_internal_/objabi/stack.go +++ b/cmd/_internal_/objabi/stack.go @@ -18,7 +18,7 @@ const ( ) // Initialize StackGuard and StackLimit according to target system. -var StackGuard = 880*stackGuardMultiplier() + StackSystem +var StackGuard = 928*stackGuardMultiplier() + StackSystem var StackLimit = StackGuard - StackSystem - StackSmall // stackGuardMultiplier returns a multiplier to apply to the default diff --git a/cmd/_internal_/objabi/symkind.go b/cmd/_internal_/objabi/symkind.go index ccbefef..129a3e9 100644 --- a/cmd/_internal_/objabi/symkind.go +++ b/cmd/_internal_/objabi/symkind.go @@ -1,5 +1,5 @@ // Derived from Inferno utils/6l/l.h and related files. -// https://bitbucket.org/inferno-os/inferno-os/src/default/utils/6l/l.h +// https://bitbucket.org/inferno-os/inferno-os/src/master/utils/6l/l.h // // Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. // Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) @@ -59,7 +59,7 @@ const ( SDWARFINFO SDWARFRANGE SDWARFLOC - SDWARFMISC + SDWARFLINES // ABI alias. An ABI alias symbol is an empty symbol with a // single relocation with 0 size that references the native // function implementation symbol. @@ -67,6 +67,8 @@ const ( // TODO(austin): Remove this and all uses once the compiler // generates real ABI wrappers rather than symbol aliases. SABIALIAS + // Coverage instrumentation counter for libfuzzer. + SLIBFUZZER_EXTRA_COUNTER // Update cmd/link/internal/sym/AbiSymKindToSymKind for new SymKind values. ) diff --git a/cmd/_internal_/objabi/symkind_string.go b/cmd/_internal_/objabi/symkind_string.go index 2b9a908..919a666 100644 --- a/cmd/_internal_/objabi/symkind_string.go +++ b/cmd/_internal_/objabi/symkind_string.go @@ -4,9 +4,29 @@ package objabi import "strconv" -const _SymKind_name = "SxxxSTEXTSRODATASNOPTRDATASDATASBSSSNOPTRBSSSTLSBSSSDWARFINFOSDWARFRANGESDWARFLOCSDWARFMISCSABIALIAS" +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[Sxxx-0] + _ = x[STEXT-1] + _ = x[SRODATA-2] + _ = x[SNOPTRDATA-3] + _ = x[SDATA-4] + _ = x[SBSS-5] + _ = x[SNOPTRBSS-6] + _ = x[STLSBSS-7] + _ = x[SDWARFINFO-8] + _ = x[SDWARFRANGE-9] + _ = x[SDWARFLOC-10] + _ = x[SDWARFLINES-11] + _ = x[SABIALIAS-12] + _ = x[SLIBFUZZER_EXTRA_COUNTER-13] +} + +const _SymKind_name = "SxxxSTEXTSRODATASNOPTRDATASDATASBSSSNOPTRBSSSTLSBSSSDWARFINFOSDWARFRANGESDWARFLOCSDWARFLINESSABIALIASSLIBFUZZER_EXTRA_COUNTER" -var _SymKind_index = [...]uint8{0, 4, 9, 16, 26, 31, 35, 44, 51, 61, 72, 81, 91, 100} +var _SymKind_index = [...]uint8{0, 4, 9, 16, 26, 31, 35, 44, 51, 61, 72, 81, 92, 101, 125} func (i SymKind) String() string { if i >= SymKind(len(_SymKind_index)-1) { diff --git a/cmd/_internal_/objabi/typekind.go b/cmd/_internal_/objabi/typekind.go index d8a62ab..5cfc825 100644 --- a/cmd/_internal_/objabi/typekind.go +++ b/cmd/_internal_/objabi/typekind.go @@ -36,6 +36,5 @@ const ( KindUnsafePointer KindDirectIface = 1 << 5 KindGCProg = 1 << 6 - KindNoPointers = 1 << 7 KindMask = (1 << 5) - 1 ) diff --git a/cmd/_internal_/objabi/util.go b/cmd/_internal_/objabi/util.go index 38c2a03..abdfe99 100644 --- a/cmd/_internal_/objabi/util.go +++ b/cmd/_internal_/objabi/util.go @@ -25,12 +25,27 @@ var ( GOARCH = envOr("GOARCH", defaultGOARCH) GOOS = envOr("GOOS", defaultGOOS) GO386 = envOr("GO386", defaultGO386) + GOAMD64 = goamd64() GOARM = goarm() GOMIPS = gomips() GOMIPS64 = gomips64() + GOPPC64 = goppc64() + GOWASM = gowasm() + GO_LDSO = defaultGO_LDSO Version = version ) +const ( + ElfRelocOffset = 256 + MachoRelocOffset = 2048 // reserve enough space for ELF relocations + Go115AMD64 = "alignedjumps" // Should be "alignedjumps" or "normaljumps"; this replaces environment variable introduced in CL 219357. +) + +// TODO(1.16): assuming no issues in 1.15 release, remove this and related constant. +func goamd64() string { + return Go115AMD64 +} + func goarm() int { switch v := envOr("GOARM", defaultGOARM); v { case "5": @@ -63,6 +78,49 @@ func gomips64() string { panic("unreachable") } +func goppc64() int { + switch v := envOr("GOPPC64", defaultGOPPC64); v { + case "power8": + return 8 + case "power9": + return 9 + } + log.Fatalf("Invalid GOPPC64 value. Must be power8 or power9.") + panic("unreachable") +} + +type gowasmFeatures struct { + SignExt bool + SatConv bool +} + +func (f gowasmFeatures) String() string { + var flags []string + if f.SatConv { + flags = append(flags, "satconv") + } + if f.SignExt { + flags = append(flags, "signext") + } + return strings.Join(flags, ",") +} + +func gowasm() (f gowasmFeatures) { + for _, opt := range strings.Split(envOr("GOWASM", ""), ",") { + switch opt { + case "satconv": + f.SatConv = true + case "signext": + f.SignExt = true + case "": + // ignore + default: + log.Fatalf("Invalid GOWASM value. No such feature: " + opt) + } + } + return +} + func Getgoextlinkenabled() string { return envOr("GO_EXTLINK_ENABLED", defaultGO_EXTLINK_ENABLED) } @@ -76,7 +134,7 @@ func init() { } func Framepointer_enabled(goos, goarch string) bool { - return framepointer_enabled != 0 && (goarch == "amd64" && goos != "nacl" || goarch == "arm64" && goos == "linux") + return framepointer_enabled != 0 && (goarch == "amd64" || goarch == "arm64" && (goos == "linux" || goos == "darwin")) } func addexp(s string) { @@ -104,6 +162,7 @@ var ( framepointer_enabled int = 1 Fieldtrack_enabled int Preemptibleloops_enabled int + Staticlockranking_enabled int ) // Toolchain experiments. @@ -117,6 +176,7 @@ var exper = []struct { {"fieldtrack", &Fieldtrack_enabled}, {"framepointer", &framepointer_enabled}, {"preemptibleloops", &Preemptibleloops_enabled}, + {"staticlockranking", &Staticlockranking_enabled}, } var defaultExpstring = Expstring() diff --git a/cmd/_internal_/objabi/zbootstrap.go b/cmd/_internal_/objabi/zbootstrap.go index 12c769f..8ec1ff1 100644 --- a/cmd/_internal_/objabi/zbootstrap.go +++ b/cmd/_internal_/objabi/zbootstrap.go @@ -5,12 +5,14 @@ package objabi import "runtime" const defaultGO386 = `sse2` -const defaultGOARM = `7` +const defaultGOARM = `5` const defaultGOMIPS = `hardfloat` const defaultGOMIPS64 = `hardfloat` +const defaultGOPPC64 = `power8` const defaultGOOS = runtime.GOOS const defaultGOARCH = runtime.GOARCH const defaultGO_EXTLINK_ENABLED = `` -const version = `go1.12.4` +const defaultGO_LDSO = `` +const version = `go1.15.2` const stackGuardMultiplierDefault = 1 const goexperiment = `` diff --git a/cmd/_internal_/sys/arch.go b/cmd/_internal_/sys/arch.go index fac4ef4..7387ff3 100644 --- a/cmd/_internal_/sys/arch.go +++ b/cmd/_internal_/sys/arch.go @@ -7,8 +7,7 @@ package sys import "encoding/binary" // ArchFamily represents a family of one or more related architectures. -// For example, amd64 and amd64p32 are both members of the AMD64 family, -// and ppc64 and ppc64le are both members of the PPC64 family. +// For example, ppc64 and ppc64le are both members of the PPC64 family. type ArchFamily byte const ( @@ -20,6 +19,7 @@ const ( MIPS MIPS64 PPC64 + RISCV64 S390X Wasm ) @@ -71,15 +71,6 @@ var ArchAMD64 = &Arch{ MinLC: 1, } -var ArchAMD64P32 = &Arch{ - Name: "amd64p32", - Family: AMD64, - ByteOrder: binary.LittleEndian, - PtrSize: 4, - RegSize: 8, - MinLC: 1, -} - var ArchARM = &Arch{ Name: "arm", Family: ARM, @@ -152,6 +143,15 @@ var ArchPPC64LE = &Arch{ MinLC: 4, } +var ArchRISCV64 = &Arch{ + Name: "riscv64", + Family: RISCV64, + ByteOrder: binary.LittleEndian, + PtrSize: 8, + RegSize: 8, + MinLC: 4, +} + var ArchS390X = &Arch{ Name: "s390x", Family: S390X, @@ -173,7 +173,6 @@ var ArchWasm = &Arch{ var Archs = [...]*Arch{ Arch386, ArchAMD64, - ArchAMD64P32, ArchARM, ArchARM64, ArchMIPS, @@ -182,6 +181,7 @@ var Archs = [...]*Arch{ ArchMIPS64LE, ArchPPC64, ArchPPC64LE, + ArchRISCV64, ArchS390X, ArchWasm, } diff --git a/cmd/_internal_/sys/supported.go b/cmd/_internal_/sys/supported.go index a53da6e..c27b3b9 100644 --- a/cmd/_internal_/sys/supported.go +++ b/cmd/_internal_/sys/supported.go @@ -6,6 +6,9 @@ package sys // RaceDetectorSupported reports whether goos/goarch supports the race // detector. There is a copy of this function in cmd/dist/test.go. +// Race detector only supports 48-bit VMA on arm64. But it will always +// return true for arm64, because we don't have VMA size information during +// the compile time. func RaceDetectorSupported(goos, goarch string) bool { switch goos { case "linux": @@ -27,3 +30,87 @@ func MSanSupported(goos, goarch string) bool { return false } } + +// MustLinkExternal reports whether goos/goarch requires external linking. +func MustLinkExternal(goos, goarch string) bool { + switch goos { + case "android": + if goarch != "arm64" { + return true + } + case "darwin": + if goarch == "arm64" { + return true + } + } + return false +} + +// BuildModeSupported reports whether goos/goarch supports the given build mode +// using the given compiler. +func BuildModeSupported(compiler, buildmode, goos, goarch string) bool { + if compiler == "gccgo" { + return true + } + + platform := goos + "/" + goarch + + switch buildmode { + case "archive": + return true + + case "c-archive": + // TODO(bcmills): This seems dubious. + // Do we really support c-archive mode on js/wasm‽ + return platform != "linux/ppc64" + + case "c-shared": + switch platform { + case "linux/amd64", "linux/arm", "linux/arm64", "linux/386", "linux/ppc64le", "linux/s390x", + "android/amd64", "android/arm", "android/arm64", "android/386", + "freebsd/amd64", + "darwin/amd64", + "windows/amd64", "windows/386": + return true + } + return false + + case "default": + return true + + case "exe": + return true + + case "pie": + switch platform { + case "linux/386", "linux/amd64", "linux/arm", "linux/arm64", "linux/ppc64le", "linux/s390x", + "android/amd64", "android/arm", "android/arm64", "android/386", + "freebsd/amd64", + "darwin/amd64", + "aix/ppc64", + "windows/386", "windows/amd64", "windows/arm": + return true + } + return false + + case "shared": + switch platform { + case "linux/386", "linux/amd64", "linux/arm", "linux/arm64", "linux/ppc64le", "linux/s390x": + return true + } + return false + + case "plugin": + switch platform { + case "linux/amd64", "linux/arm", "linux/arm64", "linux/386", "linux/s390x", "linux/ppc64le", + "android/amd64", "android/arm", "android/arm64", "android/386", + "darwin/amd64", + "freebsd/amd64": + return true + } + return false + + default: + return false + } +} diff --git a/cmd/go/_internal_/auth/auth.go b/cmd/go/_internal_/auth/auth.go new file mode 100644 index 0000000..fe5a89d --- /dev/null +++ b/cmd/go/_internal_/auth/auth.go @@ -0,0 +1,25 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package auth provides access to user-provided authentication credentials. +package auth + +import "net/http" + +// AddCredentials fills in the user's credentials for req, if any. +// The return value reports whether any matching credentials were found. +func AddCredentials(req *http.Request) (added bool) { + host := req.URL.Hostname() + + // TODO(golang.org/issue/26232): Support arbitrary user-provided credentials. + netrcOnce.Do(readNetrc) + for _, l := range netrc { + if l.machine == host { + req.SetBasicAuth(l.login, l.password) + return true + } + } + + return false +} diff --git a/cmd/go/_internal_/auth/netrc.go b/cmd/go/_internal_/auth/netrc.go new file mode 100644 index 0000000..a97827e --- /dev/null +++ b/cmd/go/_internal_/auth/netrc.go @@ -0,0 +1,111 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package auth + +import ( + "io/ioutil" + "os" + "path/filepath" + "runtime" + "strings" + "sync" +) + +type netrcLine struct { + machine string + login string + password string +} + +var ( + netrcOnce sync.Once + netrc []netrcLine + netrcErr error +) + +func parseNetrc(data string) []netrcLine { + // See https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html + // for documentation on the .netrc format. + var nrc []netrcLine + var l netrcLine + inMacro := false + for _, line := range strings.Split(data, "\n") { + if inMacro { + if line == "" { + inMacro = false + } + continue + } + + f := strings.Fields(line) + i := 0 + for ; i < len(f)-1; i += 2 { + // Reset at each "machine" token. + // “The auto-login process searches the .netrc file for a machine token + // that matches […]. Once a match is made, the subsequent .netrc tokens + // are processed, stopping when the end of file is reached or another + // machine or a default token is encountered.” + switch f[i] { + case "machine": + l = netrcLine{machine: f[i+1]} + case "default": + break + case "login": + l.login = f[i+1] + case "password": + l.password = f[i+1] + case "macdef": + // “A macro is defined with the specified name; its contents begin with + // the next .netrc line and continue until a null line (consecutive + // new-line characters) is encountered.” + inMacro = true + } + if l.machine != "" && l.login != "" && l.password != "" { + nrc = append(nrc, l) + l = netrcLine{} + } + } + + if i < len(f) && f[i] == "default" { + // “There can be only one default token, and it must be after all machine tokens.” + break + } + } + + return nrc +} + +func netrcPath() (string, error) { + if env := os.Getenv("NETRC"); env != "" { + return env, nil + } + dir, err := os.UserHomeDir() + if err != nil { + return "", err + } + base := ".netrc" + if runtime.GOOS == "windows" { + base = "_netrc" + } + return filepath.Join(dir, base), nil +} + +func readNetrc() { + path, err := netrcPath() + if err != nil { + netrcErr = err + return + } + + data, err := ioutil.ReadFile(path) + if err != nil { + if !os.IsNotExist(err) { + netrcErr = err + } + return + } + + netrc = parseNetrc(string(data)) +} diff --git a/cmd/go/_internal_/base/base.go b/cmd/go/_internal_/base/base.go index 4028a03..7695003 100644 --- a/cmd/go/_internal_/base/base.go +++ b/cmd/go/_internal_/base/base.go @@ -7,11 +7,8 @@ package base import ( - "bytes" - "errors" "flag" "fmt" - "go/scanner" "log" "os" "os/exec" @@ -30,7 +27,7 @@ type Command struct { Run func(cmd *Command, args []string) // UsageLine is the one-line usage message. - // The first word in the line is taken to be the command name. + // The words between "go" and the first flag or argument in the line are taken to be the command name. UsageLine string // Short is the short description shown in the 'go help' output. @@ -82,7 +79,8 @@ func (c *Command) Name() string { func (c *Command) Usage() { fmt.Fprintf(os.Stderr, "usage: %s\n", c.UsageLine) fmt.Fprintf(os.Stderr, "Run 'go help %s' for details.\n", c.LongName()) - os.Exit(2) + SetExitStatus(2) + Exit() } // Runnable reports whether the command can be run; otherwise @@ -131,6 +129,10 @@ func SetExitStatus(n int) { exitMu.Unlock() } +func GetExitStatus() int { + return exitStatus +} + // Run runs the command, with stdout and stderr // connected to the go command's own stdout and stderr. // If the command fails, Run reports the error using Errorf. @@ -167,25 +169,3 @@ func RunStdin(cmdline []string) { // Usage is the usage-reporting function, filled in by package main // but here for reference by other packages. var Usage func() - -// ExpandScanner expands a scanner.List error into all the errors in the list. -// The default Error method only shows the first error -// and does not shorten paths. -func ExpandScanner(err error) error { - // Look for parser errors. - if err, ok := err.(scanner.ErrorList); ok { - // Prepare error with \n before each message. - // When printed in something like context: %v - // this will put the leading file positions each on - // its own line. It will also show all the errors - // instead of just the first, as err.Error does. - var buf bytes.Buffer - for _, e := range err { - e.Pos.Filename = ShortPath(e.Pos.Filename) - buf.WriteString("\n") - buf.WriteString(e.Error()) - } - return errors.New(buf.String()) - } - return err -} diff --git a/cmd/go/_internal_/base/env.go b/cmd/go/_internal_/base/env.go index fcade9d..5f2665d 100644 --- a/cmd/go/_internal_/base/env.go +++ b/cmd/go/_internal_/base/env.go @@ -4,34 +4,12 @@ package base -import "strings" - -// EnvForDir returns a copy of the environment -// suitable for running in the given directory. -// The environment is the current process's environment -// but with an updated $PWD, so that an os.Getwd in the -// child will be faster. -func EnvForDir(dir string, base []string) []string { - // Internally we only use rooted paths, so dir is rooted. - // Even if dir is not rooted, no harm done. - return MergeEnvLists([]string{"PWD=" + dir}, base) -} - -// MergeEnvLists merges the two environment lists such that -// variables with the same name in "in" replace those in "out". -// This always returns a newly allocated slice. -func MergeEnvLists(in, out []string) []string { - out = append([]string(nil), out...) -NextVar: - for _, inkv := range in { - k := strings.SplitAfterN(inkv, "=", 2)[0] - for i, outkv := range out { - if strings.HasPrefix(outkv, k) { - out[i] = inkv - continue NextVar - } - } - out = append(out, inkv) - } - return out +// AppendPWD returns the result of appending PWD=dir to the environment base. +// +// The resulting environment makes os.Getwd more efficient for a subprocess +// running in dir. +func AppendPWD(base []string, dir string) []string { + // Internally we only use absolute paths, so dir is absolute. + // Even if dir is not absolute, no harm done. + return append(base, "PWD="+dir) } diff --git a/cmd/go/_internal_/base/flag.go b/cmd/go/_internal_/base/flag.go index 061413d..ea6e0dc 100644 --- a/cmd/go/_internal_/base/flag.go +++ b/cmd/go/_internal_/base/flag.go @@ -33,3 +33,8 @@ func AddBuildFlagsNX(flags *flag.FlagSet) { flags.BoolVar(&cfg.BuildN, "n", false, "") flags.BoolVar(&cfg.BuildX, "x", false, "") } + +// AddLoadFlags adds the -mod build flag to the flag set. +func AddLoadFlags(flags *flag.FlagSet) { + flags.StringVar(&cfg.BuildMod, "mod", "", "") +} diff --git a/cmd/go/_internal_/base/goflags.go b/cmd/go/_internal_/base/goflags.go index 59f4a19..2956ea6 100644 --- a/cmd/go/_internal_/base/goflags.go +++ b/cmd/go/_internal_/base/goflags.go @@ -7,7 +7,6 @@ package base import ( "flag" "fmt" - "os" "runtime" "strings" @@ -62,7 +61,7 @@ func InitGOFLAGS() { // (Both will show the GOFLAGS setting if let succeed.) hideErrors := cfg.CmdName == "env" || cfg.CmdName == "bug" - goflags = strings.Fields(os.Getenv("GOFLAGS")) + goflags = strings.Fields(cfg.Getenv("GOFLAGS")) if goflags == nil { goflags = []string{} // avoid work on later InitGOFLAGS call } @@ -103,7 +102,7 @@ type boolFlag interface { } // SetFromGOFLAGS sets the flags in the given flag set using settings in $GOFLAGS. -func SetFromGOFLAGS(flags flag.FlagSet) { +func SetFromGOFLAGS(flags *flag.FlagSet) { InitGOFLAGS() // This loop is similar to flag.Parse except that it ignores @@ -126,14 +125,18 @@ func SetFromGOFLAGS(flags flag.FlagSet) { if f == nil { continue } + + // Use flags.Set consistently (instead of f.Value.Set) so that a subsequent + // call to flags.Visit will correctly visit the flags that have been set. + if fb, ok := f.Value.(boolFlag); ok && fb.IsBoolFlag() { if hasValue { - if err := fb.Set(value); err != nil { + if err := flags.Set(f.Name, value); err != nil { fmt.Fprintf(flags.Output(), "go: invalid boolean value %q for flag %s (from %s): %v\n", value, name, where, err) flags.Usage() } } else { - if err := fb.Set("true"); err != nil { + if err := flags.Set(f.Name, "true"); err != nil { fmt.Fprintf(flags.Output(), "go: invalid boolean flag %s (from %s): %v\n", name, where, err) flags.Usage() } @@ -143,7 +146,7 @@ func SetFromGOFLAGS(flags flag.FlagSet) { fmt.Fprintf(flags.Output(), "go: flag needs an argument: %s (from %s)\n", name, where) flags.Usage() } - if err := f.Value.Set(value); err != nil { + if err := flags.Set(f.Name, value); err != nil { fmt.Fprintf(flags.Output(), "go: invalid value %q for flag %s (from %s): %v\n", value, name, where, err) flags.Usage() } diff --git a/cmd/go/_internal_/base/signal_unix.go b/cmd/go/_internal_/base/signal_unix.go index c109eec..342775a 100644 --- a/cmd/go/_internal_/base/signal_unix.go +++ b/cmd/go/_internal_/base/signal_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix darwin dragonfly freebsd js linux nacl netbsd openbsd solaris +// +build aix darwin dragonfly freebsd js linux netbsd openbsd solaris package base diff --git a/cmd/go/_internal_/cache/cache.go b/cmd/go/_internal_/cache/cache.go index d2c2f1f..cadc16f 100644 --- a/cmd/go/_internal_/cache/cache.go +++ b/cmd/go/_internal_/cache/cache.go @@ -33,7 +33,6 @@ type OutputID [HashSize]byte // A Cache is a package cache, backed by a file system directory tree. type Cache struct { dir string - log *os.File now func() time.Time } @@ -63,13 +62,8 @@ func Open(dir string) (*Cache, error) { return nil, err } } - f, err := os.OpenFile(filepath.Join(dir, "log.txt"), os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666) - if err != nil { - return nil, err - } c := &Cache{ dir: dir, - log: f, now: time.Now, } return c, nil @@ -80,7 +74,22 @@ func (c *Cache) fileName(id [HashSize]byte, key string) string { return filepath.Join(c.dir, fmt.Sprintf("%02x", id[0]), fmt.Sprintf("%x", id)+"-"+key) } -var errMissing = errors.New("cache entry not found") +// An entryNotFoundError indicates that a cache entry was not found, with an +// optional underlying reason. +type entryNotFoundError struct { + Err error +} + +func (e *entryNotFoundError) Error() string { + if e.Err == nil { + return "cache entry not found" + } + return fmt.Sprintf("cache entry not found: %v", e.Err) +} + +func (e *entryNotFoundError) Unwrap() error { + return e.Err +} const ( // action entry file is "v1 \n" @@ -99,6 +108,8 @@ const ( // GODEBUG=gocacheverify=1. var verify = false +var errVerifyMode = errors.New("gocacheverify=1") + // DebugTest is set when GODEBUG=gocachetest=1 is in the environment. var DebugTest = false @@ -127,7 +138,7 @@ func initEnv() { // saved file for that output ID is still available. func (c *Cache) Get(id ActionID) (Entry, error) { if verify { - return Entry{}, errMissing + return Entry{}, &entryNotFoundError{Err: errVerifyMode} } return c.get(id) } @@ -140,52 +151,62 @@ type Entry struct { // get is Get but does not respect verify mode, so that Put can use it. func (c *Cache) get(id ActionID) (Entry, error) { - missing := func() (Entry, error) { - fmt.Fprintf(c.log, "%d miss %x\n", c.now().Unix(), id) - return Entry{}, errMissing + missing := func(reason error) (Entry, error) { + return Entry{}, &entryNotFoundError{Err: reason} } f, err := os.Open(c.fileName(id, "a")) if err != nil { - return missing() + return missing(err) } defer f.Close() entry := make([]byte, entrySize+1) // +1 to detect whether f is too long - if n, err := io.ReadFull(f, entry); n != entrySize || err != io.ErrUnexpectedEOF { - return missing() + if n, err := io.ReadFull(f, entry); n > entrySize { + return missing(errors.New("too long")) + } else if err != io.ErrUnexpectedEOF { + if err == io.EOF { + return missing(errors.New("file is empty")) + } + return missing(err) + } else if n < entrySize { + return missing(errors.New("entry file incomplete")) } if entry[0] != 'v' || entry[1] != '1' || entry[2] != ' ' || entry[3+hexSize] != ' ' || entry[3+hexSize+1+hexSize] != ' ' || entry[3+hexSize+1+hexSize+1+20] != ' ' || entry[entrySize-1] != '\n' { - return missing() + return missing(errors.New("invalid header")) } eid, entry := entry[3:3+hexSize], entry[3+hexSize:] eout, entry := entry[1:1+hexSize], entry[1+hexSize:] esize, entry := entry[1:1+20], entry[1+20:] etime, entry := entry[1:1+20], entry[1+20:] var buf [HashSize]byte - if _, err := hex.Decode(buf[:], eid); err != nil || buf != id { - return missing() + if _, err := hex.Decode(buf[:], eid); err != nil { + return missing(fmt.Errorf("decoding ID: %v", err)) + } else if buf != id { + return missing(errors.New("mismatched ID")) } if _, err := hex.Decode(buf[:], eout); err != nil { - return missing() + return missing(fmt.Errorf("decoding output ID: %v", err)) } i := 0 for i < len(esize) && esize[i] == ' ' { i++ } size, err := strconv.ParseInt(string(esize[i:]), 10, 64) - if err != nil || size < 0 { - return missing() + if err != nil { + return missing(fmt.Errorf("parsing size: %v", err)) + } else if size < 0 { + return missing(errors.New("negative size")) } i = 0 for i < len(etime) && etime[i] == ' ' { i++ } tm, err := strconv.ParseInt(string(etime[i:]), 10, 64) - if err != nil || size < 0 { - return missing() + if err != nil { + return missing(fmt.Errorf("parsing timestamp: %v", err)) + } else if tm < 0 { + return missing(errors.New("negative timestamp")) } - fmt.Fprintf(c.log, "%d get %x\n", c.now().Unix(), id) - c.used(c.fileName(id, "a")) return Entry{buf, size, time.Unix(0, tm)}, nil @@ -200,8 +221,11 @@ func (c *Cache) GetFile(id ActionID) (file string, entry Entry, err error) { } file = c.OutputFile(entry.OutputID) info, err := os.Stat(file) - if err != nil || info.Size() != entry.Size { - return "", Entry{}, errMissing + if err != nil { + return "", Entry{}, &entryNotFoundError{Err: err} + } + if info.Size() != entry.Size { + return "", Entry{}, &entryNotFoundError{Err: errors.New("file incomplete")} } return file, entry, nil } @@ -216,7 +240,7 @@ func (c *Cache) GetBytes(id ActionID) ([]byte, Entry, error) { } data, _ := ioutil.ReadFile(c.OutputFile(entry.OutputID)) if sha256.Sum256(data) != entry.OutputID { - return nil, entry, errMissing + return nil, entry, &entryNotFoundError{Err: errors.New("bad checksum")} } return data, entry, nil } @@ -270,7 +294,7 @@ func (c *Cache) Trim() { // We maintain in dir/trim.txt the time of the last completed cache trim. // If the cache has been trimmed recently enough, do nothing. // This is the common case. - data, _ := ioutil.ReadFile(filepath.Join(c.dir, "trim.txt")) + data, _ := renameio.ReadFile(filepath.Join(c.dir, "trim.txt")) t, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64) if err == nil && now.Sub(time.Unix(t, 0)) < trimInterval { return @@ -287,7 +311,7 @@ func (c *Cache) Trim() { // Ignore errors from here: if we don't write the complete timestamp, the // cache will appear older than it is, and we'll trim it again next time. - renameio.WriteFile(filepath.Join(c.dir, "trim.txt"), []byte(fmt.Sprintf("%d", now.Unix()))) + renameio.WriteFile(filepath.Join(c.dir, "trim.txt"), []byte(fmt.Sprintf("%d", now.Unix())), 0666) } // trimSubdir trims a single cache subdirectory. @@ -331,7 +355,7 @@ func (c *Cache) putIndexEntry(id ActionID, out OutputID, size int64, allowVerify // in verify mode we are double-checking that the cache entries // are entirely reproducible. As just noted, this may be unrealistic // in some cases but the check is also useful for shaking out real bugs. - entry := []byte(fmt.Sprintf("v1 %x %x %20d %20d\n", id, out, size, time.Now().UnixNano())) + entry := fmt.Sprintf("v1 %x %x %20d %20d\n", id, out, size, time.Now().UnixNano()) if verify && allowVerify { old, err := c.get(id) if err == nil && (old.OutputID != out || old.Size != size) { @@ -341,7 +365,28 @@ func (c *Cache) putIndexEntry(id ActionID, out OutputID, size int64, allowVerify } } file := c.fileName(id, "a") - if err := ioutil.WriteFile(file, entry, 0666); err != nil { + + // Copy file to cache directory. + mode := os.O_WRONLY | os.O_CREATE + f, err := os.OpenFile(file, mode, 0666) + if err != nil { + return err + } + _, err = f.WriteString(entry) + if err == nil { + // Truncate the file only *after* writing it. + // (This should be a no-op, but truncate just in case of previous corruption.) + // + // This differs from ioutil.WriteFile, which truncates to 0 *before* writing + // via os.O_TRUNC. Truncating only after writing ensures that a second write + // of the same content to the same file is idempotent, and does not — even + // temporarily! — undo the effect of the first write. + err = f.Truncate(int64(len(entry))) + } + if closeErr := f.Close(); err == nil { + err = closeErr + } + if err != nil { // TODO(bcmills): This Remove potentially races with another go command writing to file. // Can we eliminate it? os.Remove(file) @@ -349,7 +394,6 @@ func (c *Cache) putIndexEntry(id ActionID, out OutputID, size int64, allowVerify } os.Chtimes(file, c.now(), c.now()) // mainly for tests - fmt.Fprintf(c.log, "%d put %x %x %d\n", c.now().Unix(), id, out, size) return nil } diff --git a/cmd/go/_internal_/cache/default.go b/cmd/go/_internal_/cache/default.go index f3de6f9..13f8b98 100644 --- a/cmd/go/_internal_/cache/default.go +++ b/cmd/go/_internal_/cache/default.go @@ -12,6 +12,7 @@ import ( "sync" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" ) // Default returns the default cache to use, or nil if no cache should be used. @@ -73,7 +74,7 @@ func DefaultDir() string { // otherwise distinguish between an explicit "off" and a UserCacheDir error. defaultDirOnce.Do(func() { - defaultDir = os.Getenv("GOCACHE") + defaultDir = cfg.Getenv("GOCACHE") if filepath.IsAbs(defaultDir) || defaultDir == "off" { return } diff --git a/cmd/go/_internal_/cfg/cfg.go b/cmd/go/_internal_/cfg/cfg.go index 0ab69cc..f40bef3 100644 --- a/cmd/go/_internal_/cfg/cfg.go +++ b/cmd/go/_internal_/cfg/cfg.go @@ -7,11 +7,16 @@ package cfg import ( + "bytes" "fmt" "go/build" + "github.com/dependabot/gomodules-extracted/_internal_/cfg" + "io/ioutil" "os" "path/filepath" "runtime" + "strings" + "sync" "github.com/dependabot/gomodules-extracted/cmd/_internal_/objabi" ) @@ -22,6 +27,7 @@ var ( BuildBuildmode string // -buildmode flag BuildContext = defaultContext() BuildMod string // -mod flag + BuildModReason string // reason -mod flag is set, if set by default BuildI bool // -i flag BuildLinkshared bool // -linkshared flag BuildMSan bool // -msan flag @@ -34,11 +40,15 @@ var ( BuildToolchainName string BuildToolchainCompiler func() string BuildToolchainLinker func() string + BuildTrimpath bool // -trimpath flag BuildV bool // -v flag BuildWork bool // -work flag BuildX bool // -x flag - CmdName string // "build", "install", "list", etc. + ModCacheRW bool // -modcacherw flag + ModFile string // -modfile flag + + CmdName string // "build", "install", "list", "mod tidy", etc. DebugActiongraph string // -debug-actiongraph flag (undocumented, unstable) ) @@ -46,6 +56,50 @@ var ( func defaultContext() build.Context { ctxt := build.Default ctxt.JoinPath = filepath.Join // back door to say "do not use go command" + + ctxt.GOROOT = findGOROOT() + if runtime.Compiler != "gccgo" { + // Note that we must use runtime.GOOS and runtime.GOARCH here, + // as the tool directory does not move based on environment + // variables. This matches the initialization of ToolDir in + // go/build, except for using ctxt.GOROOT rather than + // runtime.GOROOT. + build.ToolDir = filepath.Join(ctxt.GOROOT, "pkg/tool/"+runtime.GOOS+"_"+runtime.GOARCH) + } + + ctxt.GOPATH = envOr("GOPATH", ctxt.GOPATH) + + // Override defaults computed in go/build with defaults + // from go environment configuration file, if known. + ctxt.GOOS = envOr("GOOS", ctxt.GOOS) + ctxt.GOARCH = envOr("GOARCH", ctxt.GOARCH) + + // The go/build rule for whether cgo is enabled is: + // 1. If $CGO_ENABLED is set, respect it. + // 2. Otherwise, if this is a cross-compile, disable cgo. + // 3. Otherwise, use built-in default for GOOS/GOARCH. + // Recreate that logic here with the new GOOS/GOARCH setting. + if v := Getenv("CGO_ENABLED"); v == "0" || v == "1" { + ctxt.CgoEnabled = v[0] == '1' + } else if ctxt.GOOS != runtime.GOOS || ctxt.GOARCH != runtime.GOARCH { + ctxt.CgoEnabled = false + } else { + // Use built-in default cgo setting for GOOS/GOARCH. + // Note that ctxt.GOOS/GOARCH are derived from the preference list + // (1) environment, (2) go/env file, (3) runtime constants, + // while go/build.Default.GOOS/GOARCH are derived from the preference list + // (1) environment, (2) runtime constants. + // We know ctxt.GOOS/GOARCH == runtime.GOOS/GOARCH; + // no matter how that happened, go/build.Default will make the + // same decision (either the environment variables are set explicitly + // to match the runtime constants, or else they are unset, in which + // case go/build falls back to the runtime constants), so + // go/build.Default.GOOS/GOARCH == runtime.GOOS/GOARCH. + // So ctxt.CgoEnabled (== go/build.Default.CgoEnabled) is correct + // as is and can be left unmodified. + // Nothing to do here. + } + return ctxt } @@ -70,54 +124,167 @@ var CmdEnv []EnvVar // Global build parameters (used during package load) var ( - Goarch = BuildContext.GOARCH - Goos = BuildContext.GOOS - ExeSuffix string - Gopath = filepath.SplitList(BuildContext.GOPATH) + Goarch = BuildContext.GOARCH + Goos = BuildContext.GOOS + + ExeSuffix = exeSuffix() // ModulesEnabled specifies whether the go command is running // in module-aware mode (as opposed to GOPATH mode). // It is equal to modload.Enabled, but not all packages can import modload. ModulesEnabled bool - - // GoModInGOPATH records whether we've found a go.mod in GOPATH/src - // in GO111MODULE=auto mode. In that case, we don't use modules - // but people might expect us to, so 'go get' warns. - GoModInGOPATH string ) -func init() { +func exeSuffix() string { if Goos == "windows" { - ExeSuffix = ".exe" + return ".exe" + } + return "" +} + +var envCache struct { + once sync.Once + m map[string]string +} + +// EnvFile returns the name of the Go environment configuration file. +func EnvFile() (string, error) { + if file := os.Getenv("GOENV"); file != "" { + if file == "off" { + return "", fmt.Errorf("GOENV=off") + } + return file, nil + } + dir, err := os.UserConfigDir() + if err != nil { + return "", err + } + if dir == "" { + return "", fmt.Errorf("missing user-config dir") + } + return filepath.Join(dir, "go/env"), nil +} + +func initEnvCache() { + envCache.m = make(map[string]string) + file, _ := EnvFile() + if file == "" { + return } + data, err := ioutil.ReadFile(file) + if err != nil { + return + } + + for len(data) > 0 { + // Get next line. + line := data + i := bytes.IndexByte(data, '\n') + if i >= 0 { + line, data = line[:i], data[i+1:] + } else { + data = nil + } + + i = bytes.IndexByte(line, '=') + if i < 0 || line[0] < 'A' || 'Z' < line[0] { + // Line is missing = (or empty) or a comment or not a valid env name. Ignore. + // (This should not happen, since the file should be maintained almost + // exclusively by "go env -w", but better to silently ignore than to make + // the go command unusable just because somehow the env file has + // gotten corrupted.) + continue + } + key, val := line[:i], line[i+1:] + envCache.m[string(key)] = string(val) + } +} + +// Getenv gets the value for the configuration key. +// It consults the operating system environment +// and then the go/env file. +// If Getenv is called for a key that cannot be set +// in the go/env file (for example GODEBUG), it panics. +// This ensures that CanGetenv is accurate, so that +// 'go env -w' stays in sync with what Getenv can retrieve. +func Getenv(key string) string { + if !CanGetenv(key) { + switch key { + case "CGO_TEST_ALLOW", "CGO_TEST_DISALLOW", "CGO_test_ALLOW", "CGO_test_DISALLOW": + // used by internal/work/security_test.go; allow + default: + panic("internal error: invalid Getenv " + key) + } + } + val := os.Getenv(key) + if val != "" { + return val + } + envCache.once.Do(initEnvCache) + return envCache.m[key] +} + +// CanGetenv reports whether key is a valid go/env configuration key. +func CanGetenv(key string) bool { + return strings.Contains(cfg.KnownEnv, "\t"+key+"\n") } var ( - GOROOT = findGOROOT() - GOBIN = os.Getenv("GOBIN") + GOROOT = BuildContext.GOROOT + GOBIN = Getenv("GOBIN") GOROOTbin = filepath.Join(GOROOT, "bin") GOROOTpkg = filepath.Join(GOROOT, "pkg") GOROOTsrc = filepath.Join(GOROOT, "src") GOROOT_FINAL = findGOROOT_FINAL() + GOMODCACHE = envOr("GOMODCACHE", gopathDir("pkg/mod")) // Used in envcmd.MkEnv and build ID computations. - GOARM = fmt.Sprint(objabi.GOARM) - GO386 = objabi.GO386 - GOMIPS = objabi.GOMIPS - GOMIPS64 = objabi.GOMIPS64 + GOARM = envOr("GOARM", fmt.Sprint(objabi.GOARM)) + GO386 = envOr("GO386", objabi.GO386) + GOMIPS = envOr("GOMIPS", objabi.GOMIPS) + GOMIPS64 = envOr("GOMIPS64", objabi.GOMIPS64) + GOPPC64 = envOr("GOPPC64", fmt.Sprintf("%s%d", "power", objabi.GOPPC64)) + GOWASM = envOr("GOWASM", fmt.Sprint(objabi.GOWASM)) + + GOPROXY = envOr("GOPROXY", "https://proxy.golang.org,direct") + GOSUMDB = envOr("GOSUMDB", "sum.golang.org") + GOPRIVATE = Getenv("GOPRIVATE") + GONOPROXY = envOr("GONOPROXY", GOPRIVATE) + GONOSUMDB = envOr("GONOSUMDB", GOPRIVATE) + GOINSECURE = Getenv("GOINSECURE") ) -// Update build context to use our computed GOROOT. -func init() { - BuildContext.GOROOT = GOROOT - if runtime.Compiler != "gccgo" { - // Note that we must use runtime.GOOS and runtime.GOARCH here, - // as the tool directory does not move based on environment - // variables. This matches the initialization of ToolDir in - // go/build, except for using GOROOT rather than - // runtime.GOROOT. - build.ToolDir = filepath.Join(GOROOT, "pkg/tool/"+runtime.GOOS+"_"+runtime.GOARCH) +var SumdbDir = gopathDir("pkg/sumdb") + +// GetArchEnv returns the name and setting of the +// GOARCH-specific architecture environment variable. +// If the current architecture has no GOARCH-specific variable, +// GetArchEnv returns empty key and value. +func GetArchEnv() (key, val string) { + switch Goarch { + case "arm": + return "GOARM", GOARM + case "386": + return "GO386", GO386 + case "mips", "mipsle": + return "GOMIPS", GOMIPS + case "mips64", "mips64le": + return "GOMIPS64", GOMIPS64 + case "ppc64", "ppc64le": + return "GOPPC64", GOPPC64 + case "wasm": + return "GOWASM", GOWASM + } + return "", "" +} + +// envOr returns Getenv(key) if set, or else def. +func envOr(key, def string) string { + val := Getenv(key) + if val == "" { + val = def } + return val } // There is a copy of findGOROOT, isSameDir, and isGOROOT in @@ -131,7 +298,7 @@ func init() { // // There is a copy of this code in x/tools/cmd/godoc/goroot.go. func findGOROOT() string { - if env := os.Getenv("GOROOT"); env != "" { + if env := Getenv("GOROOT"); env != "" { return filepath.Clean(env) } def := filepath.Clean(runtime.GOROOT()) @@ -167,6 +334,8 @@ func findGOROOT() string { } func findGOROOT_FINAL() string { + // $GOROOT_FINAL is only for use during make.bash + // so it is not settable using go/env, so we use os.Getenv here. def := GOROOT if env := os.Getenv("GOROOT_FINAL"); env != "" { def = filepath.Clean(env) @@ -198,3 +367,11 @@ func isGOROOT(path string) bool { } return stat.IsDir() } + +func gopathDir(rel string) string { + list := filepath.SplitList(BuildContext.GOPATH) + if len(list) == 0 || list[0] == "" { + return "" + } + return filepath.Join(list[0], rel) +} diff --git a/cmd/go/_internal_/cfg/zdefaultcc.go b/cmd/go/_internal_/cfg/zdefaultcc.go index 7297f75..5f45a57 100644 --- a/cmd/go/_internal_/cfg/zdefaultcc.go +++ b/cmd/go/_internal_/cfg/zdefaultcc.go @@ -7,10 +7,10 @@ const DefaultPkgConfig = `pkg-config` func DefaultCC(goos, goarch string) string { switch goos + `/` + goarch { } - return "clang" + return "gcc" } func DefaultCXX(goos, goarch string) string { switch goos + `/` + goarch { } - return "clang++" + return "g++" } diff --git a/cmd/go/_internal_/cfg/zosarch.go b/cmd/go/_internal_/cfg/zosarch.go index 7eb0d9a..9dbd520 100644 --- a/cmd/go/_internal_/cfg/zosarch.go +++ b/cmd/go/_internal_/cfg/zosarch.go @@ -3,19 +3,19 @@ package cfg var OSArchSupportsCgo = map[string]bool{ - "aix/ppc64": false, + "aix/ppc64": true, "android/386": true, "android/amd64": true, "android/arm": true, "android/arm64": true, - "darwin/386": true, "darwin/amd64": true, - "darwin/arm": true, "darwin/arm64": true, "dragonfly/amd64": true, "freebsd/386": true, "freebsd/amd64": true, - "freebsd/arm": false, + "freebsd/arm": true, + "freebsd/arm64": true, + "illumos/amd64": true, "js/wasm": false, "linux/386": true, "linux/amd64": true, @@ -27,18 +27,17 @@ var OSArchSupportsCgo = map[string]bool{ "linux/mipsle": true, "linux/ppc64": false, "linux/ppc64le": true, - "linux/riscv64": true, + "linux/riscv64": false, "linux/s390x": true, "linux/sparc64": true, - "nacl/386": false, - "nacl/amd64p32": false, - "nacl/arm": false, "netbsd/386": true, "netbsd/amd64": true, "netbsd/arm": true, + "netbsd/arm64": true, "openbsd/386": true, "openbsd/amd64": true, "openbsd/arm": true, + "openbsd/arm64": true, "plan9/386": false, "plan9/amd64": false, "plan9/arm": false, diff --git a/cmd/go/_internal_/dirhash/hash.go b/cmd/go/_internal_/dirhash/hash.go deleted file mode 100644 index 2ab81b4..0000000 --- a/cmd/go/_internal_/dirhash/hash.go +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package dirhash defines hashes over directory trees. -package dirhash - -import ( - "archive/zip" - "crypto/sha256" - "encoding/base64" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "sort" - "strings" -) - -var DefaultHash = Hash1 - -type Hash func(files []string, open func(string) (io.ReadCloser, error)) (string, error) - -func Hash1(files []string, open func(string) (io.ReadCloser, error)) (string, error) { - h := sha256.New() - files = append([]string(nil), files...) - sort.Strings(files) - for _, file := range files { - if strings.Contains(file, "\n") { - return "", errors.New("filenames with newlines are not supported") - } - r, err := open(file) - if err != nil { - return "", err - } - hf := sha256.New() - _, err = io.Copy(hf, r) - r.Close() - if err != nil { - return "", err - } - fmt.Fprintf(h, "%x %s\n", hf.Sum(nil), file) - } - return "h1:" + base64.StdEncoding.EncodeToString(h.Sum(nil)), nil -} - -func HashDir(dir, prefix string, hash Hash) (string, error) { - files, err := DirFiles(dir, prefix) - if err != nil { - return "", err - } - osOpen := func(name string) (io.ReadCloser, error) { - return os.Open(filepath.Join(dir, strings.TrimPrefix(name, prefix))) - } - return hash(files, osOpen) -} - -func DirFiles(dir, prefix string) ([]string, error) { - var files []string - dir = filepath.Clean(dir) - err := filepath.Walk(dir, func(file string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - rel := file - if dir != "." { - rel = file[len(dir)+1:] - } - f := filepath.Join(prefix, rel) - files = append(files, filepath.ToSlash(f)) - return nil - }) - if err != nil { - return nil, err - } - return files, nil -} - -func HashZip(zipfile string, hash Hash) (string, error) { - z, err := zip.OpenReader(zipfile) - if err != nil { - return "", err - } - defer z.Close() - var files []string - zfiles := make(map[string]*zip.File) - for _, file := range z.File { - files = append(files, file.Name) - zfiles[file.Name] = file - } - zipOpen := func(name string) (io.ReadCloser, error) { - f := zfiles[name] - if f == nil { - return nil, fmt.Errorf("file %q not found in zip", name) // should never happen - } - return f.Open() - } - return hash(files, zipOpen) -} diff --git a/cmd/go/_internal_/get/discovery.go b/cmd/go/_internal_/get/discovery.go index c9d3f57..1b9c1a8 100644 --- a/cmd/go/_internal_/get/discovery.go +++ b/cmd/go/_internal_/get/discovery.go @@ -11,15 +11,15 @@ import ( "strings" ) -// charsetReader returns a reader for the given charset. Currently -// it only supports UTF-8 and ASCII. Otherwise, it returns a meaningful +// charsetReader returns a reader that converts from the given charset to UTF-8. +// Currently it only supports UTF-8 and ASCII. Otherwise, it returns a meaningful // error which is printed by go get, so the user can find why the package // wasn't downloaded if the encoding is not supported. Note that, in // order to reduce potential errors, ASCII is treated as UTF-8 (i.e. characters // greater than 0x7f are not rejected). func charsetReader(charset string, input io.Reader) (io.Reader, error) { switch strings.ToLower(charset) { - case "ascii": + case "utf-8", "ascii": return input, nil default: return nil, fmt.Errorf("can't decode XML document using charset %q", charset) @@ -28,16 +28,16 @@ func charsetReader(charset string, input io.Reader) (io.Reader, error) { // parseMetaGoImports returns meta imports from the HTML in r. // Parsing ends at the end of the section or the beginning of the . -func parseMetaGoImports(r io.Reader, mod ModuleMode) (imports []metaImport, err error) { +func parseMetaGoImports(r io.Reader, mod ModuleMode) ([]metaImport, error) { d := xml.NewDecoder(r) d.CharsetReader = charsetReader d.Strict = false - var t xml.Token + var imports []metaImport for { - t, err = d.RawToken() + t, err := d.RawToken() if err != nil { - if err == io.EOF || len(imports) > 0 { - err = nil + if err != io.EOF && len(imports) == 0 { + return nil, err } break } diff --git a/cmd/go/_internal_/get/get.go b/cmd/go/_internal_/get/get.go index 2dcbd02..0ed26cc 100644 --- a/cmd/go/_internal_/get/get.go +++ b/cmd/go/_internal_/get/get.go @@ -7,7 +7,6 @@ package get import ( "fmt" - "go/build" "os" "path/filepath" "runtime" @@ -108,7 +107,7 @@ var ( ) func init() { - work.AddBuildFlags(CmdGet) + work.AddBuildFlags(CmdGet, work.OmitModFlag|work.OmitModCommonFlags) CmdGet.Run = runGet // break init loop CmdGet.Flag.BoolVar(&Insecure, "insecure", Insecure, "") } @@ -118,11 +117,6 @@ func runGet(cmd *base.Command, args []string) { // Should not happen: main.go should install the separate module-enabled get code. base.Fatalf("go get: modules not implemented") } - if cfg.GoModInGOPATH != "" { - // Warn about not using modules with GO111MODULE=auto when go.mod exists. - // To silence the warning, users can set GO111MODULE=off. - fmt.Fprintf(os.Stderr, "go get: warning: modules disabled by GO111MODULE=auto in GOPATH/src;\n\tignoring %s;\n\tsee 'go help modules'\n", base.ShortPath(cfg.GoModInGOPATH)) - } work.BuildInit() @@ -177,12 +171,6 @@ func runGet(cmd *base.Command, args []string) { // everything. load.ClearPackageCache() - // In order to rebuild packages information completely, - // we need to clear commands cache. Command packages are - // referring to evicted packages from the package cache. - // This leads to duplicated loads of the standard packages. - load.ClearCmdCache() - pkgs := load.PackagesForBuild(args) // Phase 3. Install. @@ -205,12 +193,28 @@ func downloadPaths(patterns []string) []string { for _, arg := range patterns { if strings.Contains(arg, "@") { base.Fatalf("go: cannot use path@version syntax in GOPATH mode") + continue + } + + // Guard against 'go get x.go', a common mistake. + // Note that package and module paths may end with '.go', so only print an error + // if the argument has no slash or refers to an existing file. + if strings.HasSuffix(arg, ".go") { + if !strings.Contains(arg, "/") { + base.Errorf("go get %s: arguments must be package or module paths", arg) + continue + } + if fi, err := os.Stat(arg); err == nil && !fi.IsDir() { + base.Errorf("go get: %s exists as a file, but 'go get' requires package arguments", arg) + } } } + base.ExitIfErrors() + var pkgs []string for _, m := range search.ImportPathsQuiet(patterns) { - if len(m.Pkgs) == 0 && strings.Contains(m.Pattern, "...") { - pkgs = append(pkgs, m.Pattern) + if len(m.Pkgs) == 0 && strings.Contains(m.Pattern(), "...") { + pkgs = append(pkgs, m.Pattern()) } else { pkgs = append(pkgs, m.Pkgs...) } @@ -240,7 +244,8 @@ func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) } load1 := func(path string, mode int) *load.Package { if parent == nil { - return load.LoadPackageNoFlags(path, stk) + mode := 0 // don't do module or vendor resolution + return load.LoadImport(path, base.Cwd, nil, stk, nil, mode) } return load.LoadImport(path, parent.Dir, parent, stk, nil, mode|load.ResolveModule) } @@ -284,7 +289,7 @@ func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) stk.Push(arg) err := downloadPackage(p) if err != nil { - base.Errorf("%s", &load.PackageError{ImportStack: stk.Copy(), Err: err.Error()}) + base.Errorf("%s", &load.PackageError{ImportStack: stk.Copy(), Err: err}) stk.Pop() return } @@ -295,10 +300,16 @@ func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) // We delay this until after reloadPackage so that the old entry // for p has been replaced in the package cache. if wildcardOkay && strings.Contains(arg, "...") { - if build.IsLocalImport(arg) { - args = search.MatchPackagesInFS(arg).Pkgs + match := search.NewMatch(arg) + if match.IsLocal() { + match.MatchDirs() + args = match.Dirs } else { - args = search.MatchPackages(arg).Pkgs + match.MatchPackages() + args = match.Pkgs + } + for _, err := range match.Errs { + base.Errorf("%s", err) } isWildcard = true } @@ -365,7 +376,7 @@ func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) stk.Push(path) err := &load.PackageError{ ImportStack: stk.Copy(), - Err: "must be imported as " + path[j+len("vendor/"):], + Err: load.ImportErrorf(path, "%s must be imported as %s", path, path[j+len("vendor/"):]), } stk.Pop() base.Errorf("%s", err) @@ -397,7 +408,7 @@ func downloadPackage(p *load.Package) error { blindRepo bool // set if the repo has unusual configuration ) - security := web.Secure + security := web.SecureOnly if Insecure { security = web.Insecure } diff --git a/cmd/go/_internal_/get/path.go b/cmd/go/_internal_/get/path.go index d443bd2..ce2e0cd 100644 --- a/cmd/go/_internal_/get/path.go +++ b/cmd/go/_internal_/get/path.go @@ -11,7 +11,7 @@ import ( "unicode/utf8" ) -// The following functions are copied verbatim from cmd/go/internal/module/module.go, +// The following functions are copied verbatim from golang.org/x/mod/module/module.go, // with a change to additionally reject Windows short-names, // and one to accept arbitrary letters (golang.org/issue/29101). // @@ -41,8 +41,8 @@ func checkPath(path string, fileName bool) error { if path == "" { return fmt.Errorf("empty string") } - if strings.Contains(path, "..") { - return fmt.Errorf("double dot") + if path[0] == '-' { + return fmt.Errorf("leading dash") } if strings.Contains(path, "//") { return fmt.Errorf("double slash") diff --git a/cmd/go/_internal_/get/vcs.go b/cmd/go/_internal_/get/vcs.go index 5bf1df1..a204097 100644 --- a/cmd/go/_internal_/get/vcs.go +++ b/cmd/go/_internal_/get/vcs.go @@ -8,9 +8,10 @@ import ( "encoding/json" "errors" "fmt" + "github.com/dependabot/gomodules-extracted/_internal_/lazyregexp" "github.com/dependabot/gomodules-extracted/_internal_/singleflight" "log" - "net/url" + urlpkg "net/url" "os" "os/exec" "path/filepath" @@ -20,6 +21,7 @@ import ( "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/load" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/web" ) @@ -53,7 +55,7 @@ var defaultSecureScheme = map[string]bool{ } func (v *vcsCmd) isSecure(repo string) bool { - u, err := url.Parse(repo) + u, err := urlpkg.Parse(repo) if err != nil { // If repo is not a URL, it's not secure. return false @@ -111,7 +113,7 @@ var vcsHg = &vcsCmd{ name: "Mercurial", cmd: "hg", - createCmd: []string{"clone -U {repo} {dir}"}, + createCmd: []string{"clone -U -- {repo} {dir}"}, downloadCmd: []string{"pull"}, // We allow both tag and branch names as 'tags' @@ -127,7 +129,7 @@ var vcsHg = &vcsCmd{ tagSyncDefault: []string{"update default"}, scheme: []string{"https", "http", "ssh"}, - pingCmd: "identify {scheme}://{repo}", + pingCmd: "identify -- {scheme}://{repo}", remoteRepo: hgRemoteRepo, } @@ -144,7 +146,7 @@ var vcsGit = &vcsCmd{ name: "Git", cmd: "git", - createCmd: []string{"clone {repo} {dir}", "-go-internal-cd {dir} submodule update --init --recursive"}, + createCmd: []string{"clone -- {repo} {dir}", "-go-internal-cd {dir} submodule update --init --recursive"}, downloadCmd: []string{"pull --ff-only", "submodule update --init --recursive"}, tagCmd: []tagCmd{ @@ -163,14 +165,20 @@ var vcsGit = &vcsCmd{ // See golang.org/issue/9032. tagSyncDefault: []string{"submodule update --init --recursive"}, - scheme: []string{"git", "https", "http", "git+ssh", "ssh"}, + scheme: []string{"git", "https", "http", "git+ssh", "ssh"}, + + // Leave out the '--' separator in the ls-remote command: git 2.7.4 does not + // support such a separator for that command, and this use should be safe + // without it because the {scheme} value comes from the predefined list above. + // See golang.org/issue/33836. pingCmd: "ls-remote {scheme}://{repo}", + remoteRepo: gitRemoteRepo, } // scpSyntaxRe matches the SCP-like addresses used by Git to access // repositories by SSH. -var scpSyntaxRe = regexp.MustCompile(`^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.*)$`) +var scpSyntaxRe = lazyregexp.New(`^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.*)$`) func gitRemoteRepo(vcsGit *vcsCmd, rootDir string) (remoteRepo string, err error) { cmd := "config remote.origin.url" @@ -187,19 +195,19 @@ func gitRemoteRepo(vcsGit *vcsCmd, rootDir string) (remoteRepo string, err error } out := strings.TrimSpace(string(outb)) - var repoURL *url.URL + var repoURL *urlpkg.URL if m := scpSyntaxRe.FindStringSubmatch(out); m != nil { // Match SCP-like syntax and convert it to a URL. // Eg, "git@github.com:user/repo" becomes // "ssh://git@github.com/user/repo". - repoURL = &url.URL{ + repoURL = &urlpkg.URL{ Scheme: "ssh", - User: url.User(m[1]), + User: urlpkg.User(m[1]), Host: m[2], Path: m[3], } } else { - repoURL, err = url.Parse(out) + repoURL, err = urlpkg.Parse(out) if err != nil { return "", err } @@ -221,7 +229,7 @@ var vcsBzr = &vcsCmd{ name: "Bazaar", cmd: "bzr", - createCmd: []string{"branch {repo} {dir}"}, + createCmd: []string{"branch -- {repo} {dir}"}, // Without --overwrite bzr will not pull tags that changed. // Replace by --overwrite-tags after http://pad.lv/681792 goes in. @@ -232,7 +240,7 @@ var vcsBzr = &vcsCmd{ tagSyncDefault: []string{"update -r revno:-1"}, scheme: []string{"https", "http", "bzr", "bzr+ssh"}, - pingCmd: "info {scheme}://{repo}", + pingCmd: "info -- {scheme}://{repo}", remoteRepo: bzrRemoteRepo, resolveRepo: bzrResolveRepo, } @@ -283,14 +291,14 @@ var vcsSvn = &vcsCmd{ name: "Subversion", cmd: "svn", - createCmd: []string{"checkout {repo} {dir}"}, + createCmd: []string{"checkout -- {repo} {dir}"}, downloadCmd: []string{"update"}, // There is no tag command in subversion. // The branch information is all in the path names. scheme: []string{"https", "http", "svn", "svn+ssh"}, - pingCmd: "info {scheme}://{repo}", + pingCmd: "info -- {scheme}://{repo}", remoteRepo: svnRemoteRepo, } @@ -333,7 +341,7 @@ var vcsFossil = &vcsCmd{ name: "Fossil", cmd: "fossil", - createCmd: []string{"-go-internal-mkdir {dir} clone {repo} " + filepath.Join("{dir}", fossilRepoName), "-go-internal-cd {dir} open .fossil"}, + createCmd: []string{"-go-internal-mkdir {dir} clone -- {repo} " + filepath.Join("{dir}", fossilRepoName), "-go-internal-cd {dir} open .fossil"}, downloadCmd: []string{"up"}, tagCmd: []tagCmd{{"tag ls", `(.*)`}}, @@ -422,10 +430,10 @@ func (v *vcsCmd) run1(dir string, cmdline string, keyval []string, verbose bool) cmd := exec.Command(v.cmd, args...) cmd.Dir = dir - cmd.Env = base.EnvForDir(cmd.Dir, os.Environ()) + cmd.Env = base.AppendPWD(os.Environ(), cmd.Dir) if cfg.BuildX { - fmt.Printf("cd %s\n", dir) - fmt.Printf("%s %s\n", v.cmd, strings.Join(args, " ")) + fmt.Fprintf(os.Stderr, "cd %s\n", dir) + fmt.Fprintf(os.Stderr, "%s %s\n", v.cmd, strings.Join(args, " ")) } out, err := cmd.Output() if err != nil { @@ -524,14 +532,12 @@ func (v *vcsCmd) tagSync(dir, tag string) error { // A vcsPath describes how to convert an import path into a // version control system and repository name. type vcsPath struct { - prefix string // prefix this description applies to - re string // pattern for import path - repo string // repository to use (expand with match of re) - vcs string // version control system to use (expand with match of re) - check func(match map[string]string) error // additional checks - ping bool // ping for scheme to use to download repo - - regexp *regexp.Regexp // cached compiled form of re + prefix string // prefix this description applies to + regexp *lazyregexp.Regexp // compiled pattern for import path + repo string // repository to use (expand with match of re) + vcs string // version control system to use (expand with match of re) + check func(match map[string]string) error // additional checks + schemelessRepo bool // if true, the repo pattern lacks a scheme } // vcsFromDir inspects dir and its parents to determine the @@ -632,7 +638,14 @@ type RepoRoot struct { vcs *vcsCmd // internal: vcs command access } -var httpPrefixRE = regexp.MustCompile(`^https?:`) +func httpPrefix(s string) string { + for _, prefix := range [...]string{"http:", "https:"} { + if strings.HasPrefix(s, prefix) { + return prefix + } + } + return "" +} // ModuleMode specifies whether to prefer modules when looking up code sources. type ModuleMode int @@ -645,15 +658,15 @@ const ( // RepoRootForImportPath analyzes importPath to determine the // version control system, and code repository to use. func RepoRootForImportPath(importPath string, mod ModuleMode, security web.SecurityMode) (*RepoRoot, error) { - rr, err := repoRootFromVCSPaths(importPath, "", security, vcsPaths) + rr, err := repoRootFromVCSPaths(importPath, security, vcsPaths) if err == errUnknownSite { rr, err = repoRootForImportDynamic(importPath, mod, security) if err != nil { - err = fmt.Errorf("unrecognized import path %q (%v)", importPath, err) + err = load.ImportErrorf(importPath, "unrecognized import path %q: %v", importPath, err) } } if err != nil { - rr1, err1 := repoRootFromVCSPaths(importPath, "", security, vcsPathsAfterDynamic) + rr1, err1 := repoRootFromVCSPaths(importPath, security, vcsPathsAfterDynamic) if err1 == nil { rr = rr1 err = nil @@ -664,7 +677,7 @@ func RepoRootForImportPath(importPath string, mod ModuleMode, security web.Secur if err == nil && strings.Contains(importPath, "...") && strings.Contains(rr.Root, "...") { // Do not allow wildcards in the repo root. rr = nil - err = fmt.Errorf("cannot expand ... in %q", importPath) + err = load.ImportErrorf(importPath, "cannot expand ... in %q", importPath) } return rr, err } @@ -673,14 +686,13 @@ var errUnknownSite = errors.New("dynamic lookup required to find mapping") // repoRootFromVCSPaths attempts to map importPath to a repoRoot // using the mappings defined in vcsPaths. -// If scheme is non-empty, that scheme is forced. -func repoRootFromVCSPaths(importPath, scheme string, security web.SecurityMode, vcsPaths []*vcsPath) (*RepoRoot, error) { +func repoRootFromVCSPaths(importPath string, security web.SecurityMode, vcsPaths []*vcsPath) (*RepoRoot, error) { // A common error is to use https://packagepath because that's what // hg and git require. Diagnose this helpfully. - if loc := httpPrefixRE.FindStringIndex(importPath); loc != nil { + if prefix := httpPrefix(importPath); prefix != "" { // The importPath has been cleaned, so has only one slash. The pattern // ignores the slashes; the error message puts them back on the RHS at least. - return nil, fmt.Errorf("%q not allowed in import path", importPath[loc[0]:loc[1]]+"//") + return nil, fmt.Errorf("%q not allowed in import path", prefix+"//") } for _, srv := range vcsPaths { if !strings.HasPrefix(importPath, srv.prefix) { @@ -689,7 +701,7 @@ func repoRootFromVCSPaths(importPath, scheme string, security web.SecurityMode, m := srv.regexp.FindStringSubmatch(importPath) if m == nil { if srv.prefix != "" { - return nil, fmt.Errorf("invalid %s import path %q", srv.prefix, importPath) + return nil, load.ImportErrorf(importPath, "invalid %s import path %q", srv.prefix, importPath) } continue } @@ -719,26 +731,28 @@ func repoRootFromVCSPaths(importPath, scheme string, security web.SecurityMode, if vcs == nil { return nil, fmt.Errorf("unknown version control system %q", match["vcs"]) } - if srv.ping { - if scheme != "" { - match["repo"] = scheme + "://" + match["repo"] - } else { - for _, scheme := range vcs.scheme { - if security == web.Secure && !vcs.isSecureScheme(scheme) { + var repoURL string + if !srv.schemelessRepo { + repoURL = match["repo"] + } else { + scheme := vcs.scheme[0] // default to first scheme + repo := match["repo"] + if vcs.pingCmd != "" { + // If we know how to test schemes, scan to find one. + for _, s := range vcs.scheme { + if security == web.SecureOnly && !vcs.isSecureScheme(s) { continue } - if vcs.pingCmd != "" && vcs.ping(scheme, match["repo"]) == nil { - match["repo"] = scheme + "://" + match["repo"] - goto Found + if vcs.ping(s, repo) == nil { + scheme = s + break } } - // No scheme found. Fall back to the first one. - match["repo"] = vcs.scheme[0] + "://" + match["repo"] - Found: } + repoURL = scheme + "://" + repo } rr := &RepoRoot{ - Repo: match["repo"], + Repo: repoURL, Root: match["root"], VCS: vcs.cmd, vcs: vcs, @@ -748,20 +762,35 @@ func repoRootFromVCSPaths(importPath, scheme string, security web.SecurityMode, return nil, errUnknownSite } -// repoRootForImportDynamic finds a *RepoRoot for a custom domain that's not -// statically known by repoRootForImportPathStatic. +// urlForImportPath returns a partially-populated URL for the given Go import path. // -// This handles custom import paths like "name.tld/pkg/foo" or just "name.tld". -func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.SecurityMode) (*RepoRoot, error) { +// The URL leaves the Scheme field blank so that web.Get will try any scheme +// allowed by the selected security mode. +func urlForImportPath(importPath string) (*urlpkg.URL, error) { slash := strings.Index(importPath, "/") if slash < 0 { slash = len(importPath) } - host := importPath[:slash] + host, path := importPath[:slash], importPath[slash:] if !strings.Contains(host, ".") { return nil, errors.New("import path does not begin with hostname") } - urlStr, body, err := web.GetMaybeInsecure(importPath, security) + if len(path) == 0 { + path = "/" + } + return &urlpkg.URL{Host: host, Path: path, RawQuery: "go-get=1"}, nil +} + +// repoRootForImportDynamic finds a *RepoRoot for a custom domain that's not +// statically known by repoRootForImportPathStatic. +// +// This handles custom import paths like "name.tld/pkg/foo" or just "name.tld". +func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.SecurityMode) (*RepoRoot, error) { + url, err := urlForImportPath(importPath) + if err != nil { + return nil, err + } + resp, err := web.Get(security, url) if err != nil { msg := "https fetch: %v" if security == web.Insecure { @@ -769,8 +798,16 @@ func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.Se } return nil, fmt.Errorf(msg, err) } + body := resp.Body defer body.Close() imports, err := parseMetaGoImports(body, mod) + if len(imports) == 0 { + if respErr := resp.Err(); respErr != nil { + // If the server's status was not OK, prefer to report that instead of + // an XML parse error. + return nil, respErr + } + } if err != nil { return nil, fmt.Errorf("parsing %s: %v", importPath, err) } @@ -778,12 +815,12 @@ func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.Se mmi, err := matchGoImport(imports, importPath) if err != nil { if _, ok := err.(ImportMismatchError); !ok { - return nil, fmt.Errorf("parse %s: %v", urlStr, err) + return nil, fmt.Errorf("parse %s: %v", url, err) } - return nil, fmt.Errorf("parse %s: no go-import meta tags (%s)", urlStr, err) + return nil, fmt.Errorf("parse %s: no go-import meta tags (%s)", resp.URL, err) } if cfg.BuildV { - log.Printf("get %q: found meta tag %#v at %s", importPath, mmi, urlStr) + log.Printf("get %q: found meta tag %#v at %s", importPath, mmi, url) } // If the import was "uni.edu/bob/project", which said the // prefix was "uni.edu" and the RepoRoot was "evilroot.com", @@ -795,24 +832,23 @@ func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.Se if cfg.BuildV { log.Printf("get %q: verifying non-authoritative meta tag", importPath) } - urlStr0 := urlStr var imports []metaImport - urlStr, imports, err = metaImportsForPrefix(mmi.Prefix, mod, security) + url, imports, err = metaImportsForPrefix(mmi.Prefix, mod, security) if err != nil { return nil, err } metaImport2, err := matchGoImport(imports, importPath) if err != nil || mmi != metaImport2 { - return nil, fmt.Errorf("%s and %s disagree about go-import for %s", urlStr0, urlStr, mmi.Prefix) + return nil, fmt.Errorf("%s and %s disagree about go-import for %s", resp.URL, url, mmi.Prefix) } } if err := validateRepoRoot(mmi.RepoRoot); err != nil { - return nil, fmt.Errorf("%s: invalid repo root %q: %v", urlStr, mmi.RepoRoot, err) + return nil, fmt.Errorf("%s: invalid repo root %q: %v", resp.URL, mmi.RepoRoot, err) } vcs := vcsByCmd(mmi.VCS) if vcs == nil && mmi.VCS != "mod" { - return nil, fmt.Errorf("%s: unknown vcs %q", urlStr, mmi.VCS) + return nil, fmt.Errorf("%s: unknown vcs %q", resp.URL, mmi.VCS) } rr := &RepoRoot{ @@ -828,13 +864,16 @@ func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.Se // validateRepoRoot returns an error if repoRoot does not seem to be // a valid URL with scheme. func validateRepoRoot(repoRoot string) error { - url, err := url.Parse(repoRoot) + url, err := urlpkg.Parse(repoRoot) if err != nil { return err } if url.Scheme == "" { return errors.New("no scheme") } + if url.Scheme == "file" { + return errors.New("file scheme disallowed") + } return nil } @@ -850,9 +889,9 @@ var ( // // The importPath is of the form "golang.org/x/tools". // It is an error if no imports are found. -// urlStr will still be valid if err != nil. -// The returned urlStr will be of the form "https://golang.org/x/tools?go-get=1" -func metaImportsForPrefix(importPrefix string, mod ModuleMode, security web.SecurityMode) (urlStr string, imports []metaImport, err error) { +// url will still be valid if err != nil. +// The returned url will be of the form "https://golang.org/x/tools?go-get=1" +func metaImportsForPrefix(importPrefix string, mod ModuleMode, security web.SecurityMode) (*urlpkg.URL, []metaImport, error) { setCache := func(res fetchResult) (fetchResult, error) { fetchCacheMu.Lock() defer fetchCacheMu.Unlock() @@ -868,25 +907,38 @@ func metaImportsForPrefix(importPrefix string, mod ModuleMode, security web.Secu } fetchCacheMu.Unlock() - urlStr, body, err := web.GetMaybeInsecure(importPrefix, security) + url, err := urlForImportPath(importPrefix) + if err != nil { + return setCache(fetchResult{err: err}) + } + resp, err := web.Get(security, url) if err != nil { - return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("fetch %s: %v", urlStr, err)}) + return setCache(fetchResult{url: url, err: fmt.Errorf("fetching %s: %v", importPrefix, err)}) } + body := resp.Body + defer body.Close() imports, err := parseMetaGoImports(body, mod) + if len(imports) == 0 { + if respErr := resp.Err(); respErr != nil { + // If the server's status was not OK, prefer to report that instead of + // an XML parse error. + return setCache(fetchResult{url: url, err: respErr}) + } + } if err != nil { - return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("parsing %s: %v", urlStr, err)}) + return setCache(fetchResult{url: url, err: fmt.Errorf("parsing %s: %v", resp.URL, err)}) } if len(imports) == 0 { - err = fmt.Errorf("fetch %s: no go-import meta tag", urlStr) + err = fmt.Errorf("fetching %s: no go-import meta tag found in %s", importPrefix, resp.URL) } - return setCache(fetchResult{urlStr: urlStr, imports: imports, err: err}) + return setCache(fetchResult{url: url, imports: imports, err: err}) }) res := resi.(fetchResult) - return res.urlStr, res.imports, res.err + return res.url, res.imports, res.err } type fetchResult struct { - urlStr string // e.g. "https://foo.com/x/bar?go-get=1" + url *urlpkg.URL imports []metaImport err error } @@ -926,7 +978,7 @@ func (m ImportMismatchError) Error() string { // matchGoImport returns the metaImport from imports matching importPath. // An error is returned if there are multiple matches. -// errNoMatch is returned if none match. +// An ImportMismatchError is returned if none match. func matchGoImport(imports []metaImport, importPath string) (metaImport, error) { match := -1 @@ -975,7 +1027,7 @@ var vcsPaths = []*vcsPath{ // Github { prefix: "github.com/", - re: `^(?Pgithub\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[\p{L}0-9_.\-]+)*$`, + regexp: lazyregexp.New(`^(?Pgithub\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[\p{L}0-9_.\-]+)*$`), vcs: "git", repo: "https://{root}", check: noVCSSuffix, @@ -984,7 +1036,7 @@ var vcsPaths = []*vcsPath{ // Bitbucket { prefix: "bitbucket.org/", - re: `^(?Pbitbucket\.org/(?P[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`, + regexp: lazyregexp.New(`^(?Pbitbucket\.org/(?P[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`), repo: "https://{root}", check: bitbucketVCS, }, @@ -992,7 +1044,7 @@ var vcsPaths = []*vcsPath{ // IBM DevOps Services (JazzHub) { prefix: "hub.jazz.net/git/", - re: `^(?Phub\.jazz\.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`, + regexp: lazyregexp.New(`^(?Phub\.jazz\.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`), vcs: "git", repo: "https://{root}", check: noVCSSuffix, @@ -1001,7 +1053,7 @@ var vcsPaths = []*vcsPath{ // Git at Apache { prefix: "git.apache.org/", - re: `^(?Pgit\.apache\.org/[a-z0-9_.\-]+\.git)(/[A-Za-z0-9_.\-]+)*$`, + regexp: lazyregexp.New(`^(?Pgit\.apache\.org/[a-z0-9_.\-]+\.git)(/[A-Za-z0-9_.\-]+)*$`), vcs: "git", repo: "https://{root}", }, @@ -1009,7 +1061,7 @@ var vcsPaths = []*vcsPath{ // Git at OpenStack { prefix: "git.openstack.org/", - re: `^(?Pgit\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/[A-Za-z0-9_.\-]+)*$`, + regexp: lazyregexp.New(`^(?Pgit\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/[A-Za-z0-9_.\-]+)*$`), vcs: "git", repo: "https://{root}", }, @@ -1017,7 +1069,7 @@ var vcsPaths = []*vcsPath{ // chiselapp.com for fossil { prefix: "chiselapp.com/", - re: `^(?Pchiselapp\.com/user/[A-Za-z0-9]+/repository/[A-Za-z0-9_.\-]+)$`, + regexp: lazyregexp.New(`^(?Pchiselapp\.com/user/[A-Za-z0-9]+/repository/[A-Za-z0-9_.\-]+)$`), vcs: "fossil", repo: "https://{root}", }, @@ -1025,8 +1077,8 @@ var vcsPaths = []*vcsPath{ // General syntax for any server. // Must be last. { - re: `^(?P(?P([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\-]+)+?)\.(?Pbzr|fossil|git|hg|svn))(/~?[A-Za-z0-9_.\-]+)*$`, - ping: true, + regexp: lazyregexp.New(`(?P(?P([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\-]+)+?)\.(?Pbzr|fossil|git|hg|svn))(/~?[A-Za-z0-9_.\-]+)*$`), + schemelessRepo: true, }, } @@ -1038,25 +1090,13 @@ var vcsPathsAfterDynamic = []*vcsPath{ // Launchpad. See golang.org/issue/11436. { prefix: "launchpad.net/", - re: `^(?Plaunchpad\.net/((?P[A-Za-z0-9_.\-]+)(?P/[A-Za-z0-9_.\-]+)?|~[A-Za-z0-9_.\-]+/(\+junk|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`, + regexp: lazyregexp.New(`^(?Plaunchpad\.net/((?P[A-Za-z0-9_.\-]+)(?P/[A-Za-z0-9_.\-]+)?|~[A-Za-z0-9_.\-]+/(\+junk|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`), vcs: "bzr", repo: "https://{root}", check: launchpadVCS, }, } -func init() { - // fill in cached regexps. - // Doing this eagerly discovers invalid regexp syntax - // without having to run a command that needs that regexp. - for _, srv := range vcsPaths { - srv.regexp = regexp.MustCompile(srv.re) - } - for _, srv := range vcsPathsAfterDynamic { - srv.regexp = regexp.MustCompile(srv.re) - } -} - // noVCSSuffix checks that the repository name does not // end in .foo for any version control system foo. // The usual culprit is ".git". @@ -1080,8 +1120,13 @@ func bitbucketVCS(match map[string]string) error { var resp struct { SCM string `json:"scm"` } - url := expand(match, "https://api.bitbucket.org/2.0/repositories/{bitname}?fields=scm") - data, err := web.Get(url) + url := &urlpkg.URL{ + Scheme: "https", + Host: "api.bitbucket.org", + Path: expand(match, "/2.0/repositories/{bitname}"), + RawQuery: "fields=scm", + } + data, err := web.GetBytes(url) if err != nil { if httpErr, ok := err.(*web.HTTPError); ok && httpErr.StatusCode == 403 { // this may be a private repository. If so, attempt to determine which @@ -1123,7 +1168,12 @@ func launchpadVCS(match map[string]string) error { if match["project"] == "" || match["series"] == "" { return nil } - _, err := web.Get(expand(match, "https://code.launchpad.net/{project}{series}/.bzr/branch-format")) + url := &urlpkg.URL{ + Scheme: "https", + Host: "code.launchpad.net", + Path: expand(match, "/{project}{series}/.bzr/branch-format"), + } + _, err := web.GetBytes(url) if err != nil { match["root"] = expand(match, "launchpad.net/{project}") match["repo"] = expand(match, "https://{root}") diff --git a/cmd/go/_internal_/imports/build.go b/cmd/go/_internal_/imports/build.go index 11f50ed..a5b0d88 100644 --- a/cmd/go/_internal_/imports/build.go +++ b/cmd/go/_internal_/imports/build.go @@ -138,6 +138,9 @@ func matchTag(name string, tags map[string]bool, want bool) bool { if name == "linux" { have = have || tags["android"] } + if name == "solaris" { + have = have || tags["illumos"] + } return have == want } @@ -152,7 +155,9 @@ func matchTag(name string, tags map[string]bool, want bool) bool { // name_$(GOARCH)_test.* // name_$(GOOS)_$(GOARCH)_test.* // -// An exception: if GOOS=android, then files with GOOS=linux are also matched. +// Exceptions: +// if GOOS=android, then files with GOOS=linux are also matched. +// if GOOS=illumos, then files with GOOS=solaris are also matched. // // If tags["*"] is true, then MatchFile will consider all possible // GOOS and GOARCH to be available and will consequently @@ -184,28 +189,58 @@ func MatchFile(name string, tags map[string]bool) bool { } n := len(l) if n >= 2 && KnownOS[l[n-2]] && KnownArch[l[n-1]] { - return tags[l[n-2]] && tags[l[n-1]] + return matchTag(l[n-2], tags, true) && matchTag(l[n-1], tags, true) } if n >= 1 && KnownOS[l[n-1]] { - return tags[l[n-1]] + return matchTag(l[n-1], tags, true) } if n >= 1 && KnownArch[l[n-1]] { - return tags[l[n-1]] + return matchTag(l[n-1], tags, true) } return true } -var KnownOS = make(map[string]bool) -var KnownArch = make(map[string]bool) - -func init() { - for _, v := range strings.Fields(goosList) { - KnownOS[v] = true - } - for _, v := range strings.Fields(goarchList) { - KnownArch[v] = true - } +var KnownOS = map[string]bool{ + "aix": true, + "android": true, + "darwin": true, + "dragonfly": true, + "freebsd": true, + "hurd": true, + "illumos": true, + "js": true, + "linux": true, + "nacl": true, // legacy; don't remove + "netbsd": true, + "openbsd": true, + "plan9": true, + "solaris": true, + "windows": true, + "zos": true, } -const goosList = "aix android darwin dragonfly freebsd hurd js linux nacl netbsd openbsd plan9 solaris windows zos " -const goarchList = "386 amd64 amd64p32 arm armbe arm64 arm64be ppc64 ppc64le mips mipsle mips64 mips64le mips64p32 mips64p32le ppc riscv riscv64 s390 s390x sparc sparc64 wasm " +var KnownArch = map[string]bool{ + "386": true, + "amd64": true, + "amd64p32": true, // legacy; don't remove + "arm": true, + "armbe": true, + "arm64": true, + "arm64be": true, + "ppc64": true, + "ppc64le": true, + "mips": true, + "mipsle": true, + "mips64": true, + "mips64le": true, + "mips64p32": true, + "mips64p32le": true, + "ppc": true, + "riscv": true, + "riscv64": true, + "s390": true, + "s390x": true, + "sparc": true, + "sparc64": true, + "wasm": true, +} diff --git a/cmd/go/_internal_/imports/scan.go b/cmd/go/_internal_/imports/scan.go index a52af8e..06c7c1a 100644 --- a/cmd/go/_internal_/imports/scan.go +++ b/cmd/go/_internal_/imports/scan.go @@ -26,7 +26,7 @@ func ScanDir(dir string, tags map[string]bool) ([]string, []string, error) { // If the directory entry is a symlink, stat it to obtain the info for the // link target instead of the link itself. if info.Mode()&os.ModeSymlink != 0 { - info, err = os.Stat(name) + info, err = os.Stat(filepath.Join(dir, name)) if err != nil { continue // Ignore broken symlinks. } diff --git a/cmd/go/_internal_/imports/tags.go b/cmd/go/_internal_/imports/tags.go index 7355266..c6ad8ef 100644 --- a/cmd/go/_internal_/imports/tags.go +++ b/cmd/go/_internal_/imports/tags.go @@ -8,6 +8,9 @@ import "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" var tags map[string]bool +// Tags returns a set of build tags that are true for the target platform. +// It includes GOOS, GOARCH, the compiler, possibly "cgo", +// release tags like "go1.13", and user-specified build tags. func Tags() map[string]bool { if tags == nil { tags = loadTags() @@ -32,3 +35,15 @@ func loadTags() map[string]bool { } return tags } + +var anyTags map[string]bool + +// AnyTags returns a special set of build tags that satisfy nearly all +// build tag expressions. Only "ignore" and malformed build tag requirements +// are considered false. +func AnyTags() map[string]bool { + if anyTags == nil { + anyTags = map[string]bool{"*": true} + } + return anyTags +} diff --git a/cmd/go/_internal_/load/path.go b/cmd/go/_internal_/load/path.go index 0211b28..584cdff 100644 --- a/cmd/go/_internal_/load/path.go +++ b/cmd/go/_internal_/load/path.go @@ -6,32 +6,8 @@ package load import ( "path/filepath" - "strings" ) -// hasSubdir reports whether dir is a subdirectory of -// (possibly multiple levels below) root. -// If so, it sets rel to the path fragment that must be -// appended to root to reach dir. -func hasSubdir(root, dir string) (rel string, ok bool) { - if p, err := filepath.EvalSymlinks(root); err == nil { - root = p - } - if p, err := filepath.EvalSymlinks(dir); err == nil { - dir = p - } - const sep = string(filepath.Separator) - root = filepath.Clean(root) - if !strings.HasSuffix(root, sep) { - root += sep - } - dir = filepath.Clean(dir) - if !strings.HasPrefix(dir, root) { - return "", false - } - return filepath.ToSlash(dir[len(root):]), true -} - // expandPath returns the symlink-expanded form of path. func expandPath(p string) string { x, err := filepath.EvalSymlinks(p) diff --git a/cmd/go/_internal_/load/pkg.go b/cmd/go/_internal_/load/pkg.go index 46493a6..d01ec8c 100644 --- a/cmd/go/_internal_/load/pkg.go +++ b/cmd/go/_internal_/load/pkg.go @@ -7,13 +7,17 @@ package load import ( "bytes" + "encoding/json" + "errors" "fmt" "go/build" + "go/scanner" "go/token" "io/ioutil" "os" pathpkg "path" "path/filepath" + "runtime" "sort" "strconv" "strings" @@ -23,6 +27,7 @@ import ( "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modinfo" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/par" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/search" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/str" ) @@ -32,14 +37,14 @@ var ( ModInit func() // module hooks; nil if module use is disabled - ModBinDir func() string // return effective bin directory - ModLookup func(path string) (dir, realPath string, err error) // lookup effective meaning of import - ModPackageModuleInfo func(path string) *modinfo.ModulePublic // return module info for Package struct - ModImportPaths func(args []string) []*search.Match // expand import paths - ModPackageBuildInfo func(main string, deps []string) string // return module info to embed in binary - ModInfoProg func(info string) []byte // wrap module info in .go code for binary - ModImportFromFiles func([]string) // update go.mod to add modules for imports in these files - ModDirImportPath func(string) string // return effective import path for directory + ModBinDir func() string // return effective bin directory + ModLookup func(parentPath string, parentIsStd bool, path string) (dir, realPath string, err error) // lookup effective meaning of import + ModPackageModuleInfo func(path string) *modinfo.ModulePublic // return module info for Package struct + ModImportPaths func(args []string) []*search.Match // expand import paths + ModPackageBuildInfo func(main string, deps []string) string // return module info to embed in binary + ModInfoProg func(info string, isgccgo bool) []byte // wrap module info in .go code for binary + ModImportFromFiles func([]string) // update go.mod to add modules for imports in these files + ModDirImportPath func(string) string // return effective import path for directory ) var IgnoreImports bool // control whether we ignore imports in packages @@ -61,7 +66,7 @@ type PackagePublic struct { Doc string `json:",omitempty"` // package documentation string Target string `json:",omitempty"` // installed target for this package (may be executable) Shlib string `json:",omitempty"` // the shared library that contains this package (only set when -linkshared) - Root string `json:",omitempty"` // Go root or Go path dir containing this package + Root string `json:",omitempty"` // Go root, Go path dir, or module root dir containing this package ConflictDir string `json:",omitempty"` // Dir is hidden by this other directory ForTest string `json:",omitempty"` // package is only for use in named test Export string `json:",omitempty"` // file containing export data (set by go list -export) @@ -182,20 +187,17 @@ type PackageInternal struct { Gccgoflags []string // -gccgoflags for this package } +// A NoGoError indicates that no Go files for the package were applicable to the +// build for that package. +// +// That may be because there were no files whatsoever, or because all files were +// excluded, or because all non-excluded files were test sources. type NoGoError struct { Package *Package } func (e *NoGoError) Error() string { - // Count files beginning with _ and ., which we will pretend don't exist at all. - dummy := 0 - for _, name := range e.Package.IgnoredGoFiles { - if strings.HasPrefix(name, "_") || strings.HasPrefix(name, ".") { - dummy++ - } - } - - if len(e.Package.IgnoredGoFiles) > dummy { + if len(e.Package.constraintIgnoredGoFiles()) > 0 { // Go files exist, but they were ignored due to build constraints. return "build constraints exclude all Go files in " + e.Package.Dir } @@ -208,6 +210,80 @@ func (e *NoGoError) Error() string { return "no Go files in " + e.Package.Dir } +// setLoadPackageDataError presents an error found when loading package data +// as a *PackageError. It has special cases for some common errors to improve +// messages shown to users and reduce redundancy. +// +// setLoadPackageDataError returns true if it's safe to load information about +// imported packages, for example, if there was a parse error loading imports +// in one file, but other files are okay. +func (p *Package) setLoadPackageDataError(err error, path string, stk *ImportStack, importPos []token.Position) { + matchErr, isMatchErr := err.(*search.MatchError) + if isMatchErr && matchErr.Match.Pattern() == path { + if matchErr.Match.IsLiteral() { + // The error has a pattern has a pattern similar to the import path. + // It may be slightly different (./foo matching example.com/foo), + // but close enough to seem redundant. + // Unwrap the error so we don't show the pattern. + err = matchErr.Err + } + } + + // Replace (possibly wrapped) *build.NoGoError with *load.NoGoError. + // The latter is more specific about the cause. + var nogoErr *build.NoGoError + if errors.As(err, &nogoErr) { + if p.Dir == "" && nogoErr.Dir != "" { + p.Dir = nogoErr.Dir + } + err = &NoGoError{Package: p} + } + + // Take only the first error from a scanner.ErrorList. PackageError only + // has room for one position, so we report the first error with a position + // instead of all of the errors without a position. + var pos string + var isScanErr bool + if scanErr, ok := err.(scanner.ErrorList); ok && len(scanErr) > 0 { + isScanErr = true // For stack push/pop below. + + scanPos := scanErr[0].Pos + scanPos.Filename = base.ShortPath(scanPos.Filename) + pos = scanPos.String() + err = errors.New(scanErr[0].Msg) + } + + // Report the error on the importing package if the problem is with the import declaration + // for example, if the package doesn't exist or if the import path is malformed. + // On the other hand, don't include a position if the problem is with the imported package, + // for example there are no Go files (NoGoError), or there's a problem in the imported + // package's source files themselves (scanner errors). + // + // TODO(matloob): Perhaps make each of those the errors in the first group + // (including modload.ImportMissingError, and the corresponding + // "cannot find package %q in any of" GOPATH-mode error + // produced in build.(*Context).Import; modload.AmbiguousImportError, + // and modload.PackageNotInModuleError; and the malformed module path errors + // produced in golang.org/x/mod/module.CheckMod) implement an interface + // to make it easier to check for them? That would save us from having to + // move the modload errors into this package to avoid a package import cycle, + // and from having to export an error type for the errors produced in build. + if !isMatchErr && (nogoErr != nil || isScanErr) { + stk.Push(path) + defer stk.Pop() + } + + p.Error = &PackageError{ + ImportStack: stk.Copy(), + Pos: pos, + Err: err, + } + + if path != stk.Top() { + p = setErrorPos(p, importPos) + } +} + // Resolve returns the resolved version of imports, // which should be p.TestImports or p.XTestImports, NOT p.Imports. // The imports in p.TestImports and p.XTestImports are not recursively @@ -299,27 +375,90 @@ func (p *Package) copyBuild(pp *build.Package) { // A PackageError describes an error loading information about a package. type PackageError struct { - ImportStack []string // shortest path from package named on command line to this one - Pos string // position of error - Err string // the error itself - IsImportCycle bool `json:"-"` // the error is an import cycle - Hard bool `json:"-"` // whether the error is soft or hard; soft errors are ignored in some places + ImportStack []string // shortest path from package named on command line to this one + Pos string // position of error + Err error // the error itself + IsImportCycle bool // the error is an import cycle + Hard bool // whether the error is soft or hard; soft errors are ignored in some places + alwaysPrintStack bool // whether to always print the ImportStack } func (p *PackageError) Error() string { - // Import cycles deserve special treatment. - if p.IsImportCycle { - return fmt.Sprintf("%s\npackage %s\n", p.Err, strings.Join(p.ImportStack, "\n\timports ")) - } - if p.Pos != "" { + if p.Pos != "" && (len(p.ImportStack) == 0 || !p.alwaysPrintStack) { // Omit import stack. The full path to the file where the error // is the most important thing. - return p.Pos + ": " + p.Err + return p.Pos + ": " + p.Err.Error() } + + // If the error is an ImportPathError, and the last path on the stack appears + // in the error message, omit that path from the stack to avoid repetition. + // If an ImportPathError wraps another ImportPathError that matches the + // last path on the stack, we don't omit the path. An error like + // "package A imports B: error loading C caused by B" would not be clearer + // if "imports B" were omitted. if len(p.ImportStack) == 0 { - return p.Err + return p.Err.Error() + } + var optpos string + if p.Pos != "" { + optpos = "\n\t" + p.Pos + } + return "package " + strings.Join(p.ImportStack, "\n\timports ") + optpos + ": " + p.Err.Error() +} + +func (p *PackageError) Unwrap() error { return p.Err } + +// PackageError implements MarshalJSON so that Err is marshaled as a string +// and non-essential fields are omitted. +func (p *PackageError) MarshalJSON() ([]byte, error) { + perr := struct { + ImportStack []string + Pos string + Err string + }{p.ImportStack, p.Pos, p.Err.Error()} + return json.Marshal(perr) +} + +// ImportPathError is a type of error that prevents a package from being loaded +// for a given import path. When such a package is loaded, a *Package is +// returned with Err wrapping an ImportPathError: the error is attached to +// the imported package, not the importing package. +// +// The string returned by ImportPath must appear in the string returned by +// Error. Errors that wrap ImportPathError (such as PackageError) may omit +// the import path. +type ImportPathError interface { + error + ImportPath() string +} + +type importError struct { + importPath string + err error // created with fmt.Errorf +} + +var _ ImportPathError = (*importError)(nil) + +func ImportErrorf(path, format string, args ...interface{}) ImportPathError { + err := &importError{importPath: path, err: fmt.Errorf(format, args...)} + if errStr := err.Error(); !strings.Contains(errStr, path) { + panic(fmt.Sprintf("path %q not in error %q", path, errStr)) } - return "package " + strings.Join(p.ImportStack, "\n\timports ") + ": " + p.Err + return err +} + +func (e *importError) Error() string { + return e.err.Error() +} + +func (e *importError) Unwrap() error { + // Don't return e.err directly, since we're only wrapping an error if %w + // was passed to ImportErrorf. + return errors.Unwrap(e.err) +} + +func (e *importError) ImportPath() string { + return e.importPath } // An ImportStack is a stack of import paths, possibly with the suffix " (test)" appended. @@ -339,6 +478,13 @@ func (s *ImportStack) Copy() []string { return append([]string{}, *s...) } +func (s *ImportStack) Top() string { + if len(*s) == 0 { + return "" + } + return (*s)[len(*s)-1] +} + // shorterThan reports whether sp is shorter than t. // We use this to record the shortest import sequence // that leads to a particular package. @@ -356,38 +502,56 @@ func (sp *ImportStack) shorterThan(t []string) bool { return false // they are equal } -// packageCache is a lookup cache for loadPackage, +// packageCache is a lookup cache for LoadImport, // so that if we look up a package multiple times // we return the same pointer each time. var packageCache = map[string]*Package{} +// ClearPackageCache clears the in-memory package cache and the preload caches. +// It is only for use by GOPATH-based "go get". +// TODO(jayconrod): When GOPATH-based "go get" is removed, delete this function. func ClearPackageCache() { for name := range packageCache { delete(packageCache, name) } + resolvedImportCache.Clear() + packageDataCache.Clear() } +// ClearPackageCachePartial clears packages with the given import paths from the +// in-memory package cache and the preload caches. It is only for use by +// GOPATH-based "go get". +// TODO(jayconrod): When GOPATH-based "go get" is removed, delete this function. func ClearPackageCachePartial(args []string) { + shouldDelete := make(map[string]bool) for _, arg := range args { - p := packageCache[arg] - if p != nil { - delete(packageCache, p.Dir) - delete(packageCache, p.ImportPath) + shouldDelete[arg] = true + if p := packageCache[arg]; p != nil { + delete(packageCache, arg) } } + resolvedImportCache.DeleteIf(func(key interface{}) bool { + return shouldDelete[key.(importSpec).path] + }) + packageDataCache.DeleteIf(func(key interface{}) bool { + return shouldDelete[key.(string)] + }) } -// ReloadPackageNoFlags is like LoadPackageNoFlags but makes sure +// ReloadPackageNoFlags is like LoadImport but makes sure // not to use the package cache. // It is only for use by GOPATH-based "go get". // TODO(rsc): When GOPATH-based "go get" is removed, delete this function. func ReloadPackageNoFlags(arg string, stk *ImportStack) *Package { p := packageCache[arg] if p != nil { - delete(packageCache, p.Dir) - delete(packageCache, p.ImportPath) + delete(packageCache, arg) + resolvedImportCache.DeleteIf(func(key interface{}) bool { + return key.(importSpec).path == p.ImportPath + }) + packageDataCache.Delete(p.ImportPath) } - return LoadPackageNoFlags(arg, stk) + return LoadImport(arg, base.Cwd, nil, stk, nil, 0) } // dirToImportPath returns the pseudo-import path we use for a package @@ -440,134 +604,66 @@ const ( // this package, as part of a bigger load operation, and by GOPATH-based "go get". // TODO(rsc): When GOPATH-based "go get" is removed, unexport this function. func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { + return loadImport(nil, path, srcDir, parent, stk, importPos, mode) +} + +func loadImport(pre *preload, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { if path == "" { panic("LoadImport called with empty package path") } - stk.Push(path) - defer stk.Pop() - - if strings.HasPrefix(path, "mod/") { - // Paths beginning with "mod/" might accidentally - // look in the module cache directory tree in $GOPATH/pkg/mod/. - // This prefix is owned by the Go core for possible use in the - // standard library (since it does not begin with a domain name), - // so it's OK to disallow entirely. - return &Package{ - PackagePublic: PackagePublic{ - ImportPath: path, - Error: &PackageError{ - ImportStack: stk.Copy(), - Err: fmt.Sprintf("disallowed import path %q", path), - }, - }, - } + var parentPath, parentRoot string + parentIsStd := false + if parent != nil { + parentPath = parent.ImportPath + parentRoot = parent.Root + parentIsStd = parent.Standard } - - if strings.Contains(path, "@") { - var text string - if cfg.ModulesEnabled { - text = "can only use path@version syntax with 'go get'" - } else { - text = "cannot use path@version syntax in GOPATH mode" + bp, loaded, err := loadPackageData(path, parentPath, srcDir, parentRoot, parentIsStd, mode) + if loaded && pre != nil && !IgnoreImports { + pre.preloadImports(bp.Imports, bp) + } + if bp == nil { + if importErr, ok := err.(ImportPathError); !ok || importErr.ImportPath() != path { + // Only add path to the error's import stack if it's not already present on the error. + stk.Push(path) + defer stk.Pop() } return &Package{ PackagePublic: PackagePublic{ ImportPath: path, Error: &PackageError{ ImportStack: stk.Copy(), - Err: text, + Err: err, }, }, } } - parentPath := "" - if parent != nil { - parentPath = parent.ImportPath - } - - // Determine canonical identifier for this package. - // For a local import the identifier is the pseudo-import path - // we create from the full directory to the package. - // Otherwise it is the usual import path. - // For vendored imports, it is the expanded form. - importPath := path - origPath := path - isLocal := build.IsLocalImport(path) - var modDir string - var modErr error - if isLocal { - importPath = dirToImportPath(filepath.Join(srcDir, path)) - } else if cfg.ModulesEnabled { - var p string - modDir, p, modErr = ModLookup(path) - if modErr == nil { - importPath = p - } - } else if mode&ResolveImport != 0 { - // We do our own path resolution, because we want to - // find out the key to use in packageCache without the - // overhead of repeated calls to buildContext.Import. - // The code is also needed in a few other places anyway. - path = ResolveImportPath(parent, path) - importPath = path - } else if mode&ResolveModule != 0 { - path = ModuleImportPath(parent, path) - importPath = path - } - + importPath := bp.ImportPath p := packageCache[importPath] if p != nil { + stk.Push(path) p = reusePackage(p, stk) + stk.Pop() } else { p = new(Package) - p.Internal.Local = isLocal + p.Internal.Local = build.IsLocalImport(path) p.ImportPath = importPath packageCache[importPath] = p // Load package. - // Import always returns bp != nil, even if an error occurs, + // loadPackageData may return bp != nil even if an error occurs, // in order to return partial information. - var bp *build.Package - var err error - if modDir != "" { - bp, err = cfg.BuildContext.ImportDir(modDir, 0) - } else if modErr != nil { - bp = new(build.Package) - err = fmt.Errorf("unknown import path %q: %v", importPath, modErr) - } else if cfg.ModulesEnabled && path != "unsafe" { - bp = new(build.Package) - err = fmt.Errorf("unknown import path %q: internal error: module loader did not resolve import", importPath) - } else { - buildMode := build.ImportComment - if mode&ResolveImport == 0 || path != origPath { - // Not vendoring, or we already found the vendored path. - buildMode |= build.IgnoreVendor - } - bp, err = cfg.BuildContext.Import(path, srcDir, buildMode) - } - bp.ImportPath = importPath - if cfg.GOBIN != "" { - bp.BinDir = cfg.GOBIN - } else if cfg.ModulesEnabled { - bp.BinDir = ModBinDir() - } - if modDir == "" && err == nil && !isLocal && bp.ImportComment != "" && bp.ImportComment != path && - !strings.Contains(path, "/vendor/") && !strings.HasPrefix(path, "vendor/") { - err = fmt.Errorf("code in directory %s expects import %q", bp.Dir, bp.ImportComment) - } - p.load(stk, bp, err) - if p.Error != nil && p.Error.Pos == "" { - p = setErrorPos(p, importPos) - } + p.load(path, stk, importPos, bp, err) - if modDir == "" && origPath != cleanImport(origPath) { + if !cfg.ModulesEnabled && path != cleanImport(path) { p.Error = &PackageError{ ImportStack: stk.Copy(), - Err: fmt.Sprintf("non-canonical import path: %q should be %q", origPath, pathpkg.Clean(origPath)), + Err: ImportErrorf(path, "non-canonical import path %q: should be %q", path, pathpkg.Clean(path)), } p.Incomplete = true + setErrorPos(p, importPos) } } @@ -576,7 +672,7 @@ func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPo return setErrorPos(perr, importPos) } if mode&ResolveImport != 0 { - if perr := disallowVendor(srcDir, parent, parentPath, origPath, p, stk); perr != p { + if perr := disallowVendor(srcDir, path, parentPath, p, stk); perr != p { return setErrorPos(perr, importPos) } } @@ -585,16 +681,22 @@ func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPo perr := *p perr.Error = &PackageError{ ImportStack: stk.Copy(), - Err: fmt.Sprintf("import %q is a program, not an importable package", path), + Err: ImportErrorf(path, "import %q is a program, not an importable package", path), } return setErrorPos(&perr, importPos) } if p.Internal.Local && parent != nil && !parent.Internal.Local { perr := *p + var err error + if path == "." { + err = ImportErrorf(path, "%s: cannot import current directory", path) + } else { + err = ImportErrorf(path, "local import %q in non-local package", path) + } perr.Error = &PackageError{ ImportStack: stk.Copy(), - Err: fmt.Sprintf("local import %q in non-local package", path), + Err: err, } return setErrorPos(&perr, importPos) } @@ -611,6 +713,260 @@ func setErrorPos(p *Package, importPos []token.Position) *Package { return p } +// loadPackageData loads information needed to construct a *Package. The result +// is cached, and later calls to loadPackageData for the same package will return +// the same data. +// +// loadPackageData returns a non-nil package even if err is non-nil unless +// the package path is malformed (for example, the path contains "mod/" or "@"). +// +// loadPackageData returns a boolean, loaded, which is true if this is the +// first time the package was loaded. Callers may preload imports in this case. +func loadPackageData(path, parentPath, parentDir, parentRoot string, parentIsStd bool, mode int) (bp *build.Package, loaded bool, err error) { + if path == "" { + panic("loadPackageData called with empty package path") + } + + if strings.HasPrefix(path, "mod/") { + // Paths beginning with "mod/" might accidentally + // look in the module cache directory tree in $GOPATH/pkg/mod/. + // This prefix is owned by the Go core for possible use in the + // standard library (since it does not begin with a domain name), + // so it's OK to disallow entirely. + return nil, false, fmt.Errorf("disallowed import path %q", path) + } + + if strings.Contains(path, "@") { + if cfg.ModulesEnabled { + return nil, false, errors.New("can only use path@version syntax with 'go get'") + } else { + return nil, false, errors.New("cannot use path@version syntax in GOPATH mode") + } + } + + // Determine canonical package path and directory. + // For a local import the identifier is the pseudo-import path + // we create from the full directory to the package. + // Otherwise it is the usual import path. + // For vendored imports, it is the expanded form. + // + // Note that when modules are enabled, local import paths are normally + // canonicalized by modload.ImportPaths before now. However, if there's an + // error resolving a local path, it will be returned untransformed + // so that 'go list -e' reports something useful. + importKey := importSpec{ + path: path, + parentPath: parentPath, + parentDir: parentDir, + parentRoot: parentRoot, + parentIsStd: parentIsStd, + mode: mode, + } + r := resolvedImportCache.Do(importKey, func() interface{} { + var r resolvedImport + if build.IsLocalImport(path) { + r.dir = filepath.Join(parentDir, path) + r.path = dirToImportPath(r.dir) + } else if cfg.ModulesEnabled { + r.dir, r.path, r.err = ModLookup(parentPath, parentIsStd, path) + } else if mode&ResolveImport != 0 { + // We do our own path resolution, because we want to + // find out the key to use in packageCache without the + // overhead of repeated calls to buildContext.Import. + // The code is also needed in a few other places anyway. + r.path = resolveImportPath(path, parentPath, parentDir, parentRoot, parentIsStd) + } else if mode&ResolveModule != 0 { + r.path = moduleImportPath(path, parentPath, parentDir, parentRoot) + } + if r.path == "" { + r.path = path + } + return r + }).(resolvedImport) + // Invariant: r.path is set to the resolved import path. If the path cannot + // be resolved, r.path is set to path, the source import path. + // r.path is never empty. + + // Load the package from its directory. If we already found the package's + // directory when resolving its import path, use that. + data := packageDataCache.Do(r.path, func() interface{} { + loaded = true + var data packageData + if r.dir != "" { + var buildMode build.ImportMode + if !cfg.ModulesEnabled { + buildMode = build.ImportComment + } + data.p, data.err = cfg.BuildContext.ImportDir(r.dir, buildMode) + if data.p.Root == "" && cfg.ModulesEnabled { + if info := ModPackageModuleInfo(path); info != nil { + data.p.Root = info.Dir + } + } + } else if r.err != nil { + data.p = new(build.Package) + data.err = r.err + } else if cfg.ModulesEnabled && path != "unsafe" { + data.p = new(build.Package) + data.err = fmt.Errorf("unknown import path %q: internal error: module loader did not resolve import", r.path) + } else { + buildMode := build.ImportComment + if mode&ResolveImport == 0 || r.path != path { + // Not vendoring, or we already found the vendored path. + buildMode |= build.IgnoreVendor + } + data.p, data.err = cfg.BuildContext.Import(r.path, parentDir, buildMode) + } + data.p.ImportPath = r.path + + // Set data.p.BinDir in cases where go/build.Context.Import + // may give us a path we don't want. + if !data.p.Goroot { + if cfg.GOBIN != "" { + data.p.BinDir = cfg.GOBIN + } else if cfg.ModulesEnabled { + data.p.BinDir = ModBinDir() + } + } + + if !cfg.ModulesEnabled && data.err == nil && + data.p.ImportComment != "" && data.p.ImportComment != path && + !strings.Contains(path, "/vendor/") && !strings.HasPrefix(path, "vendor/") { + data.err = fmt.Errorf("code in directory %s expects import %q", data.p.Dir, data.p.ImportComment) + } + return data + }).(packageData) + + return data.p, loaded, data.err +} + +// importSpec describes an import declaration in source code. It is used as a +// cache key for resolvedImportCache. +type importSpec struct { + path string + parentPath, parentDir, parentRoot string + parentIsStd bool + mode int +} + +// resolvedImport holds a canonical identifier for a package. It may also contain +// a path to the package's directory and an error if one occurred. resolvedImport +// is the value type in resolvedImportCache. +type resolvedImport struct { + path, dir string + err error +} + +// packageData holds information loaded from a package. It is the value type +// in packageDataCache. +type packageData struct { + p *build.Package + err error +} + +// resolvedImportCache maps import strings (importSpec) to canonical package names +// (resolvedImport). +var resolvedImportCache par.Cache + +// packageDataCache maps canonical package names (string) to package metadata +// (packageData). +var packageDataCache par.Cache + +// preloadWorkerCount is the number of concurrent goroutines that can load +// packages. Experimentally, there are diminishing returns with more than +// 4 workers. This was measured on the following machines. +// +// * MacBookPro with a 4-core Intel Core i7 CPU +// * Linux workstation with 6-core Intel Xeon CPU +// * Linux workstation with 24-core Intel Xeon CPU +// +// It is very likely (though not confirmed) that this workload is limited +// by memory bandwidth. We don't have a good way to determine the number of +// workers that would saturate the bus though, so runtime.GOMAXPROCS +// seems like a reasonable default. +var preloadWorkerCount = runtime.GOMAXPROCS(0) + +// preload holds state for managing concurrent preloading of package data. +// +// A preload should be created with newPreload before loading a large +// package graph. flush must be called when package loading is complete +// to ensure preload goroutines are no longer active. This is necessary +// because of global mutable state that cannot safely be read and written +// concurrently. In particular, packageDataCache may be cleared by "go get" +// in GOPATH mode, and modload.loaded (accessed via ModLookup) may be +// modified by modload.ImportPaths (ModImportPaths). +type preload struct { + cancel chan struct{} + sema chan struct{} +} + +// newPreload creates a new preloader. flush must be called later to avoid +// accessing global state while it is being modified. +func newPreload() *preload { + pre := &preload{ + cancel: make(chan struct{}), + sema: make(chan struct{}, preloadWorkerCount), + } + return pre +} + +// preloadMatches loads data for package paths matched by patterns. +// When preloadMatches returns, some packages may not be loaded yet, but +// loadPackageData and loadImport are always safe to call. +func (pre *preload) preloadMatches(matches []*search.Match) { + for _, m := range matches { + for _, pkg := range m.Pkgs { + select { + case <-pre.cancel: + return + case pre.sema <- struct{}{}: + go func(pkg string) { + mode := 0 // don't use vendoring or module import resolution + bp, loaded, err := loadPackageData(pkg, "", base.Cwd, "", false, mode) + <-pre.sema + if bp != nil && loaded && err == nil && !IgnoreImports { + pre.preloadImports(bp.Imports, bp) + } + }(pkg) + } + } + } +} + +// preloadImports queues a list of imports for preloading. +// When preloadImports returns, some packages may not be loaded yet, +// but loadPackageData and loadImport are always safe to call. +func (pre *preload) preloadImports(imports []string, parent *build.Package) { + parentIsStd := parent.Goroot && parent.ImportPath != "" && search.IsStandardImportPath(parent.ImportPath) + for _, path := range imports { + if path == "C" || path == "unsafe" { + continue + } + select { + case <-pre.cancel: + return + case pre.sema <- struct{}{}: + go func(path string) { + bp, loaded, err := loadPackageData(path, parent.ImportPath, parent.Dir, parent.Root, parentIsStd, ResolveImport) + <-pre.sema + if bp != nil && loaded && err == nil && !IgnoreImports { + pre.preloadImports(bp.Imports, bp) + } + }(path) + } + } +} + +// flush stops pending preload operations. flush blocks until preload calls to +// loadPackageData have completed. The preloader will not make any new calls +// to loadPackageData. +func (pre *preload) flush() { + close(pre.cancel) + for i := 0; i < preloadWorkerCount; i++ { + pre.sema <- struct{}{} + } +} + func cleanImport(path string) string { orig := path path = pathpkg.Clean(path) @@ -620,18 +976,13 @@ func cleanImport(path string) string { return path } -var isDirCache = map[string]bool{} +var isDirCache par.Cache func isDir(path string) bool { - result, ok := isDirCache[path] - if ok { - return result - } - - fi, err := os.Stat(path) - result = err == nil && fi.IsDir() - isDirCache[path] = result - return result + return isDirCache.Do(path, func() interface{} { + fi, err := os.Stat(path) + return err == nil && fi.IsDir() + }).(bool) } // ResolveImportPath returns the true meaning of path when it appears in parent. @@ -640,31 +991,44 @@ func isDir(path string) bool { // If vendor expansion doesn't trigger, then the path is also subject to // Go 1.11 module legacy conversion (golang.org/issue/25069). func ResolveImportPath(parent *Package, path string) (found string) { + var parentPath, parentDir, parentRoot string + parentIsStd := false + if parent != nil { + parentPath = parent.ImportPath + parentDir = parent.Dir + parentRoot = parent.Root + parentIsStd = parent.Standard + } + return resolveImportPath(path, parentPath, parentDir, parentRoot, parentIsStd) +} + +func resolveImportPath(path, parentPath, parentDir, parentRoot string, parentIsStd bool) (found string) { if cfg.ModulesEnabled { - if _, p, e := ModLookup(path); e == nil { + if _, p, e := ModLookup(parentPath, parentIsStd, path); e == nil { return p } return path } - found = VendoredImportPath(parent, path) + found = vendoredImportPath(path, parentPath, parentDir, parentRoot) if found != path { return found } - return ModuleImportPath(parent, path) + return moduleImportPath(path, parentPath, parentDir, parentRoot) } // dirAndRoot returns the source directory and workspace root // for the package p, guaranteeing that root is a path prefix of dir. -func dirAndRoot(p *Package) (dir, root string) { - dir = filepath.Clean(p.Dir) - root = filepath.Join(p.Root, "src") - if !str.HasFilePathPrefix(dir, root) || p.ImportPath != "command-line-arguments" && filepath.Join(root, p.ImportPath) != dir { +func dirAndRoot(path string, dir, root string) (string, string) { + origDir, origRoot := dir, root + dir = filepath.Clean(dir) + root = filepath.Join(root, "src") + if !str.HasFilePathPrefix(dir, root) || path != "command-line-arguments" && filepath.Join(root, path) != dir { // Look for symlinks before reporting error. dir = expandPath(dir) root = expandPath(root) } - if !str.HasFilePathPrefix(dir, root) || len(dir) <= len(root) || dir[len(root)] != filepath.Separator || p.ImportPath != "command-line-arguments" && !p.Internal.Local && filepath.Join(root, p.ImportPath) != dir { + if !str.HasFilePathPrefix(dir, root) || len(dir) <= len(root) || dir[len(root)] != filepath.Separator || path != "command-line-arguments" && !build.IsLocalImport(path) && filepath.Join(root, path) != dir { base.Fatalf("unexpected directory layout:\n"+ " import path: %s\n"+ " root: %s\n"+ @@ -672,27 +1036,27 @@ func dirAndRoot(p *Package) (dir, root string) { " expand root: %s\n"+ " expand dir: %s\n"+ " separator: %s", - p.ImportPath, - filepath.Join(p.Root, "src"), - filepath.Clean(p.Dir), - root, - dir, + path, + filepath.Join(origRoot, "src"), + filepath.Clean(origDir), + origRoot, + origDir, string(filepath.Separator)) } return dir, root } -// VendoredImportPath returns the vendor-expansion of path when it appears in parent. +// vendoredImportPath returns the vendor-expansion of path when it appears in parent. // If parent is x/y/z, then path might expand to x/y/z/vendor/path, x/y/vendor/path, // x/vendor/path, vendor/path, or else stay path if none of those exist. -// VendoredImportPath returns the expanded path or, if no expansion is found, the original. -func VendoredImportPath(parent *Package, path string) (found string) { - if parent == nil || parent.Root == "" { +// vendoredImportPath returns the expanded path or, if no expansion is found, the original. +func vendoredImportPath(path, parentPath, parentDir, parentRoot string) (found string) { + if parentRoot == "" { return path } - dir, root := dirAndRoot(parent) + dir, root := dirAndRoot(parentPath, parentDir, parentRoot) vpath := "vendor/" + path for i := len(dir); i >= len(root); i-- { @@ -708,7 +1072,7 @@ func VendoredImportPath(parent *Package, path string) (found string) { } targ := filepath.Join(dir[:i], vpath) if isDir(targ) && hasGoFiles(targ) { - importPath := parent.ImportPath + importPath := parentPath if importPath == "command-line-arguments" { // If parent.ImportPath is 'command-line-arguments'. // set to relative directory to root (also chopped root directory) @@ -738,54 +1102,48 @@ func VendoredImportPath(parent *Package, path string) (found string) { var ( modulePrefix = []byte("\nmodule ") - goModPathCache = make(map[string]string) + goModPathCache par.Cache ) // goModPath returns the module path in the go.mod in dir, if any. func goModPath(dir string) (path string) { - path, ok := goModPathCache[dir] - if ok { - return path - } - defer func() { - goModPathCache[dir] = path - }() - - data, err := ioutil.ReadFile(filepath.Join(dir, "go.mod")) - if err != nil { - return "" - } - var i int - if bytes.HasPrefix(data, modulePrefix[1:]) { - i = 0 - } else { - i = bytes.Index(data, modulePrefix) - if i < 0 { + return goModPathCache.Do(dir, func() interface{} { + data, err := ioutil.ReadFile(filepath.Join(dir, "go.mod")) + if err != nil { return "" } - i++ - } - line := data[i:] + var i int + if bytes.HasPrefix(data, modulePrefix[1:]) { + i = 0 + } else { + i = bytes.Index(data, modulePrefix) + if i < 0 { + return "" + } + i++ + } + line := data[i:] - // Cut line at \n, drop trailing \r if present. - if j := bytes.IndexByte(line, '\n'); j >= 0 { - line = line[:j] - } - if line[len(line)-1] == '\r' { - line = line[:len(line)-1] - } - line = line[len("module "):] + // Cut line at \n, drop trailing \r if present. + if j := bytes.IndexByte(line, '\n'); j >= 0 { + line = line[:j] + } + if line[len(line)-1] == '\r' { + line = line[:len(line)-1] + } + line = line[len("module "):] - // If quoted, unquote. - path = strings.TrimSpace(string(line)) - if path != "" && path[0] == '"' { - s, err := strconv.Unquote(path) - if err != nil { - return "" + // If quoted, unquote. + path = strings.TrimSpace(string(line)) + if path != "" && path[0] == '"' { + s, err := strconv.Unquote(path) + if err != nil { + return "" + } + path = s } - path = s - } - return path + return path + }).(string) } // findVersionElement returns the slice indices of the final version element /vN in path. @@ -794,7 +1152,7 @@ func findVersionElement(path string) (i, j int) { j = len(path) for i = len(path) - 1; i >= 0; i-- { if path[i] == '/' { - if isVersionElement(path[i:j]) { + if isVersionElement(path[i+1 : j]) { return i, j } j = i @@ -806,10 +1164,10 @@ func findVersionElement(path string) (i, j int) { // isVersionElement reports whether s is a well-formed path version element: // v2, v3, v10, etc, but not v0, v05, v1. func isVersionElement(s string) bool { - if len(s) < 3 || s[0] != '/' || s[1] != 'v' || s[2] == '0' || s[2] == '1' && len(s) == 3 { + if len(s) < 2 || s[0] != 'v' || s[1] == '0' || s[1] == '1' && len(s) == 2 { return false } - for i := 2; i < len(s); i++ { + for i := 1; i < len(s); i++ { if s[i] < '0' || '9' < s[i] { return false } @@ -817,7 +1175,7 @@ func isVersionElement(s string) bool { return true } -// ModuleImportPath translates import paths found in go modules +// moduleImportPath translates import paths found in go modules // back down to paths that can be resolved in ordinary builds. // // Define “new” code as code with a go.mod file in the same directory @@ -825,8 +1183,8 @@ func isVersionElement(s string) bool { // x/y/v2/z does not exist and x/y/go.mod says “module x/y/v2”, // then go build will read the import as x/y/z instead. // See golang.org/issue/25069. -func ModuleImportPath(parent *Package, path string) (found string) { - if parent == nil || parent.Root == "" { +func moduleImportPath(path, parentPath, parentDir, parentRoot string) (found string) { + if parentRoot == "" { return path } @@ -838,7 +1196,7 @@ func ModuleImportPath(parent *Package, path string) (found string) { return path } - dir, root := dirAndRoot(parent) + dir, root := dirAndRoot(parentPath, parentDir, parentRoot) // Consider dir and parents, up to and including root. for i := len(dir); i >= len(root); i-- { @@ -919,7 +1277,7 @@ func reusePackage(p *Package, stk *ImportStack) *Package { if p.Error == nil { p.Error = &PackageError{ ImportStack: stk.Copy(), - Err: "import cycle not allowed", + Err: errors.New("import cycle not allowed"), IsImportCycle: true, } } @@ -952,7 +1310,7 @@ func disallowInternal(srcDir string, importer *Package, importerPath string, p * // as if it were generated into the testing directory tree // (it's actually in a temporary directory outside any Go tree). // This cleans up a former kludge in passing functionality to the testing package. - if strings.HasPrefix(p.ImportPath, "testing/internal") && len(*stk) >= 2 && (*stk)[len(*stk)-2] == "testmain" { + if str.HasPathPrefix(p.ImportPath, "testing/internal") && importerPath == "testmain" { return p } @@ -961,11 +1319,17 @@ func disallowInternal(srcDir string, importer *Package, importerPath string, p * return p } - // The stack includes p.ImportPath. - // If that's the only thing on the stack, we started + // The sort package depends on internal/reflectlite, but during bootstrap + // the path rewriting causes the normal internal checks to fail. + // Instead, just ignore the internal rules during bootstrap. + if p.Standard && strings.HasPrefix(importerPath, "bootstrap/") { + return p + } + + // importerPath is empty: we started // with a name given on the command line, not an // import. Anything listed on the command line is fine. - if len(*stk) == 1 { + if importerPath == "" { return p } @@ -1014,8 +1378,9 @@ func disallowInternal(srcDir string, importer *Package, importerPath string, p * // Internal is present, and srcDir is outside parent's tree. Not allowed. perr := *p perr.Error = &PackageError{ - ImportStack: stk.Copy(), - Err: "use of internal package " + p.ImportPath + " not allowed", + alwaysPrintStack: true, + ImportStack: stk.Copy(), + Err: ImportErrorf(p.ImportPath, "use of internal package "+p.ImportPath+" not allowed"), } perr.Incomplete = true return &perr @@ -1040,20 +1405,18 @@ func findInternal(path string) (index int, ok bool) { return 0, false } -// disallowVendor checks that srcDir (containing package importerPath, if non-empty) -// is allowed to import p as path. +// disallowVendor checks that srcDir is allowed to import p as path. // If the import is allowed, disallowVendor returns the original package p. // If not, it returns a new package containing just an appropriate error. -func disallowVendor(srcDir string, importer *Package, importerPath, path string, p *Package, stk *ImportStack) *Package { - // The stack includes p.ImportPath. - // If that's the only thing on the stack, we started +func disallowVendor(srcDir string, path string, importerPath string, p *Package, stk *ImportStack) *Package { + // If the importerPath is empty, we started // with a name given on the command line, not an // import. Anything listed on the command line is fine. - if len(*stk) == 1 { + if importerPath == "" { return p } - if perr := disallowVendorVisibility(srcDir, p, stk); perr != p { + if perr := disallowVendorVisibility(srcDir, p, importerPath, stk); perr != p { return perr } @@ -1062,7 +1425,7 @@ func disallowVendor(srcDir string, importer *Package, importerPath, path string, perr := *p perr.Error = &PackageError{ ImportStack: stk.Copy(), - Err: "must be imported as " + path[i+len("vendor/"):], + Err: ImportErrorf(path, "%s must be imported as %s", path, path[i+len("vendor/"):]), } perr.Incomplete = true return &perr @@ -1076,12 +1439,12 @@ func disallowVendor(srcDir string, importer *Package, importerPath, path string, // is not subject to the rules, only subdirectories of vendor. // This allows people to have packages and commands named vendor, // for maximal compatibility with existing source trees. -func disallowVendorVisibility(srcDir string, p *Package, stk *ImportStack) *Package { - // The stack includes p.ImportPath. - // If that's the only thing on the stack, we started +func disallowVendorVisibility(srcDir string, p *Package, importerPath string, stk *ImportStack) *Package { + // The stack does not include p.ImportPath. + // If there's nothing on the stack, we started // with a name given on the command line, not an // import. Anything listed on the command line is fine. - if len(*stk) == 1 { + if importerPath == "" { return p } @@ -1116,7 +1479,7 @@ func disallowVendorVisibility(srcDir string, p *Package, stk *ImportStack) *Pack perr := *p perr.Error = &PackageError{ ImportStack: stk.Copy(), - Err: "use of vendored package not allowed", + Err: errors.New("use of vendored package not allowed"), } perr.Incomplete = true return &perr @@ -1178,39 +1541,55 @@ var cgoSyscallExclude = map[string]bool{ var foldPath = make(map[string]string) -// DefaultExecName returns the default executable name -// for a package with the import path importPath. +// exeFromImportPath returns an executable name +// for a package using the import path. // -// The default executable name is the last element of the import path. -// In module-aware mode, an additional rule is used. If the last element -// is a vN path element specifying the major version, then the second last -// element of the import path is used instead. -func DefaultExecName(importPath string) string { - _, elem := pathpkg.Split(importPath) +// The executable name is the last element of the import path. +// In module-aware mode, an additional rule is used on import paths +// consisting of two or more path elements. If the last element is +// a vN path element specifying the major version, then the +// second last element of the import path is used instead. +func (p *Package) exeFromImportPath() string { + _, elem := pathpkg.Split(p.ImportPath) if cfg.ModulesEnabled { - // If this is example.com/mycmd/v2, it's more useful to install it as mycmd than as v2. - // See golang.org/issue/24667. - isVersion := func(v string) bool { - if len(v) < 2 || v[0] != 'v' || v[1] < '1' || '9' < v[1] { - return false - } - for i := 2; i < len(v); i++ { - if c := v[i]; c < '0' || '9' < c { - return false - } - } - return true - } - if isVersion(elem) { - _, elem = pathpkg.Split(pathpkg.Dir(importPath)) + // If this is example.com/mycmd/v2, it's more useful to + // install it as mycmd than as v2. See golang.org/issue/24667. + if elem != p.ImportPath && isVersionElement(elem) { + _, elem = pathpkg.Split(pathpkg.Dir(p.ImportPath)) } } return elem } +// exeFromFiles returns an executable name for a package +// using the first element in GoFiles or CgoFiles collections without the prefix. +// +// Returns empty string in case of empty collection. +func (p *Package) exeFromFiles() string { + var src string + if len(p.GoFiles) > 0 { + src = p.GoFiles[0] + } else if len(p.CgoFiles) > 0 { + src = p.CgoFiles[0] + } else { + return "" + } + _, elem := filepath.Split(src) + return elem[:len(elem)-len(".go")] +} + +// DefaultExecName returns the default executable name for a package +func (p *Package) DefaultExecName() string { + if p.Internal.CmdlineFiles { + return p.exeFromFiles() + } + return p.exeFromImportPath() +} + // load populates p using information from bp, err, which should // be the result of calling build.Context.Import. -func (p *Package) load(stk *ImportStack, bp *build.Package, err error) { +// stk contains the import stack, not including path itself. +func (p *Package) load(path string, stk *ImportStack, importPos []token.Position, bp *build.Package, err error) { p.copyBuild(bp) // The localPrefix is the path we interpret ./ imports relative to. @@ -1219,17 +1598,31 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) { p.Internal.LocalPrefix = dirToImportPath(p.Dir) } - if err != nil { - if _, ok := err.(*build.NoGoError); ok { - err = &NoGoError{Package: p} + // setError sets p.Error if it hasn't already been set. We may proceed + // after encountering some errors so that 'go list -e' has more complete + // output. If there's more than one error, we should report the first. + setError := func(err error) { + if p.Error == nil { + p.Error = &PackageError{ + ImportStack: stk.Copy(), + Err: err, + } + + // Add the importer's position information if the import position exists, and + // the current package being examined is the importer. + // If we have not yet accepted package p onto the import stack, + // then the cause of the error is not within p itself: the error + // must be either in an explicit command-line argument, + // or on the importer side (indicated by a non-empty importPos). + if path != stk.Top() && len(importPos) > 0 { + p = setErrorPos(p, importPos) + } } + } + + if err != nil { p.Incomplete = true - err = base.ExpandScanner(err) - p.Error = &PackageError{ - ImportStack: stk.Copy(), - Err: err.Error(), - } - return + p.setLoadPackageDataError(err, path, stk, importPos) } useBindir := p.Name == "main" @@ -1243,26 +1636,14 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) { if useBindir { // Report an error when the old code.google.com/p/go.tools paths are used. if InstallTargetDir(p) == StalePath { + // TODO(matloob): remove this branch, and StalePath itself. code.google.com/p/go is so + // old, even this code checking for it is stale now! newPath := strings.Replace(p.ImportPath, "code.google.com/p/go.", "golang.org/x/", 1) - e := fmt.Sprintf("the %v command has moved; use %v instead.", p.ImportPath, newPath) - p.Error = &PackageError{Err: e} + e := ImportErrorf(p.ImportPath, "the %v command has moved; use %v instead.", p.ImportPath, newPath) + setError(e) return } - _, elem := filepath.Split(p.Dir) - if cfg.ModulesEnabled { - // NOTE(rsc,dmitshur): Using p.ImportPath instead of p.Dir - // makes sure we install a package in the root of a - // cached module directory as that package name - // not name@v1.2.3. - // Using p.ImportPath instead of p.Dir - // is probably correct all the time, - // even for non-module-enabled code, - // but I'm not brave enough to change the - // non-module behavior this late in the - // release cycle. Can be done for Go 1.13. - // See golang.org/issue/26869. - elem = DefaultExecName(p.ImportPath) - } + elem := p.DefaultExecName() full := cfg.BuildContext.GOOS + "_" + cfg.BuildContext.GOARCH + "/" + elem if cfg.BuildContext.GOOS != base.ToolGOOS || cfg.BuildContext.GOARCH != base.ToolGOARCH { // Install cross-compiled binaries to subdirectories of bin. @@ -1298,7 +1679,10 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) { p.Target = "" } else { p.Target = p.Internal.Build.PkgObj - if cfg.BuildLinkshared { + if cfg.BuildLinkshared && p.Target != "" { + // TODO(bcmills): The reliance on p.Target implies that -linkshared does + // not work for any package that lacks a Target — such as a non-main + // package in module mode. We should probably fix that. shlibnamefile := p.Target[:len(p.Target)-2] + ".shlibname" shlib, err := ioutil.ReadFile(shlibnamefile) if err != nil && !os.IsNotExist(err) { @@ -1362,6 +1746,23 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) { } } + // Check for case-insensitive collisions of import paths. + fold := str.ToFold(p.ImportPath) + if other := foldPath[fold]; other == "" { + foldPath[fold] = p.ImportPath + } else if other != p.ImportPath { + setError(ImportErrorf(p.ImportPath, "case-insensitive import collision: %q and %q", p.ImportPath, other)) + return + } + + if !SafeArg(p.ImportPath) { + setError(ImportErrorf(p.ImportPath, "invalid import path %q", p.ImportPath)) + return + } + + stk.Push(path) + defer stk.Pop() + // Check for case-insensitive collision of input files. // To avoid problems on case-insensitive files, we reject any package // where two different input files have equal names under a case-insensitive @@ -1369,10 +1770,7 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) { inputs := p.AllFiles() f1, f2 := str.FoldDup(inputs) if f1 != "" { - p.Error = &PackageError{ - ImportStack: stk.Copy(), - Err: fmt.Sprintf("case-insensitive file name collision: %q and %q", f1, f2), - } + setError(fmt.Errorf("case-insensitive file name collision: %q and %q", f1, f2)) return } @@ -1385,25 +1783,12 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) { // so we shouldn't see any _cgo_ files anyway, but just be safe. for _, file := range inputs { if !SafeArg(file) || strings.HasPrefix(file, "_cgo_") { - p.Error = &PackageError{ - ImportStack: stk.Copy(), - Err: fmt.Sprintf("invalid input file name %q", file), - } + setError(fmt.Errorf("invalid input file name %q", file)) return } } if name := pathpkg.Base(p.ImportPath); !SafeArg(name) { - p.Error = &PackageError{ - ImportStack: stk.Copy(), - Err: fmt.Sprintf("invalid input directory name %q", name), - } - return - } - if !SafeArg(p.ImportPath) { - p.Error = &PackageError{ - ImportStack: stk.Copy(), - Err: fmt.Sprintf("invalid import path %q", p.ImportPath), - } + setError(fmt.Errorf("invalid input directory name %q", name)) return } @@ -1414,16 +1799,6 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) { continue } p1 := LoadImport(path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport) - if p.Standard && p.Error == nil && !p1.Standard && p1.Error == nil { - p.Error = &PackageError{ - ImportStack: stk.Copy(), - Err: fmt.Sprintf("non-standard import %q in standard package %q", path, p.ImportPath), - } - pos := p.Internal.Build.ImportPos[path] - if len(pos) > 0 { - p.Error.Pos = pos[0].String() - } - } path = p1.ImportPath importPaths[i] = path @@ -1437,41 +1812,7 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) { } } p.Internal.Imports = imports - - deps := make(map[string]*Package) - var q []*Package - q = append(q, imports...) - for i := 0; i < len(q); i++ { - p1 := q[i] - path := p1.ImportPath - // The same import path could produce an error or not, - // depending on what tries to import it. - // Prefer to record entries with errors, so we can report them. - p0 := deps[path] - if p0 == nil || p1.Error != nil && (p0.Error == nil || len(p0.Error.ImportStack) > len(p1.Error.ImportStack)) { - deps[path] = p1 - for _, p2 := range p1.Internal.Imports { - if deps[p2.ImportPath] != p2 { - q = append(q, p2) - } - } - } - } - - p.Deps = make([]string, 0, len(deps)) - for dep := range deps { - p.Deps = append(p.Deps, dep) - } - sort.Strings(p.Deps) - for _, dep := range p.Deps { - p1 := deps[dep] - if p1 == nil { - panic("impossible: missing entry in package cache for " + dep + " imported by " + p.ImportPath) - } - if p1.Error != nil { - p.DepsErrors = append(p.DepsErrors, p1.Error) - } - } + p.collectDeps() // unsafe is a fake package. if p.Standard && (p.ImportPath == "unsafe" || cfg.BuildContext.Compiler == "gccgo") { @@ -1492,55 +1833,81 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) { // code; see issue #16050). } - setError := func(msg string) { - p.Error = &PackageError{ - ImportStack: stk.Copy(), - Err: msg, - } - } - // The gc toolchain only permits C source files with cgo or SWIG. if len(p.CFiles) > 0 && !p.UsesCgo() && !p.UsesSwig() && cfg.BuildContext.Compiler == "gc" { - setError(fmt.Sprintf("C source files not allowed when not using cgo or SWIG: %s", strings.Join(p.CFiles, " "))) + setError(fmt.Errorf("C source files not allowed when not using cgo or SWIG: %s", strings.Join(p.CFiles, " "))) return } // C++, Objective-C, and Fortran source files are permitted only with cgo or SWIG, // regardless of toolchain. if len(p.CXXFiles) > 0 && !p.UsesCgo() && !p.UsesSwig() { - setError(fmt.Sprintf("C++ source files not allowed when not using cgo or SWIG: %s", strings.Join(p.CXXFiles, " "))) + setError(fmt.Errorf("C++ source files not allowed when not using cgo or SWIG: %s", strings.Join(p.CXXFiles, " "))) return } if len(p.MFiles) > 0 && !p.UsesCgo() && !p.UsesSwig() { - setError(fmt.Sprintf("Objective-C source files not allowed when not using cgo or SWIG: %s", strings.Join(p.MFiles, " "))) + setError(fmt.Errorf("Objective-C source files not allowed when not using cgo or SWIG: %s", strings.Join(p.MFiles, " "))) return } if len(p.FFiles) > 0 && !p.UsesCgo() && !p.UsesSwig() { - setError(fmt.Sprintf("Fortran source files not allowed when not using cgo or SWIG: %s", strings.Join(p.FFiles, " "))) + setError(fmt.Errorf("Fortran source files not allowed when not using cgo or SWIG: %s", strings.Join(p.FFiles, " "))) return } - // Check for case-insensitive collisions of import paths. - fold := str.ToFold(p.ImportPath) - if other := foldPath[fold]; other == "" { - foldPath[fold] = p.ImportPath - } else if other != p.ImportPath { - setError(fmt.Sprintf("case-insensitive import collision: %q and %q", p.ImportPath, other)) - return - } - - if cfg.ModulesEnabled { + if cfg.ModulesEnabled && p.Error == nil { mainPath := p.ImportPath if p.Internal.CmdlineFiles { mainPath = "command-line-arguments" } p.Module = ModPackageModuleInfo(mainPath) - if p.Name == "main" { + if p.Name == "main" && len(p.DepsErrors) == 0 { p.Internal.BuildInfo = ModPackageBuildInfo(mainPath, p.Deps) } } } +// collectDeps populates p.Deps and p.DepsErrors by iterating over +// p.Internal.Imports. +// +// TODO(jayconrod): collectDeps iterates over transitive imports for every +// package. We should only need to visit direct imports. +func (p *Package) collectDeps() { + deps := make(map[string]*Package) + var q []*Package + q = append(q, p.Internal.Imports...) + for i := 0; i < len(q); i++ { + p1 := q[i] + path := p1.ImportPath + // The same import path could produce an error or not, + // depending on what tries to import it. + // Prefer to record entries with errors, so we can report them. + p0 := deps[path] + if p0 == nil || p1.Error != nil && (p0.Error == nil || len(p0.Error.ImportStack) > len(p1.Error.ImportStack)) { + deps[path] = p1 + for _, p2 := range p1.Internal.Imports { + if deps[p2.ImportPath] != p2 { + q = append(q, p2) + } + } + } + } + + p.Deps = make([]string, 0, len(deps)) + for dep := range deps { + p.Deps = append(p.Deps, dep) + } + sort.Strings(p.Deps) + for _, dep := range p.Deps { + p1 := deps[dep] + if p1 == nil { + panic("impossible: missing entry in package cache for " + dep + " imported by " + p.ImportPath) + } + if p1.Error != nil { + p.DepsErrors = append(p.DepsErrors, p1.Error) + } + } +} + // SafeArg reports whether arg is a "safe" command-line argument, // meaning that when it appears in a command-line, it probably // doesn't have some special meaning other than its own name. @@ -1589,10 +1956,11 @@ func externalLinkingForced(p *Package) bool { // Some targets must use external linking even inside GOROOT. switch cfg.BuildContext.GOOS { case "android": - return true + if cfg.BuildContext.GOARCH != "arm64" { + return true + } case "darwin": - switch cfg.BuildContext.GOARCH { - case "arm", "arm64": + if cfg.BuildContext.GOARCH == "arm64" { return true } } @@ -1650,13 +2018,22 @@ func (p *Package) InternalXGoFiles() []string { // using absolute paths. "Possibly relevant" means that files are not excluded // due to build tags, but files with names beginning with . or _ are still excluded. func (p *Package) InternalAllGoFiles() []string { - var extra []string + return p.mkAbs(str.StringList(p.constraintIgnoredGoFiles(), p.GoFiles, p.CgoFiles, p.TestGoFiles, p.XTestGoFiles)) +} + +// constraintIgnoredGoFiles returns the list of Go files ignored for reasons +// other than having a name beginning with '.' or '_'. +func (p *Package) constraintIgnoredGoFiles() []string { + if len(p.IgnoredGoFiles) == 0 { + return nil + } + files := make([]string, 0, len(p.IgnoredGoFiles)) for _, f := range p.IgnoredGoFiles { - if f != "" && f[0] != '.' || f[0] != '_' { - extra = append(extra, f) + if f != "" && f[0] != '.' && f[0] != '_' { + files = append(files, f) } } - return p.mkAbs(str.StringList(extra, p.GoFiles, p.CgoFiles, p.TestGoFiles, p.XTestGoFiles)) + return files } // usesSwig reports whether the package needs to run SWIG. @@ -1727,99 +2104,17 @@ func TestPackageList(roots []*Package) []*Package { return all } -var cmdCache = map[string]*Package{} - -func ClearCmdCache() { - for name := range cmdCache { - delete(cmdCache, name) - } -} - -// LoadPackage loads the package named by arg. -func LoadPackage(arg string, stk *ImportStack) *Package { - p := loadPackage(arg, stk) +// LoadImportWithFlags loads the package with the given import path and +// sets tool flags on that package. This function is useful loading implicit +// dependencies (like sync/atomic for coverage). +// TODO(jayconrod): delete this function and set flags automatically +// in LoadImport instead. +func LoadImportWithFlags(path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { + p := LoadImport(path, srcDir, parent, stk, importPos, mode) setToolFlags(p) return p } -// LoadPackageNoFlags is like LoadPackage -// but does not guarantee that the build tool flags are set in the result. -// It is only for use by GOPATH-based "go get" -// and is only appropriate for preliminary loading of packages. -// A real load using LoadPackage or (more likely) -// Packages, PackageAndErrors, or PackagesForBuild -// must be done before passing the package to any build -// steps, so that the tool flags can be set properly. -// TODO(rsc): When GOPATH-based "go get" is removed, delete this function. -func LoadPackageNoFlags(arg string, stk *ImportStack) *Package { - return loadPackage(arg, stk) -} - -// loadPackage is like loadImport but is used for command-line arguments, -// not for paths found in import statements. In addition to ordinary import paths, -// loadPackage accepts pseudo-paths beginning with cmd/ to denote commands -// in the Go command directory, as well as paths to those directories. -func loadPackage(arg string, stk *ImportStack) *Package { - if arg == "" { - panic("loadPackage called with empty package path") - } - if build.IsLocalImport(arg) { - dir := arg - if !filepath.IsAbs(dir) { - if abs, err := filepath.Abs(dir); err == nil { - // interpret relative to current directory - dir = abs - } - } - if sub, ok := hasSubdir(cfg.GOROOTsrc, dir); ok && strings.HasPrefix(sub, "cmd/") && !strings.Contains(sub[4:], "/") { - arg = sub - } - } - if strings.HasPrefix(arg, "cmd/") && !strings.Contains(arg[4:], "/") { - if p := cmdCache[arg]; p != nil { - return p - } - stk.Push(arg) - defer stk.Pop() - - bp, err := cfg.BuildContext.ImportDir(filepath.Join(cfg.GOROOTsrc, arg), 0) - bp.ImportPath = arg - bp.Goroot = true - bp.BinDir = cfg.GOROOTbin - bp.Root = cfg.GOROOT - bp.SrcRoot = cfg.GOROOTsrc - p := new(Package) - cmdCache[arg] = p - p.load(stk, bp, err) - if p.Error == nil && p.Name != "main" { - p.Error = &PackageError{ - ImportStack: stk.Copy(), - Err: fmt.Sprintf("expected package main but found package %s in %s", p.Name, p.Dir), - } - } - return p - } - - // Wasn't a command; must be a package. - // If it is a local import path but names a standard package, - // we treat it as if the user specified the standard package. - // This lets you run go test ./ioutil in package io and be - // referring to io/ioutil rather than a hypothetical import of - // "./ioutil". - if build.IsLocalImport(arg) || filepath.IsAbs(arg) { - dir := arg - if !filepath.IsAbs(arg) { - dir = filepath.Join(base.Cwd, arg) - } - bp, _ := cfg.BuildContext.ImportDir(dir, build.FindOnly) - if bp.ImportPath != "" && bp.ImportPath != "." { - arg = bp.ImportPath - } - } - - return LoadImport(arg, base.Cwd, nil, stk, nil, 0) -} - // Packages returns the packages named by the // command line arguments 'args'. If a named package // cannot be loaded at all (for example, if the directory does not exist), @@ -1832,7 +2127,7 @@ func Packages(args []string) []*Package { var pkgs []*Package for _, pkg := range PackagesAndErrors(args) { if pkg.Error != nil { - base.Errorf("can't load package: %s", pkg.Error) + base.Errorf("%v", pkg.Error) continue } pkgs = append(pkgs, pkg) @@ -1845,8 +2140,17 @@ func Packages(args []string) []*Package { // cannot be loaded at all. // The packages that fail to load will have p.Error != nil. func PackagesAndErrors(patterns []string) []*Package { - if len(patterns) > 0 && strings.HasSuffix(patterns[0], ".go") { - return []*Package{GoFilesPackage(patterns)} + for _, p := range patterns { + // Listing is only supported with all patterns referring to either: + // - Files that are part of the same directory. + // - Explicit package paths or patterns. + if strings.HasSuffix(p, ".go") { + // We need to test whether the path is an actual Go file and not a + // package path or pattern ending in '.go' (see golang.org/issue/34653). + if fi, err := os.Stat(p); err == nil && !fi.IsDir() { + return []*Package{GoFilesPackage(patterns)} + } + } } matches := ImportPaths(patterns) @@ -1856,16 +2160,20 @@ func PackagesAndErrors(patterns []string) []*Package { seenPkg = make(map[*Package]bool) ) + pre := newPreload() + defer pre.flush() + pre.preloadMatches(matches) + for _, m := range matches { for _, pkg := range m.Pkgs { if pkg == "" { - panic(fmt.Sprintf("ImportPaths returned empty package for pattern %s", m.Pattern)) + panic(fmt.Sprintf("ImportPaths returned empty package for pattern %s", m.Pattern())) } - p := loadPackage(pkg, &stk) - p.Match = append(p.Match, m.Pattern) + p := loadImport(pre, pkg, base.Cwd, nil, &stk, nil, 0) + p.Match = append(p.Match, m.Pattern()) p.Internal.CmdlinePkg = true - if m.Literal { - // Note: do not set = m.Literal unconditionally + if m.IsLiteral() { + // Note: do not set = m.IsLiteral unconditionally // because maybe we'll see p matching both // a literal and also a non-literal pattern. p.Internal.CmdlinePkgLiteral = true @@ -1876,6 +2184,25 @@ func PackagesAndErrors(patterns []string) []*Package { seenPkg[p] = true pkgs = append(pkgs, p) } + + if len(m.Errs) > 0 { + // In addition to any packages that were actually resolved from the + // pattern, there was some error in resolving the pattern itself. + // Report it as a synthetic package. + p := new(Package) + p.ImportPath = m.Pattern() + // Pass an empty ImportStack and nil importPos: the error arose from a pattern, not an import. + var stk ImportStack + var importPos []token.Position + p.setLoadPackageDataError(m.Errs[0], m.Pattern(), &stk, importPos) + p.Incomplete = true + p.Match = append(p.Match, m.Pattern()) + p.Internal.CmdlinePkg = true + if m.IsLiteral() { + p.Internal.CmdlinePkgLiteral = true + } + pkgs = append(pkgs, p) + } } // Now that CmdlinePkg is set correctly, @@ -1911,7 +2238,7 @@ func PackagesForBuild(args []string) []*Package { printed := map[*PackageError]bool{} for _, pkg := range pkgs { if pkg.Error != nil { - base.Errorf("can't load package: %s", pkg.Error) + base.Errorf("%v", pkg.Error) printed[pkg.Error] = true } for _, err := range pkg.DepsErrors { @@ -1921,7 +2248,7 @@ func PackagesForBuild(args []string) []*Package { // Only print each once. if !printed[err] { printed[err] = true - base.Errorf("%s", err) + base.Errorf("%v", err) } } } @@ -1954,7 +2281,14 @@ func GoFilesPackage(gofiles []string) *Package { for _, f := range gofiles { if !strings.HasSuffix(f, ".go") { - base.Fatalf("named files must be .go files") + pkg := new(Package) + pkg.Internal.Local = true + pkg.Internal.CmdlineFiles = true + pkg.Name = f + pkg.Error = &PackageError{ + Err: fmt.Errorf("named files must be .go files: %s", pkg.Name), + } + return pkg } } @@ -2006,20 +2340,15 @@ func GoFilesPackage(gofiles []string) *Package { pkg := new(Package) pkg.Internal.Local = true pkg.Internal.CmdlineFiles = true - stk.Push("main") - pkg.load(&stk, bp, err) - stk.Pop() + pkg.load("command-line-arguments", &stk, nil, bp, err) pkg.Internal.LocalPrefix = dirToImportPath(dir) pkg.ImportPath = "command-line-arguments" pkg.Target = "" pkg.Match = gofiles if pkg.Name == "main" { - _, elem := filepath.Split(gofiles[0]) - exe := elem[:len(elem)-len(".go")] + cfg.ExeSuffix - if cfg.BuildO == "" { - cfg.BuildO = exe - } + exe := pkg.DefaultExecName() + cfg.ExeSuffix + if cfg.GOBIN != "" { pkg.Target = filepath.Join(cfg.GOBIN, exe) } else if cfg.ModulesEnabled { diff --git a/cmd/go/_internal_/load/test.go b/cmd/go/_internal_/load/test.go index 3c808d7..2aff72e 100644 --- a/cmd/go/_internal_/load/test.go +++ b/cmd/go/_internal_/load/test.go @@ -6,7 +6,6 @@ package load import ( "bytes" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/str" "errors" "fmt" @@ -15,10 +14,10 @@ import ( "go/doc" "go/parser" "go/token" + "github.com/dependabot/gomodules-extracted/_internal_/lazytemplate" "path/filepath" "sort" "strings" - "text/template" "unicode" "unicode/utf8" ) @@ -26,6 +25,7 @@ import ( var TestMainDeps = []string{ // Dependencies for testmain. "os", + "reflect", "testing", "testing/internal/testdeps", } @@ -39,10 +39,42 @@ type TestCover struct { DeclVars func(*Package, ...string) map[string]*CoverVar } -// TestPackagesFor returns three packages: +// TestPackagesFor is like TestPackagesAndErrors but it returns +// an error if the test packages or their dependencies have errors. +// Only test packages without errors are returned. +func TestPackagesFor(p *Package, cover *TestCover) (pmain, ptest, pxtest *Package, err error) { + pmain, ptest, pxtest = TestPackagesAndErrors(p, cover) + for _, p1 := range []*Package{ptest, pxtest, pmain} { + if p1 == nil { + // pxtest may be nil + continue + } + if p1.Error != nil { + err = p1.Error + break + } + if len(p1.DepsErrors) > 0 { + perr := p1.DepsErrors[0] + err = perr + break + } + } + if pmain.Error != nil || len(pmain.DepsErrors) > 0 { + pmain = nil + } + if ptest.Error != nil || len(ptest.DepsErrors) > 0 { + ptest = nil + } + if pxtest != nil && (pxtest.Error != nil || len(pxtest.DepsErrors) > 0) { + pxtest = nil + } + return pmain, ptest, pxtest, err +} + +// TestPackagesAndErrors returns three packages: +// - pmain, the package main corresponding to the test binary (running tests in ptest and pxtest). // - ptest, the package p compiled with added "package p" test files. // - pxtest, the result of compiling any "package p_test" (external) test files. -// - pmain, the package main corresponding to the test binary (running tests in ptest and pxtest). // // If the package has no "package p_test" test files, pxtest will be nil. // If the non-test compilation of package p can be reused @@ -50,33 +82,36 @@ type TestCover struct { // package p need not be instrumented for coverage or any other reason), // then the returned ptest == p. // +// An error is returned if the testmain source cannot be completely generated +// (for example, due to a syntax error in a test file). No error will be +// returned for errors loading packages, but the Error or DepsError fields +// of the returned packages may be set. +// // The caller is expected to have checked that len(p.TestGoFiles)+len(p.XTestGoFiles) > 0, // or else there's no point in any of this. -func TestPackagesFor(p *Package, cover *TestCover) (pmain, ptest, pxtest *Package, err error) { +func TestPackagesAndErrors(p *Package, cover *TestCover) (pmain, ptest, pxtest *Package) { + pre := newPreload() + defer pre.flush() + allImports := append([]string{}, p.TestImports...) + allImports = append(allImports, p.XTestImports...) + pre.preloadImports(allImports, p.Internal.Build) + + var ptestErr, pxtestErr *PackageError var imports, ximports []*Package var stk ImportStack stk.Push(p.ImportPath + " (test)") rawTestImports := str.StringList(p.TestImports) for i, path := range p.TestImports { - p1 := LoadImport(path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], ResolveImport) - if p1.Error != nil { - return nil, nil, nil, p1.Error - } - if len(p1.DepsErrors) > 0 { - err := p1.DepsErrors[0] - err.Pos = "" // show full import stack - return nil, nil, nil, err - } + p1 := loadImport(pre, path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], ResolveImport) if str.Contains(p1.Deps, p.ImportPath) || p1.ImportPath == p.ImportPath { // Same error that loadPackage returns (via reusePackage) in pkg.go. // Can't change that code, because that code is only for loading the // non-test copy of a package. - err := &PackageError{ + ptestErr = &PackageError{ ImportStack: testImportStack(stk[0], p1, p.ImportPath), - Err: "import cycle not allowed in test", + Err: errors.New("import cycle not allowed in test"), IsImportCycle: true, } - return nil, nil, nil, err } p.TestImports[i] = p1.ImportPath imports = append(imports, p1) @@ -86,15 +121,7 @@ func TestPackagesFor(p *Package, cover *TestCover) (pmain, ptest, pxtest *Packag pxtestNeedsPtest := false rawXTestImports := str.StringList(p.XTestImports) for i, path := range p.XTestImports { - p1 := LoadImport(path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], ResolveImport) - if p1.Error != nil { - return nil, nil, nil, p1.Error - } - if len(p1.DepsErrors) > 0 { - err := p1.DepsErrors[0] - err.Pos = "" // show full import stack - return nil, nil, nil, err - } + p1 := loadImport(pre, path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], ResolveImport) if p1.ImportPath == p.ImportPath { pxtestNeedsPtest = true } else { @@ -108,6 +135,7 @@ func TestPackagesFor(p *Package, cover *TestCover) (pmain, ptest, pxtest *Packag if len(p.TestGoFiles) > 0 || p.Name == "main" || cover != nil && cover.Local { ptest = new(Package) *ptest = *p + ptest.Error = ptestErr ptest.ForTest = p.ImportPath ptest.GoFiles = nil ptest.GoFiles = append(ptest.GoFiles, p.GoFiles...) @@ -140,6 +168,7 @@ func TestPackagesFor(p *Package, cover *TestCover) (pmain, ptest, pxtest *Packag m[k] = append(m[k], v...) } ptest.Internal.Build.ImportPos = m + ptest.collectDeps() } else { ptest = p } @@ -152,9 +181,11 @@ func TestPackagesFor(p *Package, cover *TestCover) (pmain, ptest, pxtest *Packag ImportPath: p.ImportPath + "_test", Root: p.Root, Dir: p.Dir, + Goroot: p.Goroot, GoFiles: p.XTestGoFiles, Imports: p.XTestImports, ForTest: p.ImportPath, + Error: pxtestErr, }, Internal: PackageInternal{ LocalPrefix: p.Internal.LocalPrefix, @@ -173,6 +204,7 @@ func TestPackagesFor(p *Package, cover *TestCover) (pmain, ptest, pxtest *Packag if pxtestNeedsPtest { pxtest.Internal.Imports = append(pxtest.Internal.Imports, ptest) } + pxtest.collectDeps() } // Build main package. @@ -206,10 +238,7 @@ func TestPackagesFor(p *Package, cover *TestCover) (pmain, ptest, pxtest *Packag if dep == ptest.ImportPath { pmain.Internal.Imports = append(pmain.Internal.Imports, ptest) } else { - p1 := LoadImport(dep, "", nil, &stk, nil, 0) - if p1.Error != nil { - return nil, nil, nil, p1.Error - } + p1 := loadImport(pre, dep, "", nil, &stk, nil, 0) pmain.Internal.Imports = append(pmain.Internal.Imports, p1) } } @@ -240,8 +269,8 @@ func TestPackagesFor(p *Package, cover *TestCover) (pmain, ptest, pxtest *Packag // The list of imports is used by recompileForTest and by the loop // afterward that gathers t.Cover information. t, err := loadTestFuncs(ptest) - if err != nil { - return nil, nil, nil, err + if err != nil && pmain.Error == nil { + pmain.setLoadPackageDataError(err, p.ImportPath, &stk, nil) } t.Cover = cover if len(ptest.GoFiles)+len(ptest.CgoFiles) > 0 { @@ -254,6 +283,7 @@ func TestPackagesFor(p *Package, cover *TestCover) (pmain, ptest, pxtest *Packag pmain.Imports = append(pmain.Imports, pxtest.ImportPath) t.ImportXtest = true } + pmain.collectDeps() // Sort and dedup pmain.Imports. // Only matters for go list -test output. @@ -290,12 +320,14 @@ func TestPackagesFor(p *Package, cover *TestCover) (pmain, ptest, pxtest *Packag } data, err := formatTestmain(t) - if err != nil { - return nil, nil, nil, err + if err != nil && pmain.Error == nil { + pmain.Error = &PackageError{Err: err} + } + if data != nil { + pmain.Internal.TestmainGo = &data } - pmain.Internal.TestmainGo = &data - return pmain, ptest, pxtest, nil + return pmain, ptest, pxtest } func testImportStack(top string, p *Package, target string) []string { @@ -321,9 +353,10 @@ Search: // preal, packages that import the package under test should get ptest instead // of preal. This is particularly important if pxtest depends on functionality // exposed in test sources in ptest. Second, if there is a main package -// (other than pmain) anywhere, we need to clear p.Internal.BuildInfo in -// the test copy to prevent link conflicts. This may happen if both -coverpkg -// and the command line patterns include multiple main packages. +// (other than pmain) anywhere, we need to set p.Internal.ForceLibrary and +// clear p.Internal.BuildInfo in the test copy to prevent link conflicts. +// This may happen if both -coverpkg and the command line patterns include +// multiple main packages. func recompileForTest(pmain, preal, ptest, pxtest *Package) { // The "test copy" of preal is ptest. // For each package that depends on preal, make a "test copy" @@ -354,6 +387,7 @@ func recompileForTest(pmain, preal, ptest, pxtest *Package) { p = p1 p.Target = "" p.Internal.BuildInfo = "" + p.Internal.ForceLibrary = true } // Update p.Internal.Imports to use test copies. @@ -364,10 +398,13 @@ func recompileForTest(pmain, preal, ptest, pxtest *Package) { } } - // Don't compile build info from a main package. This can happen - // if -coverpkg patterns include main packages, since those packages - // are imported by pmain. See golang.org/issue/30907. - if p.Internal.BuildInfo != "" && p != pmain { + // Force main packages the test imports to be built as libraries. + // Normal imports of main packages are forbidden by the package loader, + // but this can still happen if -coverpkg patterns include main packages: + // covered packages are imported by pmain. Linking multiple packages + // compiled with '-p main' causes duplicate symbol errors. + // See golang.org/issue/30907, golang.org/issue/34114. + if p.Name == "main" && p != pmain && p != ptest { split() } } @@ -419,21 +456,24 @@ type coverInfo struct { } // loadTestFuncs returns the testFuncs describing the tests that will be run. +// The returned testFuncs is always non-nil, even if an error occurred while +// processing test files. func loadTestFuncs(ptest *Package) (*testFuncs, error) { t := &testFuncs{ Package: ptest, } + var err error for _, file := range ptest.TestGoFiles { - if err := t.load(filepath.Join(ptest.Dir, file), "_test", &t.ImportTest, &t.NeedTest); err != nil { - return nil, err + if lerr := t.load(filepath.Join(ptest.Dir, file), "_test", &t.ImportTest, &t.NeedTest); lerr != nil && err == nil { + err = lerr } } for _, file := range ptest.XTestGoFiles { - if err := t.load(filepath.Join(ptest.Dir, file), "_xtest", &t.ImportXtest, &t.NeedXtest); err != nil { - return nil, err + if lerr := t.load(filepath.Join(ptest.Dir, file), "_xtest", &t.ImportXtest, &t.NeedXtest); lerr != nil && err == nil { + err = lerr } } - return t, nil + return t, err } // formatTestmain returns the content of the _testmain.go file for t. @@ -499,7 +539,7 @@ var testFileSet = token.NewFileSet() func (t *testFuncs) load(filename, pkg string, doImport, seen *bool) error { f, err := parser.ParseFile(testFileSet, filename, nil, parser.ParseComments) if err != nil { - return base.ExpandScanner(err) + return err } for _, d := range f.Decls { n, ok := d.(*ast.FuncDecl) @@ -565,12 +605,15 @@ func checkTestFunc(fn *ast.FuncDecl, arg string) error { return nil } -var testmainTmpl = template.Must(template.New("main").Parse(` +var testmainTmpl = lazytemplate.New("main", ` +// Code generated by 'go test'. DO NOT EDIT. + package main import ( -{{if not .TestMain}} "os" +{{if .TestMain}} + "reflect" {{end}} "testing" "testing/internal/testdeps" @@ -661,9 +704,10 @@ func main() { m := testing.MainStart(testdeps.TestDeps{}, tests, benchmarks, examples) {{with .TestMain}} {{.Package}}.{{.Name}}(m) + os.Exit(int(reflect.ValueOf(m).Elem().FieldByName("exitCode").Int())) {{else}} os.Exit(m.Run()) {{end}} } -`)) +`) diff --git a/cmd/go/_internal_/lockedfile/lockedfile.go b/cmd/go/_internal_/lockedfile/lockedfile.go index c940db9..decda06 100644 --- a/cmd/go/_internal_/lockedfile/lockedfile.go +++ b/cmd/go/_internal_/lockedfile/lockedfile.go @@ -120,3 +120,68 @@ func Write(name string, content io.Reader, perm os.FileMode) (err error) { } return err } + +// Transform invokes t with the result of reading the named file, with its lock +// still held. +// +// If t returns a nil error, Transform then writes the returned contents back to +// the file, making a best effort to preserve existing contents on error. +// +// t must not modify the slice passed to it. +func Transform(name string, t func([]byte) ([]byte, error)) (err error) { + f, err := Edit(name) + if err != nil { + return err + } + defer f.Close() + + old, err := ioutil.ReadAll(f) + if err != nil { + return err + } + + new, err := t(old) + if err != nil { + return err + } + + if len(new) > len(old) { + // The overall file size is increasing, so write the tail first: if we're + // about to run out of space on the disk, we would rather detect that + // failure before we have overwritten the original contents. + if _, err := f.WriteAt(new[len(old):], int64(len(old))); err != nil { + // Make a best effort to remove the incomplete tail. + f.Truncate(int64(len(old))) + return err + } + } + + // We're about to overwrite the old contents. In case of failure, make a best + // effort to roll back before we close the file. + defer func() { + if err != nil { + if _, err := f.WriteAt(old, 0); err == nil { + f.Truncate(int64(len(old))) + } + } + }() + + if len(new) >= len(old) { + if _, err := f.WriteAt(new[:len(old)], 0); err != nil { + return err + } + } else { + if _, err := f.WriteAt(new, 0); err != nil { + return err + } + // The overall file size is decreasing, so shrink the file to its final size + // after writing. We do this after writing (instead of before) so that if + // the write fails, enough filesystem space will likely still be reserved + // to contain the previous contents. + if err := f.Truncate(int64(len(new))); err != nil { + return err + } + } + + return nil +} diff --git a/cmd/go/_internal_/lockedfile/mutex.go b/cmd/go/_internal_/lockedfile/mutex.go index e96bc8c..23706f7 100644 --- a/cmd/go/_internal_/lockedfile/mutex.go +++ b/cmd/go/_internal_/lockedfile/mutex.go @@ -7,6 +7,7 @@ package lockedfile import ( "fmt" "os" + "sync" ) // A Mutex provides mutual exclusion within and across processes by locking a @@ -21,7 +22,8 @@ import ( // must not be copied after first use. The Path field must be set before first // use and must not be change thereafter. type Mutex struct { - Path string // The path to the well-known lock file. Must be non-empty. + Path string // The path to the well-known lock file. Must be non-empty. + mu sync.Mutex // A redundant mutex. The race detector doesn't know about file locking, so in tests we may need to lock something that it understands. } // MutexAt returns a new Mutex with Path set to the given non-empty path. @@ -56,5 +58,10 @@ func (mu *Mutex) Lock() (unlock func(), err error) { if err != nil { return nil, err } - return func() { f.Close() }, nil + mu.mu.Lock() + + return func() { + mu.mu.Unlock() + f.Close() + }, nil } diff --git a/cmd/go/_internal_/modconv/convert.go b/cmd/go/_internal_/modconv/convert.go index cea33d3..7e8e36d 100644 --- a/cmd/go/_internal_/modconv/convert.go +++ b/cmd/go/_internal_/modconv/convert.go @@ -13,10 +13,11 @@ import ( "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfile" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/module" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/par" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/semver" + + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" + "golang.org/x/mod/semver" ) // ConvertLegacyConfig converts legacy config to modfile. @@ -41,19 +42,29 @@ func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error { // Convert requirements block, which may use raw SHA1 hashes as versions, // to valid semver requirement list, respecting major versions. - var work par.Work + var ( + work par.Work + mu sync.Mutex + need = make(map[string]string) + replace = make(map[string]*modfile.Replace) + ) + + for _, r := range mf.Replace { + replace[r.New.Path] = r + replace[r.Old.Path] = r + } for _, r := range mf.Require { m := r.Mod if m.Path == "" { continue } + if re, ok := replace[m.Path]; ok { + work.Add(re.New) + continue + } work.Add(r.Mod) } - var ( - mu sync.Mutex - need = make(map[string]string) - ) work.Do(10, func(item interface{}) { r := item.(module.Version) repo, info, err := modfetch.ImportRepoRev(r.Path, r.Version) @@ -76,15 +87,15 @@ func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error { } sort.Strings(paths) for _, path := range paths { + if re, ok := replace[path]; ok { + err := f.AddReplace(re.Old.Path, re.Old.Version, path, need[path]) + if err != nil { + return fmt.Errorf("add replace: %v", err) + } + } f.AddNewRequire(path, need[path], false) } - for _, r := range mf.Replace { - err := f.AddReplace(r.Old.Path, r.Old.Version, r.New.Path, r.New.Version) - if err != nil { - return fmt.Errorf("add replace: %v", err) - } - } f.Cleanup() return nil } diff --git a/cmd/go/_internal_/modconv/dep.go b/cmd/go/_internal_/modconv/dep.go index a361380..d84fea7 100644 --- a/cmd/go/_internal_/modconv/dep.go +++ b/cmd/go/_internal_/modconv/dep.go @@ -6,18 +6,26 @@ package modconv import ( "fmt" + "github.com/dependabot/gomodules-extracted/_internal_/lazyregexp" + "net/url" + "path" "strconv" "strings" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfile" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/module" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/semver" + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" + "golang.org/x/mod/semver" ) func ParseGopkgLock(file string, data []byte) (*modfile.File, error) { + type pkg struct { + Path string + Version string + Source string + } mf := new(modfile.File) - var list []module.Version - var r *module.Version + var list []pkg + var r *pkg for lineno, line := range strings.Split(string(data), "\n") { lineno++ if i := strings.Index(line, "#"); i >= 0 { @@ -25,7 +33,7 @@ func ParseGopkgLock(file string, data []byte) (*modfile.File, error) { } line = strings.TrimSpace(line) if line == "[[projects]]" { - list = append(list, module.Version{}) + list = append(list, pkg{}) r = &list[len(list)-1] continue } @@ -52,6 +60,8 @@ func ParseGopkgLock(file string, data []byte) (*modfile.File, error) { switch key { case "name": r.Path = val + case "source": + r.Source = val case "revision", "version": // Note: key "version" should take priority over "revision", // and it does, because dep writes toml keys in alphabetical order, @@ -68,7 +78,55 @@ func ParseGopkgLock(file string, data []byte) (*modfile.File, error) { if r.Path == "" || r.Version == "" { return nil, fmt.Errorf("%s: empty [[projects]] stanza (%s)", file, r.Path) } - mf.Require = append(mf.Require, &modfile.Require{Mod: r}) + mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: r.Path, Version: r.Version}}) + + if r.Source != "" { + // Convert "source" to import path, such as + // git@test.com:x/y.git and https://test.com/x/y.git. + // We get "test.com/x/y" at last. + source, err := decodeSource(r.Source) + if err != nil { + return nil, err + } + old := module.Version{Path: r.Path, Version: r.Version} + new := module.Version{Path: source, Version: r.Version} + mf.Replace = append(mf.Replace, &modfile.Replace{Old: old, New: new}) + } } return mf, nil } + +var scpSyntaxReg = lazyregexp.New(`^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.*)$`) + +func decodeSource(source string) (string, error) { + var u *url.URL + var p string + if m := scpSyntaxReg.FindStringSubmatch(source); m != nil { + // Match SCP-like syntax and convert it to a URL. + // Eg, "git@github.com:user/repo" becomes + // "ssh://git@github.com/user/repo". + u = &url.URL{ + Scheme: "ssh", + User: url.User(m[1]), + Host: m[2], + Path: "/" + m[3], + } + } else { + var err error + u, err = url.Parse(source) + if err != nil { + return "", fmt.Errorf("%q is not a valid URI", source) + } + } + + // If no scheme was passed, then the entire path will have been put into + // u.Path. Either way, construct the normalized path correctly. + if u.Host == "" { + p = source + } else { + p = path.Join(u.Host, u.Path) + } + p = strings.TrimSuffix(p, ".git") + p = strings.TrimSuffix(p, ".hg") + return p, nil +} diff --git a/cmd/go/_internal_/modconv/glide.go b/cmd/go/_internal_/modconv/glide.go index 8eb268b..d1de3f7 100644 --- a/cmd/go/_internal_/modconv/glide.go +++ b/cmd/go/_internal_/modconv/glide.go @@ -7,16 +7,15 @@ package modconv import ( "strings" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfile" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/module" + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" ) func ParseGlideLock(file string, data []byte) (*modfile.File, error) { mf := new(modfile.File) imports := false name := "" - for lineno, line := range strings.Split(string(data), "\n") { - lineno++ + for _, line := range strings.Split(string(data), "\n") { if line == "" { continue } diff --git a/cmd/go/_internal_/modconv/glock.go b/cmd/go/_internal_/modconv/glock.go index 4c62890..b8dc204 100644 --- a/cmd/go/_internal_/modconv/glock.go +++ b/cmd/go/_internal_/modconv/glock.go @@ -7,14 +7,13 @@ package modconv import ( "strings" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfile" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/module" + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" ) func ParseGLOCKFILE(file string, data []byte) (*modfile.File, error) { mf := new(modfile.File) - for lineno, line := range strings.Split(string(data), "\n") { - lineno++ + for _, line := range strings.Split(string(data), "\n") { f := strings.Fields(line) if len(f) >= 2 && f[0] != "cmd" { mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: f[0], Version: f[1]}}) diff --git a/cmd/go/_internal_/modconv/godeps.go b/cmd/go/_internal_/modconv/godeps.go index f3c9b7f..5900d53 100644 --- a/cmd/go/_internal_/modconv/godeps.go +++ b/cmd/go/_internal_/modconv/godeps.go @@ -7,8 +7,8 @@ package modconv import ( "encoding/json" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfile" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/module" + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" ) func ParseGodepsJSON(file string, data []byte) (*modfile.File, error) { diff --git a/cmd/go/_internal_/modconv/modconv.go b/cmd/go/_internal_/modconv/modconv.go index 677e054..22e3fe2 100644 --- a/cmd/go/_internal_/modconv/modconv.go +++ b/cmd/go/_internal_/modconv/modconv.go @@ -4,7 +4,7 @@ package modconv -import "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfile" +import "golang.org/x/mod/modfile" var Converters = map[string]func(string, []byte) (*modfile.File, error){ "GLOCKFILE": ParseGLOCKFILE, diff --git a/cmd/go/_internal_/modconv/tsv.go b/cmd/go/_internal_/modconv/tsv.go index 97cbf1a..4649579 100644 --- a/cmd/go/_internal_/modconv/tsv.go +++ b/cmd/go/_internal_/modconv/tsv.go @@ -7,14 +7,13 @@ package modconv import ( "strings" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfile" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/module" + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" ) func ParseDependenciesTSV(file string, data []byte) (*modfile.File, error) { mf := new(modfile.File) - for lineno, line := range strings.Split(string(data), "\n") { - lineno++ + for _, line := range strings.Split(string(data), "\n") { f := strings.Split(line, "\t") if len(f) >= 3 { mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: f[0], Version: f[2]}}) diff --git a/cmd/go/_internal_/modconv/vconf.go b/cmd/go/_internal_/modconv/vconf.go index 6a0c09f..9bad2ba 100644 --- a/cmd/go/_internal_/modconv/vconf.go +++ b/cmd/go/_internal_/modconv/vconf.go @@ -7,14 +7,13 @@ package modconv import ( "strings" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfile" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/module" + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" ) func ParseVendorConf(file string, data []byte) (*modfile.File, error) { mf := new(modfile.File) - for lineno, line := range strings.Split(string(data), "\n") { - lineno++ + for _, line := range strings.Split(string(data), "\n") { if i := strings.Index(line, "#"); i >= 0 { line = line[:i] } diff --git a/cmd/go/_internal_/modconv/vjson.go b/cmd/go/_internal_/modconv/vjson.go index 2c85d65..851e209 100644 --- a/cmd/go/_internal_/modconv/vjson.go +++ b/cmd/go/_internal_/modconv/vjson.go @@ -7,8 +7,8 @@ package modconv import ( "encoding/json" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfile" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/module" + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" ) func ParseVendorJSON(file string, data []byte) (*modfile.File, error) { diff --git a/cmd/go/_internal_/modconv/vmanifest.go b/cmd/go/_internal_/modconv/vmanifest.go index 33f786d..7f5931e 100644 --- a/cmd/go/_internal_/modconv/vmanifest.go +++ b/cmd/go/_internal_/modconv/vmanifest.go @@ -7,8 +7,8 @@ package modconv import ( "encoding/json" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfile" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/module" + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" ) func ParseVendorManifest(file string, data []byte) (*modfile.File, error) { diff --git a/cmd/go/_internal_/modconv/vyml.go b/cmd/go/_internal_/modconv/vyml.go index b7d3740..cfa4194 100644 --- a/cmd/go/_internal_/modconv/vyml.go +++ b/cmd/go/_internal_/modconv/vyml.go @@ -7,16 +7,15 @@ package modconv import ( "strings" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfile" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/module" + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" ) func ParseVendorYML(file string, data []byte) (*modfile.File, error) { mf := new(modfile.File) vendors := false path := "" - for lineno, line := range strings.Split(string(data), "\n") { - lineno++ + for _, line := range strings.Split(string(data), "\n") { if line == "" { continue } diff --git a/cmd/go/_internal_/modfetch/cache.go b/cmd/go/_internal_/modfetch/cache.go index e82152c..833ad24 100644 --- a/cmd/go/_internal_/modfetch/cache.go +++ b/cmd/go/_internal_/modfetch/cache.go @@ -7,6 +7,7 @@ package modfetch import ( "bytes" "encoding/json" + "errors" "fmt" "io" "io/ioutil" @@ -15,27 +16,27 @@ import ( "strings" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/lockedfile" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch/codehost" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/module" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/par" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/renameio" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/semver" -) - -var QuietLookup bool // do not print about lookups -var PkgMod string // $GOPATH/pkg/mod; set by package modload + "golang.org/x/mod/module" + "golang.org/x/mod/semver" +) func cacheDir(path string) (string, error) { - if PkgMod == "" { - return "", fmt.Errorf("internal error: modfetch.PkgMod not set") + if cfg.GOMODCACHE == "" { + // modload.Init exits if GOPATH[0] is empty, and cfg.GOMODCACHE + // is set to GOPATH[0]/pkg/mod if GOMODCACHE is empty, so this should never happen. + return "", fmt.Errorf("internal error: cfg.GOMODCACHE not set") } - enc, err := module.EncodePath(path) + enc, err := module.EscapePath(path) if err != nil { return "", err } - return filepath.Join(PkgMod, "cache/download", enc, "/@v"), nil + return filepath.Join(cfg.GOMODCACHE, "cache/download", enc, "/@v"), nil } func CachePath(m module.Version, suffix string) (string, error) { @@ -49,20 +50,25 @@ func CachePath(m module.Version, suffix string) (string, error) { if module.CanonicalVersion(m.Version) != m.Version { return "", fmt.Errorf("non-canonical module version %q", m.Version) } - encVer, err := module.EncodeVersion(m.Version) + encVer, err := module.EscapeVersion(m.Version) if err != nil { return "", err } return filepath.Join(dir, encVer+"."+suffix), nil } -// DownloadDir returns the directory to which m should be downloaded. -// Note that the directory may not yet exist. +// DownloadDir returns the directory to which m should have been downloaded. +// An error will be returned if the module path or version cannot be escaped. +// An error satisfying errors.Is(err, os.ErrNotExist) will be returned +// along with the directory if the directory does not exist or if the directory +// is not completely populated. func DownloadDir(m module.Version) (string, error) { - if PkgMod == "" { - return "", fmt.Errorf("internal error: modfetch.PkgMod not set") + if cfg.GOMODCACHE == "" { + // modload.Init exits if GOPATH[0] is empty, and cfg.GOMODCACHE + // is set to GOPATH[0]/pkg/mod if GOMODCACHE is empty, so this should never happen. + return "", fmt.Errorf("internal error: cfg.GOMODCACHE not set") } - enc, err := module.EncodePath(m.Path) + enc, err := module.EscapePath(m.Path) if err != nil { return "", err } @@ -72,13 +78,43 @@ func DownloadDir(m module.Version) (string, error) { if module.CanonicalVersion(m.Version) != m.Version { return "", fmt.Errorf("non-canonical module version %q", m.Version) } - encVer, err := module.EncodeVersion(m.Version) + encVer, err := module.EscapeVersion(m.Version) if err != nil { return "", err } - return filepath.Join(PkgMod, enc+"@"+encVer), nil + + dir := filepath.Join(cfg.GOMODCACHE, enc+"@"+encVer) + if fi, err := os.Stat(dir); os.IsNotExist(err) { + return dir, err + } else if err != nil { + return dir, &DownloadDirPartialError{dir, err} + } else if !fi.IsDir() { + return dir, &DownloadDirPartialError{dir, errors.New("not a directory")} + } + partialPath, err := CachePath(m, "partial") + if err != nil { + return dir, err + } + if _, err := os.Stat(partialPath); err == nil { + return dir, &DownloadDirPartialError{dir, errors.New("not completely extracted")} + } else if !os.IsNotExist(err) { + return dir, err + } + return dir, nil } +// DownloadDirPartialError is returned by DownloadDir if a module directory +// exists but was not completely populated. +// +// DownloadDirPartialError is equivalent to os.ErrNotExist. +type DownloadDirPartialError struct { + Dir string + Err error +} + +func (e *DownloadDirPartialError) Error() string { return fmt.Sprintf("%s: %v", e.Dir, e.Err) } +func (e *DownloadDirPartialError) Is(err error) bool { return err == os.ErrNotExist } + // lockVersion locks a file within the module cache that guards the downloading // and extraction of the zipfile for the given module version. func lockVersion(mod module.Version) (unlock func(), err error) { @@ -92,22 +128,23 @@ func lockVersion(mod module.Version) (unlock func(), err error) { return lockedfile.MutexAt(path).Lock() } -// SideLock locks a file within the module cache that that guards edits to files -// outside the cache, such as go.sum and go.mod files in the user's working -// directory. It returns a function that must be called to unlock the file. -func SideLock() (unlock func()) { - if PkgMod == "" { - base.Fatalf("go: internal error: modfetch.PkgMod not set") +// SideLock locks a file within the module cache that that previously guarded +// edits to files outside the cache, such as go.sum and go.mod files in the +// user's working directory. +// If err is nil, the caller MUST eventually call the unlock function. +func SideLock() (unlock func(), err error) { + if cfg.GOMODCACHE == "" { + // modload.Init exits if GOPATH[0] is empty, and cfg.GOMODCACHE + // is set to GOPATH[0]/pkg/mod if GOMODCACHE is empty, so this should never happen. + base.Fatalf("go: internal error: cfg.GOMODCACHE not set") } - path := filepath.Join(PkgMod, "cache", "lock") + + path := filepath.Join(cfg.GOMODCACHE, "cache", "lock") if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil { - base.Fatalf("go: failed to create cache directory %s: %v", filepath.Dir(path), err) - } - unlock, err := lockedfile.MutexAt(path).Lock() - if err != nil { - base.Fatalf("go: failed to lock file at %v", path) + return nil, fmt.Errorf("failed to create cache directory: %w", err) } - return unlock + + return lockedfile.MutexAt(path).Lock() } // A cachingRepo is a cache around an underlying Repo, @@ -160,9 +197,6 @@ func (r *cachingRepo) Stat(rev string) (*RevInfo, error) { return cachedInfo{info, nil} } - if !QuietLookup { - fmt.Fprintf(os.Stderr, "go: finding %s %s\n", r.path, rev) - } info, err = r.r.Stat(rev) if err == nil { // If we resolved, say, 1234abcde to v0.0.0-20180604122334-1234abcdef78, @@ -190,9 +224,6 @@ func (r *cachingRepo) Stat(rev string) (*RevInfo, error) { func (r *cachingRepo) Latest() (*RevInfo, error) { c := r.cache.Do("latest:", func() interface{} { - if !QuietLookup { - fmt.Fprintf(os.Stderr, "go: finding %s latest\n", r.path) - } info, err := r.r.Latest() // Save info for likely future Stat call. @@ -215,29 +246,23 @@ func (r *cachingRepo) Latest() (*RevInfo, error) { return &info, nil } -func (r *cachingRepo) GoMod(rev string) ([]byte, error) { +func (r *cachingRepo) GoMod(version string) ([]byte, error) { type cached struct { text []byte err error } - c := r.cache.Do("gomod:"+rev, func() interface{} { - file, text, err := readDiskGoMod(r.path, rev) + c := r.cache.Do("gomod:"+version, func() interface{} { + file, text, err := readDiskGoMod(r.path, version) if err == nil { // Note: readDiskGoMod already called checkGoMod. return cached{text, nil} } - // Convert rev to canonical version - // so that we use the right identifier in the go.sum check. - info, err := r.Stat(rev) - if err != nil { - return cached{nil, err} - } - rev = info.Version - - text, err = r.r.GoMod(rev) + text, err = r.r.GoMod(version) if err == nil { - checkGoMod(r.path, rev, text) + if err := checkGoMod(r.path, version, text); err != nil { + return cached{text, err} + } if err := writeDiskGoMod(file, text); err != nil { fmt.Fprintf(os.Stderr, "go: writing go.mod cache: %v\n", err) } @@ -258,12 +283,12 @@ func (r *cachingRepo) Zip(dst io.Writer, version string) error { // Stat is like Lookup(path).Stat(rev) but avoids the // repository path resolution in Lookup if the result is // already cached on local disk. -func Stat(path, rev string) (*RevInfo, error) { +func Stat(proxy, path, rev string) (*RevInfo, error) { _, info, err := readDiskStat(path, rev) if err == nil { return info, nil } - repo, err := Lookup(path) + repo, err := Lookup(proxy, path) if err != nil { return nil, err } @@ -276,9 +301,22 @@ func InfoFile(path, version string) (string, error) { if !semver.IsValid(version) { return "", fmt.Errorf("invalid version %q", version) } - if _, err := Stat(path, version); err != nil { + + if file, _, err := readDiskStat(path, version); err == nil { + return file, nil + } + + err := TryProxies(func(proxy string) error { + repo, err := Lookup(proxy, path) + if err == nil { + _, err = repo.Stat(version) + } + return err + }) + if err != nil { return "", err } + // Stat should have populated the disk cache for us. file, _, err := readDiskStat(path, version) if err != nil { @@ -294,21 +332,39 @@ func GoMod(path, rev string) ([]byte, error) { // Convert commit hash to pseudo-version // to increase cache hit rate. if !semver.IsValid(rev) { - info, err := Stat(path, rev) - if err != nil { - return nil, err + if _, info, err := readDiskStat(path, rev); err == nil { + rev = info.Version + } else { + err := TryProxies(func(proxy string) error { + repo, err := Lookup(proxy, path) + if err != nil { + return err + } + info, err := repo.Stat(rev) + if err == nil { + rev = info.Version + } + return err + }) + if err != nil { + return nil, err + } } - rev = info.Version } + _, data, err := readDiskGoMod(path, rev) if err == nil { return data, nil } - repo, err := Lookup(path) - if err != nil { - return nil, err - } - return repo.GoMod(rev) + + err = TryProxies(func(proxy string) error { + repo, err := Lookup(proxy, path) + if err == nil { + data, err = repo.GoMod(rev) + } + return err + }) + return data, err } // GoModFile is like GoMod but returns the name of the file containing @@ -354,8 +410,29 @@ var errNotCached = fmt.Errorf("not in cache") func readDiskStat(path, rev string) (file string, info *RevInfo, err error) { file, data, err := readDiskCache(path, rev, "info") if err != nil { - if file, info, err := readDiskStatByHash(path, rev); err == nil { - return file, info, nil + // If the cache already contains a pseudo-version with the given hash, we + // would previously return that pseudo-version without checking upstream. + // However, that produced an unfortunate side-effect: if the author added a + // tag to the repository, 'go get' would not pick up the effect of that new + // tag on the existing commits, and 'go' commands that referred to those + // commits would use the previous name instead of the new one. + // + // That's especially problematic if the original pseudo-version starts with + // v0.0.0-, as was the case for all pseudo-versions during vgo development, + // since a v0.0.0- pseudo-version has lower precedence than pretty much any + // tagged version. + // + // In practice, we're only looking up by hash during initial conversion of a + // legacy config and during an explicit 'go get', and a little extra latency + // for those operations seems worth the benefit of picking up more accurate + // versions. + // + // Fall back to this resolution scheme only if the GOPROXY setting prohibits + // us from resolving upstream tags. + if cfg.GOPROXY == "off" { + if file, info, err := readDiskStatByHash(path, rev); err == nil { + return file, info, nil + } } return file, nil, err } @@ -383,7 +460,7 @@ func readDiskStat(path, rev string) (file string, info *RevInfo, err error) { // just to find out about a commit we already know about // (and have cached under its pseudo-version). func readDiskStatByHash(path, rev string) (file string, info *RevInfo, err error) { - if PkgMod == "" { + if cfg.GOMODCACHE == "" { // Do not download to current directory. return "", nil, errNotCached } @@ -405,13 +482,23 @@ func readDiskStatByHash(path, rev string) (file string, info *RevInfo, err error if err != nil { return "", nil, errNotCached } + + // A given commit hash may map to more than one pseudo-version, + // depending on which tags are present on the repository. + // Take the highest such version. + var maxVersion string suffix := "-" + rev + ".info" + err = errNotCached for _, name := range names { - if strings.HasSuffix(name, suffix) && IsPseudoVersion(strings.TrimSuffix(name, ".info")) { - return readDiskStat(path, strings.TrimSuffix(name, ".info")) + if strings.HasSuffix(name, suffix) { + v := strings.TrimSuffix(name, ".info") + if IsPseudoVersion(v) && semver.Max(maxVersion, v) == v { + maxVersion = v + file, info, err = readDiskStat(path, strings.TrimSuffix(name, ".info")) + } } } - return "", nil, errNotCached + return file, info, err } // oldVgoPrefix is the prefix in the old auto-generated cached go.mod files. @@ -435,7 +522,9 @@ func readDiskGoMod(path, rev string) (file string, data []byte, err error) { } if err == nil { - checkGoMod(path, rev, data) + if err := checkGoMod(path, rev, data); err != nil { + return "", nil, err + } } return file, data, err @@ -451,7 +540,7 @@ func readDiskCache(path, rev, suffix string) (file string, data []byte, err erro if err != nil { return "", nil, errNotCached } - data, err = ioutil.ReadFile(file) + data, err = renameio.ReadFile(file) if err != nil { return file, nil, errNotCached } @@ -488,7 +577,7 @@ func writeDiskCache(file string, data []byte) error { return err } - if err := renameio.WriteFile(file, data); err != nil { + if err := renameio.WriteFile(file, data, 0666); err != nil { return err } @@ -545,12 +634,12 @@ func rewriteVersionList(dir string) { buf.WriteString(v) buf.WriteString("\n") } - old, _ := ioutil.ReadFile(listFile) + old, _ := renameio.ReadFile(listFile) if bytes.Equal(buf.Bytes(), old) { return } - if err := renameio.WriteFile(listFile, buf.Bytes()); err != nil { + if err := renameio.WriteFile(listFile, buf.Bytes(), 0666); err != nil { base.Fatalf("go: failed to write version list: %v", err) } } diff --git a/cmd/go/_internal_/modfetch/codehost/codehost.go b/cmd/go/_internal_/modfetch/codehost/codehost.go index a1307b3..40fd7f1 100644 --- a/cmd/go/_internal_/modfetch/codehost/codehost.go +++ b/cmd/go/_internal_/modfetch/codehost/codehost.go @@ -73,20 +73,21 @@ type Repo interface { // ReadZip downloads a zip file for the subdir subdirectory // of the given revision to a new file in a given temporary directory. // It should refuse to read more than maxSize bytes. - // It returns a ReadCloser for a streamed copy of the zip file, - // along with the actual subdirectory (possibly shorter than subdir) - // contained in the zip file. All files in the zip file are expected to be + // It returns a ReadCloser for a streamed copy of the zip file. + // All files in the zip file are expected to be // nested in a single top-level directory, whose name is not specified. - ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, actualSubdir string, err error) - - // RecentTag returns the most recent tag at or before the given rev - // with the given prefix. It should make a best-effort attempt to - // find a tag that is a valid semantic version (following the prefix), - // or else the result is not useful to the caller, but it need not - // incur great expense in doing so. For example, the git implementation - // of RecentTag limits git's search to tags matching the glob expression - // "v[0-9]*.[0-9]*.[0-9]*" (after the prefix). - RecentTag(rev, prefix string) (tag string, err error) + ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, err error) + + // RecentTag returns the most recent tag on rev or one of its predecessors + // with the given prefix and major version. + // An empty major string matches any major version. + RecentTag(rev, prefix, major string) (tag string, err error) + + // DescendsFrom reports whether rev or any of its ancestors has the given tag. + // + // DescendsFrom must return true for any tag returned by RecentTag for the + // same revision. + DescendsFrom(rev, tag string) (bool, error) } // A Rev describes a single revision in a source code repository. @@ -105,6 +106,32 @@ type FileRev struct { Err error // error if any; os.IsNotExist(Err)==true if rev exists but file does not exist in that rev } +// UnknownRevisionError is an error equivalent to os.ErrNotExist, but for a +// revision rather than a file. +type UnknownRevisionError struct { + Rev string +} + +func (e *UnknownRevisionError) Error() string { + return "unknown revision " + e.Rev +} +func (UnknownRevisionError) Is(err error) bool { + return err == os.ErrNotExist +} + +// ErrNoCommits is an error equivalent to os.ErrNotExist indicating that a given +// repository or module contains no commits. +var ErrNoCommits error = noCommitsError{} + +type noCommitsError struct{} + +func (noCommitsError) Error() string { + return "no commits" +} +func (noCommitsError) Is(err error) bool { + return err == os.ErrNotExist +} + // AllHex reports whether the revision rev is entirely lower-case hexadecimal digits. func AllHex(rev string) bool { for i := 0; i < len(rev); i++ { @@ -126,15 +153,11 @@ func ShortenSHA1(rev string) string { return rev } -// WorkRoot is the root of the cached work directory. -// It is set by cmd/go/internal/modload.InitMod. -var WorkRoot string - // WorkDir returns the name of the cached work directory to use for the // given repository type and name. func WorkDir(typ, name string) (dir, lockfile string, err error) { - if WorkRoot == "" { - return "", "", fmt.Errorf("codehost.WorkRoot not set") + if cfg.GOMODCACHE == "" { + return "", "", fmt.Errorf("neither GOPATH nor GOMODCACHE are set") } // We name the work directory for the SHA256 hash of the type and name. @@ -146,7 +169,7 @@ func WorkDir(typ, name string) (dir, lockfile string, err error) { return "", "", fmt.Errorf("codehost.WorkDir: type cannot contain colon") } key := typ + ":" + name - dir = filepath.Join(WorkRoot, fmt.Sprintf("%x", sha256.Sum256([]byte(key)))) + dir = filepath.Join(cfg.GOMODCACHE, "cache/vcs", fmt.Sprintf("%x", sha256.Sum256([]byte(key)))) if cfg.BuildX { fmt.Fprintf(os.Stderr, "mkdir -p %s # %s %s\n", filepath.Dir(dir), typ, name) diff --git a/cmd/go/_internal_/modfetch/codehost/git.go b/cmd/go/_internal_/modfetch/codehost/git.go index aef4ff9..993c170 100644 --- a/cmd/go/_internal_/modfetch/codehost/git.go +++ b/cmd/go/_internal_/modfetch/codehost/git.go @@ -6,10 +6,13 @@ package codehost import ( "bytes" + "errors" "fmt" "io" "io/ioutil" + "net/url" "os" + "os/exec" "path/filepath" "sort" "strconv" @@ -19,12 +22,10 @@ import ( "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/lockedfile" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/par" -) + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/web" -// GitRepo returns the code repository at the given Git remote reference. -func GitRepo(remote string) (Repo, error) { - return newGitRepoCached(remote, false) -} + "golang.org/x/mod/semver" +) // LocalGitRepo is like Repo but accepts both Git remote references // and paths to repositories on the local file system. @@ -32,7 +33,16 @@ func LocalGitRepo(remote string) (Repo, error) { return newGitRepoCached(remote, true) } -const gitWorkDirType = "git2" +// A notExistError wraps another error to retain its original text +// but makes it opaquely equivalent to os.ErrNotExist. +type notExistError struct { + err error +} + +func (e notExistError) Error() string { return e.err.Error() } +func (notExistError) Is(err error) bool { return err == os.ErrNotExist } + +const gitWorkDirType = "git3" var gitRepoCache par.Cache @@ -79,12 +89,13 @@ func newGitRepo(remote string, localOK bool) (Repo, error) { // but this lets us say git fetch origin instead, which // is a little nicer. More importantly, using a named remote // avoids a problem with Git LFS. See golang.org/issue/25605. - if _, err := Run(r.dir, "git", "remote", "add", "origin", r.remote); err != nil { + if _, err := Run(r.dir, "git", "remote", "add", "origin", "--", r.remote); err != nil { os.RemoveAll(r.dir) return nil, err } - r.remote = "origin" } + r.remoteURL = r.remote + r.remote = "origin" } else { // Local path. // Disallow colon (not in ://) because sometimes @@ -111,9 +122,9 @@ func newGitRepo(remote string, localOK bool) (Repo, error) { } type gitRepo struct { - remote string - local bool - dir string + remote, remoteURL string + local bool + dir string mu lockedfile.Mutex // protects fetchLevel and git repo state @@ -122,8 +133,10 @@ type gitRepo struct { statCache par.Cache refsOnce sync.Once - refs map[string]string - refsErr error + // refs maps branch and tag refs (e.g., "HEAD", "refs/heads/master") + // to commits (e.g., "37ffd2e798afde829a34e8955b716ab730b2a6d6") + refs map[string]string + refsErr error localTagsOnce sync.Once localTags map[string]bool @@ -162,14 +175,25 @@ func (r *gitRepo) loadRefs() { // The git protocol sends all known refs and ls-remote filters them on the client side, // so we might as well record both heads and tags in one shot. // Most of the time we only care about tags but sometimes we care about heads too. - out, err := Run(r.dir, "git", "ls-remote", "-q", r.remote) - if err != nil { - if rerr, ok := err.(*RunError); ok { + out, gitErr := Run(r.dir, "git", "ls-remote", "-q", r.remote) + if gitErr != nil { + if rerr, ok := gitErr.(*RunError); ok { if bytes.Contains(rerr.Stderr, []byte("fatal: could not read Username")) { - rerr.HelpText = "If this is a private repository, see https://golang.org/doc/faq#git_https for additional information." + rerr.HelpText = "Confirm the import path was entered correctly.\nIf this is a private repository, see https://golang.org/doc/faq#git_https for additional information." + } + } + + // If the remote URL doesn't exist at all, ideally we should treat the whole + // repository as nonexistent by wrapping the error in a notExistError. + // For HTTP and HTTPS, that's easy to detect: we'll try to fetch the URL + // ourselves and see what code it serves. + if u, err := url.Parse(r.remoteURL); err == nil && (u.Scheme == "http" || u.Scheme == "https") { + if _, err := web.GetBytes(u); errors.Is(err, os.ErrNotExist) { + gitErr = notExistError{gitErr} } } - r.refsErr = err + + r.refsErr = gitErr return } @@ -218,7 +242,7 @@ func (r *gitRepo) Latest() (*RevInfo, error) { return nil, r.refsErr } if r.refs["HEAD"] == "" { - return nil, fmt.Errorf("no commits") + return nil, ErrNoCommits } return r.Stat(r.refs["HEAD"]) } @@ -237,13 +261,6 @@ func (r *gitRepo) findRef(hash string) (ref string, ok bool) { return "", false } -func unshallow(gitDir string) []string { - if _, err := os.Stat(filepath.Join(gitDir, "shallow")); err == nil { - return []string{"--unshallow"} - } - return []string{} -} - // minHashDigits is the minimum number of digits to require // before accepting a hex digit sequence as potentially identifying // a specific commit in a git repo. (Of course, users can always @@ -315,7 +332,7 @@ func (r *gitRepo) stat(rev string) (*RevInfo, error) { hash = rev } } else { - return nil, fmt.Errorf("unknown revision %s", rev) + return nil, &UnknownRevisionError{Rev: rev} } // Protect r.fetchLevel and the "fetch more and more" sequence. @@ -339,8 +356,14 @@ func (r *gitRepo) stat(rev string) (*RevInfo, error) { } } - // If we know a specific commit we need, fetch it. - if r.fetchLevel <= fetchSome && hash != "" && !r.local { + // If we know a specific commit we need and its ref, fetch it. + // We do NOT fetch arbitrary hashes (when we don't know the ref) + // because we want to avoid ever importing a commit that isn't + // reachable from refs/tags/* or refs/heads/* or HEAD. + // Both Gerrit and GitHub expose every CL/PR as a named ref, + // and we don't want those commits masquerading as being real + // pseudo-versions in the main repo. + if r.fetchLevel <= fetchSome && ref != "" && hash != "" && !r.local { r.fetchLevel = fetchSome var refspec string if ref != "" && ref != "HEAD" { @@ -369,40 +392,51 @@ func (r *gitRepo) stat(rev string) (*RevInfo, error) { // Last resort. // Fetch all heads and tags and hope the hash we want is in the history. - if r.fetchLevel < fetchAll { - // TODO(bcmills): should we wait to upgrade fetchLevel until after we check - // err? If there is a temporary server error, we want subsequent fetches to - // try again instead of proceeding with an incomplete repo. - r.fetchLevel = fetchAll - if err := r.fetchUnshallow("refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"); err != nil { - return nil, err - } + if err := r.fetchRefsLocked(); err != nil { + return nil, err } return r.statLocal(rev, rev) } -func (r *gitRepo) fetchUnshallow(refSpecs ...string) error { - // To work around a protocol version 2 bug that breaks --unshallow, - // add -c protocol.version=0. - // TODO(rsc): The bug is believed to be server-side, meaning only - // on Google's Git servers. Once the servers are fixed, drop the - // protocol.version=0. See Google-internal bug b/110495752. - var protoFlag []string - unshallowFlag := unshallow(r.dir) - if len(unshallowFlag) > 0 { - protoFlag = []string{"-c", "protocol.version=0"} - } - _, err := Run(r.dir, "git", protoFlag, "fetch", unshallowFlag, "-f", r.remote, refSpecs) - return err +// fetchRefsLocked fetches all heads and tags from the origin, along with the +// ancestors of those commits. +// +// We only fetch heads and tags, not arbitrary other commits: we don't want to +// pull in off-branch commits (such as rejected GitHub pull requests) that the +// server may be willing to provide. (See the comments within the stat method +// for more detail.) +// +// fetchRefsLocked requires that r.mu remain locked for the duration of the call. +func (r *gitRepo) fetchRefsLocked() error { + if r.fetchLevel < fetchAll { + // NOTE: To work around a bug affecting Git clients up to at least 2.23.0 + // (2019-08-16), we must first expand the set of local refs, and only then + // unshallow the repository as a separate fetch operation. (See + // golang.org/issue/34266 and + // https://github.com/git/git/blob/4c86140027f4a0d2caaa3ab4bd8bfc5ce3c11c8a/transport.c#L1303-L1309.) + + if _, err := Run(r.dir, "git", "fetch", "-f", r.remote, "refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"); err != nil { + return err + } + + if _, err := os.Stat(filepath.Join(r.dir, "shallow")); err == nil { + if _, err := Run(r.dir, "git", "fetch", "--unshallow", "-f", r.remote); err != nil { + return err + } + } + + r.fetchLevel = fetchAll + } + return nil } // statLocal returns a RevInfo describing rev in the local git repository. // It uses version as info.Version. func (r *gitRepo) statLocal(version, rev string) (*RevInfo, error) { - out, err := Run(r.dir, "git", "-c", "log.showsignature=false", "log", "-n1", "--format=format:%H %ct %D", rev) + out, err := Run(r.dir, "git", "-c", "log.showsignature=false", "log", "-n1", "--format=format:%H %ct %D", rev, "--") if err != nil { - return nil, fmt.Errorf("unknown revision %s", rev) + return nil, &UnknownRevisionError{Rev: rev} } f := strings.Fields(string(out)) if len(f) < 2 { @@ -516,39 +550,10 @@ func (r *gitRepo) ReadFileRevs(revs []string, file string, maxSize int64) (map[s } defer unlock() - var refs []string - var protoFlag []string - var unshallowFlag []string - for _, tag := range redo { - refs = append(refs, "refs/tags/"+tag+":refs/tags/"+tag) - } - if len(refs) > 1 { - unshallowFlag = unshallow(r.dir) - if len(unshallowFlag) > 0 { - // To work around a protocol version 2 bug that breaks --unshallow, - // add -c protocol.version=0. - // TODO(rsc): The bug is believed to be server-side, meaning only - // on Google's Git servers. Once the servers are fixed, drop the - // protocol.version=0. See Google-internal bug b/110495752. - protoFlag = []string{"-c", "protocol.version=0"} - } - } - if _, err := Run(r.dir, "git", protoFlag, "fetch", unshallowFlag, "-f", r.remote, refs); err != nil { + if err := r.fetchRefsLocked(); err != nil { return nil, err } - // TODO(bcmills): after the 1.11 freeze, replace the block above with: - // if r.fetchLevel <= fetchSome { - // r.fetchLevel = fetchSome - // var refs []string - // for _, tag := range redo { - // refs = append(refs, "refs/tags/"+tag+":refs/tags/"+tag) - // } - // if _, err := Run(r.dir, "git", "fetch", "--update-shallow", "-f", r.remote, refs); err != nil { - // return nil, err - // } - // } - if _, err := r.readFileRevs(redo, file, files); err != nil { return nil, err } @@ -639,23 +644,51 @@ func (r *gitRepo) readFileRevs(tags []string, file string, fileMap map[string]*F return missing, nil } -func (r *gitRepo) RecentTag(rev, prefix string) (tag string, err error) { +func (r *gitRepo) RecentTag(rev, prefix, major string) (tag string, err error) { info, err := r.Stat(rev) if err != nil { return "", err } rev = info.Name // expand hash prefixes - // describe sets tag and err using 'git describe' and reports whether the + // describe sets tag and err using 'git for-each-ref' and reports whether the // result is definitive. describe := func() (definitive bool) { var out []byte - out, err = Run(r.dir, "git", "describe", "--first-parent", "--always", "--abbrev=0", "--match", prefix+"v[0-9]*.[0-9]*.[0-9]*", "--tags", rev) + out, err = Run(r.dir, "git", "for-each-ref", "--format", "%(refname)", "refs/tags", "--merged", rev) if err != nil { - return true // Because we use "--always", describe should never fail. + return true + } + + // prefixed tags aren't valid semver tags so compare without prefix, but only tags with correct prefix + var highest string + for _, line := range strings.Split(string(out), "\n") { + line = strings.TrimSpace(line) + // git do support lstrip in for-each-ref format, but it was added in v2.13.0. Stripping here + // instead gives support for git v2.7.0. + if !strings.HasPrefix(line, "refs/tags/") { + continue + } + line = line[len("refs/tags/"):] + + if !strings.HasPrefix(line, prefix) { + continue + } + + semtag := line[len(prefix):] + // Consider only tags that are valid and complete (not just major.minor prefixes). + // NOTE: Do not replace the call to semver.Compare with semver.Max. + // We want to return the actual tag, not a canonicalized version of it, + // and semver.Max currently canonicalizes (see golang.org/issue/32700). + if c := semver.Canonical(semtag); c != "" && strings.HasPrefix(semtag, c) && (major == "" || semver.Major(c) == major) && semver.Compare(semtag, highest) > 0 { + highest = semtag + } + } + + if highest != "" { + tag = prefix + highest } - tag = string(bytes.TrimSpace(out)) return tag != "" && !AllHex(tag) } @@ -682,12 +715,8 @@ func (r *gitRepo) RecentTag(rev, prefix string) (tag string, err error) { } defer unlock() - if r.fetchLevel < fetchAll { - // Fetch all heads and tags and see if that gives us enough history. - if err := r.fetchUnshallow("refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"); err != nil { - return "", err - } - r.fetchLevel = fetchAll + if err := r.fetchRefsLocked(); err != nil { + return "", err } // If we've reached this point, we have all of the commits that are reachable @@ -704,7 +733,68 @@ func (r *gitRepo) RecentTag(rev, prefix string) (tag string, err error) { return tag, err } -func (r *gitRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, actualSubdir string, err error) { +func (r *gitRepo) DescendsFrom(rev, tag string) (bool, error) { + // The "--is-ancestor" flag was added to "git merge-base" in version 1.8.0, so + // this won't work with Git 1.7.1. According to golang.org/issue/28550, cmd/go + // already doesn't work with Git 1.7.1, so at least it's not a regression. + // + // git merge-base --is-ancestor exits with status 0 if rev is an ancestor, or + // 1 if not. + _, err := Run(r.dir, "git", "merge-base", "--is-ancestor", "--", tag, rev) + + // Git reports "is an ancestor" with exit code 0 and "not an ancestor" with + // exit code 1. + // Unfortunately, if we've already fetched rev with a shallow history, git + // merge-base has been observed to report a false-negative, so don't stop yet + // even if the exit code is 1! + if err == nil { + return true, nil + } + + // See whether the tag and rev even exist. + tags, err := r.Tags(tag) + if err != nil { + return false, err + } + if len(tags) == 0 { + return false, nil + } + + // NOTE: r.stat is very careful not to fetch commits that we shouldn't know + // about, like rejected GitHub pull requests, so don't try to short-circuit + // that here. + if _, err = r.stat(rev); err != nil { + return false, err + } + + // Now fetch history so that git can search for a path. + unlock, err := r.mu.Lock() + if err != nil { + return false, err + } + defer unlock() + + if r.fetchLevel < fetchAll { + // Fetch the complete history for all refs and heads. It would be more + // efficient to only fetch the history from rev to tag, but that's much more + // complicated, and any kind of shallow fetch is fairly likely to trigger + // bugs in JGit servers and/or the go command anyway. + if err := r.fetchRefsLocked(); err != nil { + return false, err + } + } + + _, err = Run(r.dir, "git", "merge-base", "--is-ancestor", "--", tag, rev) + if err == nil { + return true, nil + } + if ee, ok := err.(*RunError).Err.(*exec.ExitError); ok && ee.ExitCode() == 1 { + return false, nil + } + return false, err +} + +func (r *gitRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, err error) { // TODO: Use maxSize or drop it. args := []string{} if subdir != "" { @@ -712,17 +802,17 @@ func (r *gitRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, } info, err := r.Stat(rev) // download rev into local git repo if err != nil { - return nil, "", err + return nil, err } unlock, err := r.mu.Lock() if err != nil { - return nil, "", err + return nil, err } defer unlock() if err := ensureGitAttributes(r.dir); err != nil { - return nil, "", err + return nil, err } // Incredibly, git produces different archives depending on whether @@ -733,12 +823,12 @@ func (r *gitRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, archive, err := Run(r.dir, "git", "-c", "core.autocrlf=input", "-c", "core.eol=lf", "archive", "--format=zip", "--prefix=prefix/", info.Name, args) if err != nil { if bytes.Contains(err.(*RunError).Stderr, []byte("did not match any files")) { - return nil, "", os.ErrNotExist + return nil, os.ErrNotExist } - return nil, "", err + return nil, err } - return ioutil.NopCloser(bytes.NewReader(archive)), "", nil + return ioutil.NopCloser(bytes.NewReader(archive)), nil } // ensureGitAttributes makes sure export-subst and export-ignore features are diff --git a/cmd/go/_internal_/modfetch/codehost/svn.go b/cmd/go/_internal_/modfetch/codehost/svn.go new file mode 100644 index 0000000..9806f74 --- /dev/null +++ b/cmd/go/_internal_/modfetch/codehost/svn.go @@ -0,0 +1,154 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package codehost + +import ( + "archive/zip" + "encoding/xml" + "fmt" + "io" + "os" + "path" + "path/filepath" + "time" +) + +func svnParseStat(rev, out string) (*RevInfo, error) { + var log struct { + Logentry struct { + Revision int64 `xml:"revision,attr"` + Date string `xml:"date"` + } `xml:"logentry"` + } + if err := xml.Unmarshal([]byte(out), &log); err != nil { + return nil, vcsErrorf("unexpected response from svn log --xml: %v\n%s", err, out) + } + + t, err := time.Parse(time.RFC3339, log.Logentry.Date) + if err != nil { + return nil, vcsErrorf("unexpected response from svn log --xml: %v\n%s", err, out) + } + + info := &RevInfo{ + Name: fmt.Sprintf("%d", log.Logentry.Revision), + Short: fmt.Sprintf("%012d", log.Logentry.Revision), + Time: t.UTC(), + Version: rev, + } + return info, nil +} + +func svnReadZip(dst io.Writer, workDir, rev, subdir, remote string) (err error) { + // The subversion CLI doesn't provide a command to write the repository + // directly to an archive, so we need to export it to the local filesystem + // instead. Unfortunately, the local filesystem might apply arbitrary + // normalization to the filenames, so we need to obtain those directly. + // + // 'svn export' prints the filenames as they are written, but from reading the + // svn source code (as of revision 1868933), those filenames are encoded using + // the system locale rather than preserved byte-for-byte from the origin. For + // our purposes, that won't do, but we don't want to go mucking around with + // the user's locale settings either — that could impact error messages, and + // we don't know what locales the user has available or what LC_* variables + // their platform supports. + // + // Instead, we'll do a two-pass export: first we'll run 'svn list' to get the + // canonical filenames, then we'll 'svn export' and look for those filenames + // in the local filesystem. (If there is an encoding problem at that point, we + // would probably reject the resulting module anyway.) + + remotePath := remote + if subdir != "" { + remotePath += "/" + subdir + } + + out, err := Run(workDir, []string{ + "svn", "list", + "--non-interactive", + "--xml", + "--incremental", + "--recursive", + "--revision", rev, + "--", remotePath, + }) + if err != nil { + return err + } + + type listEntry struct { + Kind string `xml:"kind,attr"` + Name string `xml:"name"` + Size int64 `xml:"size"` + } + var list struct { + Entries []listEntry `xml:"entry"` + } + if err := xml.Unmarshal(out, &list); err != nil { + return vcsErrorf("unexpected response from svn list --xml: %v\n%s", err, out) + } + + exportDir := filepath.Join(workDir, "export") + // Remove any existing contents from a previous (failed) run. + if err := os.RemoveAll(exportDir); err != nil { + return err + } + defer os.RemoveAll(exportDir) // best-effort + + _, err = Run(workDir, []string{ + "svn", "export", + "--non-interactive", + "--quiet", + + // Suppress any platform- or host-dependent transformations. + "--native-eol", "LF", + "--ignore-externals", + "--ignore-keywords", + + "--revision", rev, + "--", remotePath, + exportDir, + }) + if err != nil { + return err + } + + // Scrape the exported files out of the filesystem and encode them in the zipfile. + + // “All files in the zip file are expected to be + // nested in a single top-level directory, whose name is not specified.” + // We'll (arbitrarily) choose the base of the remote path. + basePath := path.Join(path.Base(remote), subdir) + + zw := zip.NewWriter(dst) + for _, e := range list.Entries { + if e.Kind != "file" { + continue + } + + zf, err := zw.Create(path.Join(basePath, e.Name)) + if err != nil { + return err + } + + f, err := os.Open(filepath.Join(exportDir, e.Name)) + if err != nil { + if os.IsNotExist(err) { + return vcsErrorf("file reported by 'svn list', but not written by 'svn export': %s", e.Name) + } + return fmt.Errorf("error opening file created by 'svn export': %v", err) + } + + n, err := io.Copy(zf, f) + f.Close() + if err != nil { + return err + } + if n != e.Size { + return vcsErrorf("file size differs between 'svn list' and 'svn export': file %s listed as %v bytes, but exported as %v bytes", e.Name, e.Size, n) + } + } + + return zw.Close() +} diff --git a/cmd/go/_internal_/modfetch/codehost/vcs.go b/cmd/go/_internal_/modfetch/codehost/vcs.go index 9031fb8..7abe7f5 100644 --- a/cmd/go/_internal_/modfetch/codehost/vcs.go +++ b/cmd/go/_internal_/modfetch/codehost/vcs.go @@ -5,13 +5,13 @@ package codehost import ( - "encoding/xml" + "errors" "fmt" + "github.com/dependabot/gomodules-extracted/_internal_/lazyregexp" "io" "io/ioutil" "os" "path/filepath" - "regexp" "sort" "strconv" "strings" @@ -29,8 +29,9 @@ import ( // The caller should report this error instead of continuing to probe // other possible module paths. // -// TODO(bcmills): See if we can invert this. (Return a distinguished error for -// “repo not found” and treat everything else as terminal.) +// TODO(golang.org/issue/31730): See if we can invert this. (Return a +// distinguished error for “repo not found” and treat everything else +// as terminal.) type VCSError struct { Err error } @@ -121,28 +122,29 @@ func newVCSRepo(vcs, remote string) (Repo, error) { const vcsWorkDirType = "vcs1." type vcsCmd struct { - vcs string // vcs name "hg" - init func(remote string) []string // cmd to init repo to track remote - tags func(remote string) []string // cmd to list local tags - tagRE *regexp.Regexp // regexp to extract tag names from output of tags cmd - branches func(remote string) []string // cmd to list local branches - branchRE *regexp.Regexp // regexp to extract branch names from output of tags cmd - badLocalRevRE *regexp.Regexp // regexp of names that must not be served out of local cache without doing fetch first - statLocal func(rev, remote string) []string // cmd to stat local rev - parseStat func(rev, out string) (*RevInfo, error) // cmd to parse output of statLocal - fetch []string // cmd to fetch everything from remote - latest string // name of latest commit on remote (tip, HEAD, etc) - readFile func(rev, file, remote string) []string // cmd to read rev's file - readZip func(rev, subdir, remote, target string) []string // cmd to read rev's subdir as zip file + vcs string // vcs name "hg" + init func(remote string) []string // cmd to init repo to track remote + tags func(remote string) []string // cmd to list local tags + tagRE *lazyregexp.Regexp // regexp to extract tag names from output of tags cmd + branches func(remote string) []string // cmd to list local branches + branchRE *lazyregexp.Regexp // regexp to extract branch names from output of tags cmd + badLocalRevRE *lazyregexp.Regexp // regexp of names that must not be served out of local cache without doing fetch first + statLocal func(rev, remote string) []string // cmd to stat local rev + parseStat func(rev, out string) (*RevInfo, error) // cmd to parse output of statLocal + fetch []string // cmd to fetch everything from remote + latest string // name of latest commit on remote (tip, HEAD, etc) + readFile func(rev, file, remote string) []string // cmd to read rev's file + readZip func(rev, subdir, remote, target string) []string // cmd to read rev's subdir as zip file + doReadZip func(dst io.Writer, workDir, rev, subdir, remote string) error // arbitrary function to read rev's subdir as zip file } -var re = regexp.MustCompile +var re = lazyregexp.New var vcsCmds = map[string]*vcsCmd{ "hg": { vcs: "hg", init: func(remote string) []string { - return []string{"hg", "clone", "-U", remote, "."} + return []string{"hg", "clone", "-U", "--", remote, "."} }, tags: func(remote string) []string { return []string{"hg", "tags", "-q"} @@ -167,7 +169,7 @@ var vcsCmds = map[string]*vcsCmd{ if subdir != "" { pattern = []string{"-I", subdir + "/**"} } - return str.StringList("hg", "archive", "-t", "zip", "--no-decode", "-r", rev, "--prefix=prefix/", pattern, target) + return str.StringList("hg", "archive", "-t", "zip", "--no-decode", "-r", rev, "--prefix=prefix/", pattern, "--", target) }, }, @@ -175,7 +177,7 @@ var vcsCmds = map[string]*vcsCmd{ vcs: "svn", init: nil, // no local checkout tags: func(remote string) []string { - return []string{"svn", "list", strings.TrimSuffix(remote, "/trunk") + "/tags"} + return []string{"svn", "list", "--", strings.TrimSuffix(remote, "/trunk") + "/tags"} }, tagRE: re(`(?m)^(.*?)/?$`), statLocal: func(rev, remote string) []string { @@ -183,20 +185,20 @@ var vcsCmds = map[string]*vcsCmd{ if rev == "latest" { suffix = "" } - return []string{"svn", "log", "-l1", "--xml", remote + suffix} + return []string{"svn", "log", "-l1", "--xml", "--", remote + suffix} }, parseStat: svnParseStat, latest: "latest", readFile: func(rev, file, remote string) []string { - return []string{"svn", "cat", remote + "/" + file + "@" + rev} + return []string{"svn", "cat", "--", remote + "/" + file + "@" + rev} }, - // TODO: zip + doReadZip: svnReadZip, }, "bzr": { vcs: "bzr", init: func(remote string) []string { - return []string{"bzr", "branch", "--use-existing-dir", remote, "."} + return []string{"bzr", "branch", "--use-existing-dir", "--", remote, "."} }, fetch: []string{ "bzr", "pull", "--overwrite-tags", @@ -219,14 +221,14 @@ var vcsCmds = map[string]*vcsCmd{ if subdir != "" { extra = []string{"./" + subdir} } - return str.StringList("bzr", "export", "--format=zip", "-r", rev, "--root=prefix/", target, extra) + return str.StringList("bzr", "export", "--format=zip", "-r", rev, "--root=prefix/", "--", target, extra) }, }, "fossil": { vcs: "fossil", init: func(remote string) []string { - return []string{"fossil", "clone", remote, ".fossil"} + return []string{"fossil", "clone", "--", remote, ".fossil"} }, fetch: []string{"fossil", "pull", "-R", ".fossil"}, tags: func(remote string) []string { @@ -248,7 +250,7 @@ var vcsCmds = map[string]*vcsCmd{ } // Note that vcsRepo.ReadZip below rewrites this command // to run in a different directory, to work around a fossil bug. - return str.StringList("fossil", "zip", "-R", ".fossil", "--name", "prefix", extra, rev, target) + return str.StringList("fossil", "zip", "-R", ".fossil", "--name", "prefix", extra, "--", rev, target) }, }, } @@ -340,13 +342,15 @@ func (r *vcsRepo) Stat(rev string) (*RevInfo, error) { } func (r *vcsRepo) fetch() { - _, r.fetchErr = Run(r.dir, r.cmd.fetch) + if len(r.cmd.fetch) > 0 { + _, r.fetchErr = Run(r.dir, r.cmd.fetch) + } } func (r *vcsRepo) statLocal(rev string) (*RevInfo, error) { out, err := Run(r.dir, r.cmd.statLocal(rev, r.remote)) if err != nil { - return nil, vcsErrorf("unknown revision %s", rev) + return nil, &UnknownRevisionError{Rev: rev} } return r.cmd.parseStat(rev, string(out)) } @@ -391,7 +395,7 @@ func (r *vcsRepo) ReadFileRevs(revs []string, file string, maxSize int64) (map[s return nil, vcsErrorf("ReadFileRevs not implemented") } -func (r *vcsRepo) RecentTag(rev, prefix string) (tag string, err error) { +func (r *vcsRepo) RecentTag(rev, prefix, major string) (tag string, err error) { // We don't technically need to lock here since we're returning an error // uncondititonally, but doing so anyway will help to avoid baking in // lock-inversion bugs. @@ -404,14 +408,24 @@ func (r *vcsRepo) RecentTag(rev, prefix string) (tag string, err error) { return "", vcsErrorf("RecentTag not implemented") } -func (r *vcsRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, actualSubdir string, err error) { - if r.cmd.readZip == nil { - return nil, "", vcsErrorf("ReadZip not implemented for %s", r.cmd.vcs) +func (r *vcsRepo) DescendsFrom(rev, tag string) (bool, error) { + unlock, err := r.mu.Lock() + if err != nil { + return false, err + } + defer unlock() + + return false, vcsErrorf("DescendsFrom not implemented") +} + +func (r *vcsRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, err error) { + if r.cmd.readZip == nil && r.cmd.doReadZip == nil { + return nil, vcsErrorf("ReadZip not implemented for %s", r.cmd.vcs) } unlock, err := r.mu.Lock() if err != nil { - return nil, "", err + return nil, err } defer unlock() @@ -420,9 +434,19 @@ func (r *vcsRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, } f, err := ioutil.TempFile("", "go-readzip-*.zip") if err != nil { - return nil, "", err + return nil, err } - if r.cmd.vcs == "fossil" { + if r.cmd.doReadZip != nil { + lw := &limitedWriter{ + W: f, + N: maxSize, + ErrLimitReached: errors.New("ReadZip: encoded file exceeds allowed size"), + } + err = r.cmd.doReadZip(lw, r.dir, rev, subdir, r.remote) + if err == nil { + _, err = f.Seek(0, io.SeekStart) + } + } else if r.cmd.vcs == "fossil" { // If you run // fossil zip -R .fossil --name prefix trunk /tmp/x.zip // fossil fails with "unable to create directory /tmp" [sic]. @@ -441,9 +465,9 @@ func (r *vcsRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, if err != nil { f.Close() os.Remove(f.Name()) - return nil, "", err + return nil, err } - return &deleteCloser{f}, "", nil + return &deleteCloser{f}, nil } // deleteCloser is a file that gets deleted on Close. @@ -489,31 +513,6 @@ func hgParseStat(rev, out string) (*RevInfo, error) { return info, nil } -func svnParseStat(rev, out string) (*RevInfo, error) { - var log struct { - Logentry struct { - Revision int64 `xml:"revision,attr"` - Date string `xml:"date"` - } `xml:"logentry"` - } - if err := xml.Unmarshal([]byte(out), &log); err != nil { - return nil, vcsErrorf("unexpected response from svn log --xml: %v\n%s", err, out) - } - - t, err := time.Parse(time.RFC3339, log.Logentry.Date) - if err != nil { - return nil, vcsErrorf("unexpected response from svn log --xml: %v\n%s", err, out) - } - - info := &RevInfo{ - Name: fmt.Sprintf("%d", log.Logentry.Revision), - Short: fmt.Sprintf("%012d", log.Logentry.Revision), - Time: t.UTC(), - Version: rev, - } - return info, nil -} - func bzrParseStat(rev, out string) (*RevInfo, error) { var revno int64 var tm time.Time @@ -593,3 +592,25 @@ func fossilParseStat(rev, out string) (*RevInfo, error) { } return nil, vcsErrorf("unexpected response from fossil info: %q", out) } + +type limitedWriter struct { + W io.Writer + N int64 + ErrLimitReached error +} + +func (l *limitedWriter) Write(p []byte) (n int, err error) { + if l.N > 0 { + max := len(p) + if l.N < int64(max) { + max = int(l.N) + } + n, err = l.W.Write(p[:max]) + l.N -= int64(n) + if err != nil || n >= len(p) { + return n, err + } + } + + return n, l.ErrLimitReached +} diff --git a/cmd/go/_internal_/modfetch/coderepo.go b/cmd/go/_internal_/modfetch/coderepo.go index c872ce0..d29dfe9 100644 --- a/cmd/go/_internal_/modfetch/coderepo.go +++ b/cmd/go/_internal_/modfetch/coderepo.go @@ -6,17 +6,23 @@ package modfetch import ( "archive/zip" + "bytes" + "errors" "fmt" "io" "io/ioutil" "os" "path" + "sort" "strings" + "time" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch/codehost" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfile" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/module" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/semver" + + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" + "golang.org/x/mod/semver" + modzip "golang.org/x/mod/zip" ) // A codeRepo implements modfetch.Repo using an underlying codehost.Repo. @@ -29,7 +35,7 @@ type codeRepo struct { codeRoot string // codeDir is the directory (relative to root) at which we expect to find the module. // If pathMajor is non-empty and codeRoot is not the full modPath, - // then we look in both codeDir and codeDir+modPath + // then we look in both codeDir and codeDir/pathMajor[1:]. codeDir string // pathMajor is the suffix of modPath that indicates its major version, @@ -42,12 +48,10 @@ type codeRepo struct { // It is used only for logging. pathPrefix string - // pseudoMajor is the major version prefix to use when generating - // pseudo-versions for this module, derived from the module path. - // - // TODO(golang.org/issue/29262): We can't distinguish v0 from v1 using the - // path alone: we have to compute it by examining the tags at a particular - // revision. + // pseudoMajor is the major version prefix to require when generating + // pseudo-versions for this module, derived from the module path. pseudoMajor + // is empty if the module path does not include a version suffix (that is, + // accepts either v0 or v1). pseudoMajor string } @@ -65,10 +69,7 @@ func newCodeRepo(code codehost.Repo, codeRoot, path string) (Repo, error) { if codeRoot == path { pathPrefix = path } - pseudoMajor := "v0" - if pathMajor != "" { - pseudoMajor = pathMajor[1:] - } + pseudoMajor := module.PathMajorPrefix(pathMajor) // Compute codeDir = bar, the subdirectory within the repo // corresponding to the module root. @@ -143,11 +144,13 @@ func (r *codeRepo) Versions(prefix string) ([]string, error) { } tags, err := r.code.Tags(p) if err != nil { - return nil, err + return nil, &module.ModuleError{ + Path: r.modPath, + Err: err, + } } - list := []string{} - var incompatible []string + var list, incompatible []string for _, tag := range tags { if !strings.HasPrefix(tag, p) { continue @@ -159,32 +162,107 @@ func (r *codeRepo) Versions(prefix string) ([]string, error) { if v == "" || v != module.CanonicalVersion(v) || IsPseudoVersion(v) { continue } - if !module.MatchPathMajor(v, r.pathMajor) { + + if err := module.CheckPathMajor(v, r.pathMajor); err != nil { if r.codeDir == "" && r.pathMajor == "" && semver.Major(v) > "v1" { incompatible = append(incompatible, v) } continue } + list = append(list, v) } + SortVersions(list) + SortVersions(incompatible) + + return r.appendIncompatibleVersions(list, incompatible) +} - if len(incompatible) > 0 { - // Check for later versions that were created not following semantic import versioning, - // as indicated by the absence of a go.mod file. Those versions can be addressed - // by referring to them with a +incompatible suffix, as in v17.0.0+incompatible. - files, err := r.code.ReadFileRevs(incompatible, "go.mod", codehost.MaxGoMod) +// appendIncompatibleVersions appends "+incompatible" versions to list if +// appropriate, returning the final list. +// +// The incompatible list contains candidate versions without the '+incompatible' +// prefix. +// +// Both list and incompatible must be sorted in semantic order. +func (r *codeRepo) appendIncompatibleVersions(list, incompatible []string) ([]string, error) { + if len(incompatible) == 0 || r.pathMajor != "" { + // No +incompatible versions are possible, so no need to check them. + return list, nil + } + + versionHasGoMod := func(v string) (bool, error) { + _, err := r.code.ReadFile(v, "go.mod", codehost.MaxGoMod) + if err == nil { + return true, nil + } + if !os.IsNotExist(err) { + return false, &module.ModuleError{ + Path: r.modPath, + Err: err, + } + } + return false, nil + } + + if len(list) > 0 { + ok, err := versionHasGoMod(list[len(list)-1]) if err != nil { return nil, err } - for _, rev := range incompatible { - f := files[rev] - if os.IsNotExist(f.Err) { - list = append(list, rev+"+incompatible") + if ok { + // The latest compatible version has a go.mod file, so assume that all + // subsequent versions do as well, and do not include any +incompatible + // versions. Even if we are wrong, the author clearly intends module + // consumers to be on the v0/v1 line instead of a higher +incompatible + // version. (See https://golang.org/issue/34189.) + // + // We know of at least two examples where this behavior is desired + // (github.com/russross/blackfriday@v2.0.0 and + // github.com/libp2p/go-libp2p@v6.0.23), and (as of 2019-10-29) have no + // concrete examples for which it is undesired. + return list, nil + } + } + + var ( + lastMajor string + lastMajorHasGoMod bool + ) + for i, v := range incompatible { + major := semver.Major(v) + + if major != lastMajor { + rem := incompatible[i:] + j := sort.Search(len(rem), func(j int) bool { + return semver.Major(rem[j]) != major + }) + latestAtMajor := rem[j-1] + + var err error + lastMajor = major + lastMajorHasGoMod, err = versionHasGoMod(latestAtMajor) + if err != nil { + return nil, err } } + + if lastMajorHasGoMod { + // The latest release of this major version has a go.mod file, so it is + // not allowed as +incompatible. It would be confusing to include some + // minor versions of this major version as +incompatible but require + // semantic import versioning for others, so drop all +incompatible + // versions for this major version. + // + // If we're wrong about a minor version in the middle, users will still be + // able to 'go get' specific tags for that version explicitly — they just + // won't appear in 'go list' or as the results for queries with inequality + // bounds. + continue + } + list = append(list, v+"+incompatible") } - SortVersions(list) return list, nil } @@ -195,7 +273,13 @@ func (r *codeRepo) Stat(rev string) (*RevInfo, error) { codeRev := r.revToRev(rev) info, err := r.code.Stat(codeRev) if err != nil { - return nil, err + return nil, &module.ModuleError{ + Path: r.modPath, + Err: &module.InvalidVersionError{ + Version: rev, + Err: err, + }, + } } return r.convert(info, rev) } @@ -208,6 +292,11 @@ func (r *codeRepo) Latest() (*RevInfo, error) { return r.convert(info, "") } +// convert converts a version as reported by the code host to a version as +// interpreted by the module system. +// +// If statVers is a valid module version, it is used for the Version field. +// Otherwise, the Version is derived from the passed-in info and recent tags. func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, error) { info2 := &RevInfo{ Name: info.Name, @@ -215,79 +304,353 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e Time: info.Time, } - // Determine version. - if module.CanonicalVersion(statVers) == statVers && module.MatchPathMajor(statVers, r.pathMajor) { - // The original call was repo.Stat(statVers), and requestedVersion is OK, so use it. - info2.Version = statVers - } else { - // Otherwise derive a version from a code repo tag. - // Tag must have a prefix matching codeDir. - p := "" - if r.codeDir != "" { - p = r.codeDir + "/" - } - - // If this is a plain tag (no dir/ prefix) - // and the module path is unversioned, - // and if the underlying file tree has no go.mod, - // then allow using the tag with a +incompatible suffix. - canUseIncompatible := false + // If this is a plain tag (no dir/ prefix) + // and the module path is unversioned, + // and if the underlying file tree has no go.mod, + // then allow using the tag with a +incompatible suffix. + var canUseIncompatible func() bool + canUseIncompatible = func() bool { + var ok bool if r.codeDir == "" && r.pathMajor == "" { _, errGoMod := r.code.ReadFile(info.Name, "go.mod", codehost.MaxGoMod) if errGoMod != nil { - canUseIncompatible = true + ok = true } } + canUseIncompatible = func() bool { return ok } + return ok + } - tagToVersion := func(v string) string { - if !strings.HasPrefix(v, p) { - return "" + invalidf := func(format string, args ...interface{}) error { + return &module.ModuleError{ + Path: r.modPath, + Err: &module.InvalidVersionError{ + Version: info2.Version, + Err: fmt.Errorf(format, args...), + }, + } + } + + // checkGoMod verifies that the go.mod file for the module exists or does not + // exist as required by info2.Version and the module path represented by r. + checkGoMod := func() (*RevInfo, error) { + // If r.codeDir is non-empty, then the go.mod file must exist: the module + // author — not the module consumer, — gets to decide how to carve up the repo + // into modules. + // + // Conversely, if the go.mod file exists, the module author — not the module + // consumer — gets to determine the module's path + // + // r.findDir verifies both of these conditions. Execute it now so that + // r.Stat will correctly return a notExistError if the go.mod location or + // declared module path doesn't match. + _, _, _, err := r.findDir(info2.Version) + if err != nil { + // TODO: It would be nice to return an error like "not a module". + // Right now we return "missing go.mod", which is a little confusing. + return nil, &module.ModuleError{ + Path: r.modPath, + Err: &module.InvalidVersionError{ + Version: info2.Version, + Err: notExistError{err: err}, + }, + } + } + + // If the version is +incompatible, then the go.mod file must not exist: + // +incompatible is not an ongoing opt-out from semantic import versioning. + if strings.HasSuffix(info2.Version, "+incompatible") { + if !canUseIncompatible() { + if r.pathMajor != "" { + return nil, invalidf("+incompatible suffix not allowed: module path includes a major version suffix, so major version must match") + } else { + return nil, invalidf("+incompatible suffix not allowed: module contains a go.mod file, so semantic import versioning is required") + } } - v = v[len(p):] - if module.CanonicalVersion(v) != v || IsPseudoVersion(v) { - return "" + + if err := module.CheckPathMajor(strings.TrimSuffix(info2.Version, "+incompatible"), r.pathMajor); err == nil { + return nil, invalidf("+incompatible suffix not allowed: major version %s is compatible", semver.Major(info2.Version)) } - if module.MatchPathMajor(v, r.pathMajor) { - return v + } + + return info2, nil + } + + // Determine version. + // + // If statVers is canonical, then the original call was repo.Stat(statVers). + // Since the version is canonical, we must not resolve it to anything but + // itself, possibly with a '+incompatible' annotation: we do not need to do + // the work required to look for an arbitrary pseudo-version. + if statVers != "" && statVers == module.CanonicalVersion(statVers) { + info2.Version = statVers + + if IsPseudoVersion(info2.Version) { + if err := r.validatePseudoVersion(info, info2.Version); err != nil { + return nil, err } - if canUseIncompatible { - return v + "+incompatible" + return checkGoMod() + } + + if err := module.CheckPathMajor(info2.Version, r.pathMajor); err != nil { + if canUseIncompatible() { + info2.Version += "+incompatible" + return checkGoMod() + } else { + if vErr, ok := err.(*module.InvalidVersionError); ok { + // We're going to describe why the version is invalid in more detail, + // so strip out the existing “invalid version” wrapper. + err = vErr.Err + } + return nil, invalidf("module contains a go.mod file, so major version must be compatible: %v", err) } - return "" } - // If info.Version is OK, use it. - if v := tagToVersion(info.Version); v != "" { - info2.Version = v - } else { - // Otherwise look through all known tags for latest in semver ordering. - for _, tag := range info.Tags { - if v := tagToVersion(tag); v != "" && semver.Compare(info2.Version, v) < 0 { + return checkGoMod() + } + + // statVers is empty or non-canonical, so we need to resolve it to a canonical + // version or pseudo-version. + + // Derive or verify a version from a code repo tag. + // Tag must have a prefix matching codeDir. + tagPrefix := "" + if r.codeDir != "" { + tagPrefix = r.codeDir + "/" + } + + // tagToVersion returns the version obtained by trimming tagPrefix from tag. + // If the tag is invalid or a pseudo-version, tagToVersion returns an empty + // version. + tagToVersion := func(tag string) (v string, tagIsCanonical bool) { + if !strings.HasPrefix(tag, tagPrefix) { + return "", false + } + trimmed := tag[len(tagPrefix):] + // Tags that look like pseudo-versions would be confusing. Ignore them. + if IsPseudoVersion(tag) { + return "", false + } + + v = semver.Canonical(trimmed) // Not module.Canonical: we don't want to pick up an explicit "+incompatible" suffix from the tag. + if v == "" || !strings.HasPrefix(trimmed, v) { + return "", false // Invalid or incomplete version (just vX or vX.Y). + } + if v == trimmed { + tagIsCanonical = true + } + + if err := module.CheckPathMajor(v, r.pathMajor); err != nil { + if canUseIncompatible() { + return v + "+incompatible", tagIsCanonical + } + return "", false + } + + return v, tagIsCanonical + } + + // If the VCS gave us a valid version, use that. + if v, tagIsCanonical := tagToVersion(info.Version); tagIsCanonical { + info2.Version = v + return checkGoMod() + } + + // Look through the tags on the revision for either a usable canonical version + // or an appropriate base for a pseudo-version. + var pseudoBase string + for _, pathTag := range info.Tags { + v, tagIsCanonical := tagToVersion(pathTag) + if tagIsCanonical { + if statVers != "" && semver.Compare(v, statVers) == 0 { + // The user requested a non-canonical version, but the tag for the + // canonical equivalent refers to the same revision. Use it. + info2.Version = v + return checkGoMod() + } else { + // Save the highest canonical tag for the revision. If we don't find a + // better match, we'll use it as the canonical version. + // + // NOTE: Do not replace this with semver.Max. Despite the name, + // semver.Max *also* canonicalizes its arguments, which uses + // semver.Canonical instead of module.CanonicalVersion and thereby + // strips our "+incompatible" suffix. + if semver.Compare(info2.Version, v) < 0 { info2.Version = v } } - // Otherwise make a pseudo-version. - if info2.Version == "" { - tag, _ := r.code.RecentTag(statVers, p) - v = tagToVersion(tag) - // TODO: Check that v is OK for r.pseudoMajor or else is OK for incompatible. - info2.Version = PseudoVersion(r.pseudoMajor, v, info.Time, info.Short) + } else if v != "" && semver.Compare(v, statVers) == 0 { + // The user explicitly requested something equivalent to this tag. We + // can't use the version from the tag directly: since the tag is not + // canonical, it could be ambiguous. For example, tags v0.0.1+a and + // v0.0.1+b might both exist and refer to different revisions. + // + // The tag is otherwise valid for the module, so we can at least use it as + // the base of an unambiguous pseudo-version. + // + // If multiple tags match, tagToVersion will canonicalize them to the same + // base version. + pseudoBase = v + } + } + + // If we found any canonical tag for the revision, return it. + // Even if we found a good pseudo-version base, a canonical version is better. + if info2.Version != "" { + return checkGoMod() + } + + if pseudoBase == "" { + var tag string + if r.pseudoMajor != "" || canUseIncompatible() { + tag, _ = r.code.RecentTag(info.Name, tagPrefix, r.pseudoMajor) + } else { + // Allow either v1 or v0, but not incompatible higher versions. + tag, _ = r.code.RecentTag(info.Name, tagPrefix, "v1") + if tag == "" { + tag, _ = r.code.RecentTag(info.Name, tagPrefix, "v0") } } + pseudoBase, _ = tagToVersion(tag) // empty if the tag is invalid } - // Do not allow a successful stat of a pseudo-version for a subdirectory - // unless the subdirectory actually does have a go.mod. - if IsPseudoVersion(info2.Version) && r.codeDir != "" { - _, _, _, err := r.findDir(info2.Version) + info2.Version = PseudoVersion(r.pseudoMajor, pseudoBase, info.Time, info.Short) + return checkGoMod() +} + +// validatePseudoVersion checks that version has a major version compatible with +// r.modPath and encodes a base version and commit metadata that agrees with +// info. +// +// Note that verifying a nontrivial base version in particular may be somewhat +// expensive: in order to do so, r.code.DescendsFrom will need to fetch at least +// enough of the commit history to find a path between version and its base. +// Fortunately, many pseudo-versions — such as those for untagged repositories — +// have trivial bases! +func (r *codeRepo) validatePseudoVersion(info *codehost.RevInfo, version string) (err error) { + defer func() { if err != nil { - // TODO: It would be nice to return an error like "not a module". - // Right now we return "missing go.mod", which is a little confusing. - return nil, err + if _, ok := err.(*module.ModuleError); !ok { + if _, ok := err.(*module.InvalidVersionError); !ok { + err = &module.InvalidVersionError{Version: version, Pseudo: true, Err: err} + } + err = &module.ModuleError{Path: r.modPath, Err: err} + } + } + }() + + if err := module.CheckPathMajor(version, r.pathMajor); err != nil { + return err + } + + rev, err := PseudoVersionRev(version) + if err != nil { + return err + } + if rev != info.Short { + switch { + case strings.HasPrefix(rev, info.Short): + return fmt.Errorf("revision is longer than canonical (%s)", info.Short) + case strings.HasPrefix(info.Short, rev): + return fmt.Errorf("revision is shorter than canonical (%s)", info.Short) + default: + return fmt.Errorf("does not match short name of revision (%s)", info.Short) + } + } + + t, err := PseudoVersionTime(version) + if err != nil { + return err + } + if !t.Equal(info.Time.Truncate(time.Second)) { + return fmt.Errorf("does not match version-control timestamp (expected %s)", info.Time.UTC().Format(pseudoVersionTimestampFormat)) + } + + tagPrefix := "" + if r.codeDir != "" { + tagPrefix = r.codeDir + "/" + } + + // A pseudo-version should have a precedence just above its parent revisions, + // and no higher. Otherwise, it would be possible for library authors to "pin" + // dependency versions (and bypass the usual minimum version selection) by + // naming an extremely high pseudo-version rather than an accurate one. + // + // Moreover, if we allow a pseudo-version to use any arbitrary pre-release + // tag, we end up with infinitely many possible names for each commit. Each + // name consumes resources in the module cache and proxies, so we want to + // restrict them to a finite set under control of the module author. + // + // We address both of these issues by requiring the tag upon which the + // pseudo-version is based to refer to some ancestor of the revision. We + // prefer the highest such tag when constructing a new pseudo-version, but do + // not enforce that property when resolving existing pseudo-versions: we don't + // know when the parent tags were added, and the highest-tagged parent may not + // have existed when the pseudo-version was first resolved. + base, err := PseudoVersionBase(strings.TrimSuffix(version, "+incompatible")) + if err != nil { + return err + } + if base == "" { + if r.pseudoMajor == "" && semver.Major(version) == "v1" { + return fmt.Errorf("major version without preceding tag must be v0, not v1") } + return nil + } else { + for _, tag := range info.Tags { + versionOnly := strings.TrimPrefix(tag, tagPrefix) + if versionOnly == base { + // The base version is canonical, so if the version from the tag is + // literally equal (not just equivalent), then the tag is canonical too. + // + // We allow pseudo-versions to be derived from non-canonical tags on the + // same commit, so that tags like "v1.1.0+some-metadata" resolve as + // close as possible to the canonical version ("v1.1.0") while still + // enforcing a total ordering ("v1.1.1-0.[…]" with a unique suffix). + // + // However, canonical tags already have a total ordering, so there is no + // reason not to use the canonical tag directly, and we know that the + // canonical tag must already exist because the pseudo-version is + // derived from it. In that case, referring to the revision by a + // pseudo-version derived from its own canonical tag is just confusing. + return fmt.Errorf("tag (%s) found on revision %s is already canonical, so should not be replaced with a pseudo-version derived from that tag", tag, rev) + } + } + } + + tags, err := r.code.Tags(tagPrefix + base) + if err != nil { + return err } - return info2, nil + var lastTag string // Prefer to log some real tag rather than a canonically-equivalent base. + ancestorFound := false + for _, tag := range tags { + versionOnly := strings.TrimPrefix(tag, tagPrefix) + if semver.Compare(versionOnly, base) == 0 { + lastTag = tag + ancestorFound, err = r.code.DescendsFrom(info.Name, tag) + if ancestorFound { + break + } + } + } + + if lastTag == "" { + return fmt.Errorf("preceding tag (%s) not found", base) + } + + if !ancestorFound { + if err != nil { + return err + } + rev, err := PseudoVersionRev(version) + if err != nil { + return fmt.Errorf("not a descendent of preceding tag (%s)", lastTag) + } + return fmt.Errorf("revision %s is not a descendent of preceding tag (%s)", rev, lastTag) + } + return nil } func (r *codeRepo) revToRev(rev string) string { @@ -309,11 +672,21 @@ func (r *codeRepo) revToRev(rev string) string { func (r *codeRepo) versionToRev(version string) (rev string, err error) { if !semver.IsValid(version) { - return "", fmt.Errorf("malformed semantic version %q", version) + return "", &module.ModuleError{ + Path: r.modPath, + Err: &module.InvalidVersionError{ + Version: version, + Err: errors.New("syntax error"), + }, + } } return r.revToRev(version), nil } +// findDir locates the directory within the repo containing the module. +// +// If r.pathMajor is non-empty, this can be either r.codeDir or — if a go.mod +// file exists — r.codeDir/r.pathMajor[1:]. func (r *codeRepo) findDir(version string) (rev, dir string, gomod []byte, err error) { rev, err = r.versionToRev(version) if err != nil { @@ -328,7 +701,7 @@ func (r *codeRepo) findDir(version string) (rev, dir string, gomod []byte, err e return "", "", nil, fmt.Errorf("reading %s/%s at revision %s: %v", r.pathPrefix, file1, rev, err1) } mpath1 := modfile.ModulePath(gomod1) - found1 := err1 == nil && isMajor(mpath1, r.pathMajor) + found1 := err1 == nil && (isMajor(mpath1, r.pathMajor) || r.canReplaceMismatchedVersionDueToBug(mpath1)) var file2 string if r.pathMajor != "" && r.codeRoot != r.modPath && !strings.HasPrefix(r.pathMajor, ".") { @@ -339,9 +712,6 @@ func (r *codeRepo) findDir(version string) (rev, dir string, gomod []byte, err e // because of replacement modules. This might be a fork of // the real module, found at a different path, usable only in // a replace directive. - // - // TODO(bcmills): This doesn't seem right. Investigate futher. - // (Notably: why can't we replace foo/v2 with fork-of-foo/v3?) dir2 := path.Join(r.codeDir, r.pathMajor[1:]) file2 = path.Join(dir2, "go.mod") gomod2, err2 := r.code.ReadFile(rev, file2, codehost.MaxGoMod) @@ -367,11 +737,11 @@ func (r *codeRepo) findDir(version string) (rev, dir string, gomod []byte, err e // Not v2/go.mod, so it's either go.mod or nothing. Which is it? if found1 { - // Explicit go.mod with matching module path OK. + // Explicit go.mod with matching major version ok. return rev, r.codeDir, gomod1, nil } if err1 == nil { - // Explicit go.mod with non-matching module path disallowed. + // Explicit go.mod with non-matching major version disallowed. suffix := "" if file2 != "" { suffix = fmt.Sprintf(" (and ...%s/go.mod does not exist)", r.pathMajor) @@ -382,6 +752,9 @@ func (r *codeRepo) findDir(version string) (rev, dir string, gomod []byte, err e if r.pathMajor != "" { // ".v1", ".v2" for gopkg.in return "", "", nil, fmt.Errorf("%s has non-...%s module path %q%s at revision %s", file1, r.pathMajor, mpath1, suffix, rev) } + if _, _, ok := module.SplitPathVersion(mpath1); !ok { + return "", "", nil, fmt.Errorf("%s has malformed module path %q%s at revision %s", file1, mpath1, suffix, rev) + } return "", "", nil, fmt.Errorf("%s has post-%s module path %q%s at revision %s", file1, semver.Major(version), mpath1, suffix, rev) } @@ -398,27 +771,72 @@ func (r *codeRepo) findDir(version string) (rev, dir string, gomod []byte, err e return "", "", nil, fmt.Errorf("missing %s/go.mod at revision %s", r.pathPrefix, rev) } +// isMajor reports whether the versions allowed for mpath are compatible with +// the major version(s) implied by pathMajor, or false if mpath has an invalid +// version suffix. func isMajor(mpath, pathMajor string) bool { if mpath == "" { + // If we don't have a path, we don't know what version(s) it is compatible with. + return false + } + _, mpathMajor, ok := module.SplitPathVersion(mpath) + if !ok { + // An invalid module path is not compatible with any version. return false } if pathMajor == "" { - // mpath must NOT have version suffix. - i := len(mpath) - for i > 0 && '0' <= mpath[i-1] && mpath[i-1] <= '9' { - i-- - } - if i < len(mpath) && i >= 2 && mpath[i-1] == 'v' && mpath[i-2] == '/' { - // Found valid suffix. + // All of the valid versions for a gopkg.in module that requires major + // version v0 or v1 are compatible with the "v0 or v1" implied by an empty + // pathMajor. + switch module.PathMajorPrefix(mpathMajor) { + case "", "v0", "v1": + return true + default: return false } - return true } - // Otherwise pathMajor is ".v1", ".v2" (gopkg.in), or "/v2", "/v3" etc. - return strings.HasSuffix(mpath, pathMajor) + if mpathMajor == "" { + // Even if pathMajor is ".v0" or ".v1", we can't be sure that a module + // without a suffix is tagged appropriately. Besides, we don't expect clones + // of non-gopkg.in modules to have gopkg.in paths, so a non-empty, + // non-gopkg.in mpath is probably the wrong module for any such pathMajor + // anyway. + return false + } + // If both pathMajor and mpathMajor are non-empty, then we only care that they + // have the same major-version validation rules. A clone fetched via a /v2 + // path might replace a module with path gopkg.in/foo.v2-unstable, and that's + // ok. + return pathMajor[1:] == mpathMajor[1:] +} + +// canReplaceMismatchedVersionDueToBug reports whether versions of r +// could replace versions of mpath with otherwise-mismatched major versions +// due to a historical bug in the Go command (golang.org/issue/34254). +func (r *codeRepo) canReplaceMismatchedVersionDueToBug(mpath string) bool { + // The bug caused us to erroneously accept unversioned paths as replacements + // for versioned gopkg.in paths. + unversioned := r.pathMajor == "" + replacingGopkgIn := strings.HasPrefix(mpath, "gopkg.in/") + return unversioned && replacingGopkgIn } func (r *codeRepo) GoMod(version string) (data []byte, err error) { + if version != module.CanonicalVersion(version) { + return nil, fmt.Errorf("version %s is not canonical", version) + } + + if IsPseudoVersion(version) { + // findDir ignores the metadata encoded in a pseudo-version, + // only using the revision at the end. + // Invoke Stat to verify the metadata explicitly so we don't return + // a bogus file for an invalid version. + _, err := r.Stat(version) + if err != nil { + return nil, err + } + } + rev, dir, gomod, err := r.findDir(version) if err != nil { return nil, err @@ -452,19 +870,31 @@ func (r *codeRepo) modPrefix(rev string) string { } func (r *codeRepo) Zip(dst io.Writer, version string) error { - rev, dir, _, err := r.findDir(version) + if version != module.CanonicalVersion(version) { + return fmt.Errorf("version %s is not canonical", version) + } + + if IsPseudoVersion(version) { + // findDir ignores the metadata encoded in a pseudo-version, + // only using the revision at the end. + // Invoke Stat to verify the metadata explicitly so we don't return + // a bogus file for an invalid version. + _, err := r.Stat(version) + if err != nil { + return err + } + } + + rev, subdir, _, err := r.findDir(version) if err != nil { return err } - dl, actualDir, err := r.code.ReadZip(rev, dir, codehost.MaxZipFile) + dl, err := r.code.ReadZip(rev, subdir, codehost.MaxZipFile) if err != nil { return err } defer dl.Close() - if actualDir != "" && !hasPathPrefix(dir, actualDir) { - return fmt.Errorf("internal error: downloading %v %v: dir=%q but actualDir=%q", r.modPath, rev, dir, actualDir) - } - subdir := strings.Trim(strings.TrimPrefix(dir, actualDir), "/") + subdir = strings.Trim(subdir, "/") // Spool to local file. f, err := ioutil.TempFile("", "go-codehost-") @@ -495,49 +925,13 @@ func (r *codeRepo) Zip(dst io.Writer, version string) error { return err } - zw := zip.NewWriter(dst) + var files []modzip.File if subdir != "" { subdir += "/" } haveLICENSE := false topPrefix := "" - haveGoMod := make(map[string]bool) - for _, zf := range zr.File { - if topPrefix == "" { - i := strings.Index(zf.Name, "/") - if i < 0 { - return fmt.Errorf("missing top-level directory prefix") - } - topPrefix = zf.Name[:i+1] - } - if !strings.HasPrefix(zf.Name, topPrefix) { - return fmt.Errorf("zip file contains more than one top-level directory") - } - dir, file := path.Split(zf.Name) - if file == "go.mod" { - haveGoMod[dir] = true - } - } - root := topPrefix + subdir - inSubmodule := func(name string) bool { - for { - dir, _ := path.Split(name) - if len(dir) <= len(root) { - return false - } - if haveGoMod[dir] { - return true - } - name = dir[:len(dir)-1] - } - } - for _, zf := range zr.File { - if !zf.FileInfo().Mode().IsRegular() { - // Skip symlinks (golang.org/issue/27093). - continue - } - if topPrefix == "" { i := strings.Index(zf.Name, "/") if i < 0 { @@ -545,9 +939,6 @@ func (r *codeRepo) Zip(dst io.Writer, version string) error { } topPrefix = zf.Name[:i+1] } - if strings.HasSuffix(zf.Name, "/") { // drop directory dummy entries - continue - } if !strings.HasPrefix(zf.Name, topPrefix) { return fmt.Errorf("zip file contains more than one top-level directory") } @@ -555,61 +946,57 @@ func (r *codeRepo) Zip(dst io.Writer, version string) error { if !strings.HasPrefix(name, subdir) { continue } - if name == ".hg_archival.txt" { - // Inserted by hg archive. - // Not correct to drop from other version control systems, but too bad. - continue - } name = strings.TrimPrefix(name, subdir) - if isVendoredPackage(name) { - continue - } - if inSubmodule(zf.Name) { + if name == "" || strings.HasSuffix(name, "/") { continue } - base := path.Base(name) - if strings.ToLower(base) == "go.mod" && base != "go.mod" { - return fmt.Errorf("zip file contains %s, want all lower-case go.mod", zf.Name) - } + files = append(files, zipFile{name: name, f: zf}) if name == "LICENSE" { haveLICENSE = true } - size := int64(zf.UncompressedSize64) - if size < 0 || maxSize < size { - return fmt.Errorf("module source tree too big") - } - maxSize -= size - - rc, err := zf.Open() - if err != nil { - return err - } - w, err := zw.Create(r.modPrefix(version) + "/" + name) - lr := &io.LimitedReader{R: rc, N: size + 1} - if _, err := io.Copy(w, lr); err != nil { - return err - } - if lr.N <= 0 { - return fmt.Errorf("individual file too large") - } } if !haveLICENSE && subdir != "" { data, err := r.code.ReadFile(rev, "LICENSE", codehost.MaxLICENSE) if err == nil { - w, err := zw.Create(r.modPrefix(version) + "/LICENSE") - if err != nil { - return err - } - if _, err := w.Write(data); err != nil { - return err - } + files = append(files, dataFile{name: "LICENSE", data: data}) } } - return zw.Close() + return modzip.Create(dst, module.Version{Path: r.modPath, Version: version}, files) } +type zipFile struct { + name string + f *zip.File +} + +func (f zipFile) Path() string { return f.name } +func (f zipFile) Lstat() (os.FileInfo, error) { return f.f.FileInfo(), nil } +func (f zipFile) Open() (io.ReadCloser, error) { return f.f.Open() } + +type dataFile struct { + name string + data []byte +} + +func (f dataFile) Path() string { return f.name } +func (f dataFile) Lstat() (os.FileInfo, error) { return dataFileInfo{f}, nil } +func (f dataFile) Open() (io.ReadCloser, error) { + return ioutil.NopCloser(bytes.NewReader(f.data)), nil +} + +type dataFileInfo struct { + f dataFile +} + +func (fi dataFileInfo) Name() string { return path.Base(fi.f.name) } +func (fi dataFileInfo) Size() int64 { return int64(len(fi.f.data)) } +func (fi dataFileInfo) Mode() os.FileMode { return 0644 } +func (fi dataFileInfo) ModTime() time.Time { return time.Time{} } +func (fi dataFileInfo) IsDir() bool { return false } +func (fi dataFileInfo) Sys() interface{} { return nil } + // hasPathPrefix reports whether the path s begins with the // elements in prefix. func hasPathPrefix(s, prefix string) bool { @@ -625,15 +1012,3 @@ func hasPathPrefix(s, prefix string) bool { return s[len(prefix)] == '/' && s[:len(prefix)] == prefix } } - -func isVendoredPackage(name string) bool { - var i int - if strings.HasPrefix(name, "vendor/") { - i += len("vendor/") - } else if j := strings.Index(name, "/vendor/"); j >= 0 { - i += len("/vendor/") - } else { - return false - } - return strings.Contains(name[i:], "/") -} diff --git a/cmd/go/_internal_/modfetch/fetch.go b/cmd/go/_internal_/modfetch/fetch.go index 73e22f1..9bc1ce5 100644 --- a/cmd/go/_internal_/modfetch/fetch.go +++ b/cmd/go/_internal_/modfetch/fetch.go @@ -7,6 +7,7 @@ package modfetch import ( "archive/zip" "bytes" + "errors" "fmt" "io" "io/ioutil" @@ -18,10 +19,14 @@ import ( "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/dirhash" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/module" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/lockedfile" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/par" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/renameio" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/robustio" + + "golang.org/x/mod/module" + "golang.org/x/mod/sumdb/dirhash" + modzip "golang.org/x/mod/zip" ) var downloadCache par.Cache @@ -30,9 +35,10 @@ var downloadCache par.Cache // local download cache and returns the name of the directory // corresponding to the root of the module's file tree. func Download(mod module.Version) (dir string, err error) { - if PkgMod == "" { - // Do not download to current directory. - return "", fmt.Errorf("missing modfetch.PkgMod") + if cfg.GOMODCACHE == "" { + // modload.Init exits if GOPATH[0] is empty, and cfg.GOMODCACHE + // is set to GOPATH[0]/pkg/mod if GOMODCACHE is empty, so this should never happen. + base.Fatalf("go: internal error: cfg.GOMODCACHE not set") } // The par.Cache here avoids duplicate work. @@ -41,24 +47,26 @@ func Download(mod module.Version) (dir string, err error) { err error } c := downloadCache.Do(mod, func() interface{} { - dir, err := DownloadDir(mod) + dir, err := download(mod) if err != nil { return cached{"", err} } - if err := download(mod, dir); err != nil { - return cached{"", err} - } - checkSum(mod) + checkMod(mod) return cached{dir, nil} }).(cached) return c.dir, c.err } -func download(mod module.Version, dir string) (err error) { - // If the directory exists, the module has already been extracted. - fi, err := os.Stat(dir) - if err == nil && fi.IsDir() { - return nil +func download(mod module.Version) (dir string, err error) { + // If the directory exists, and no .partial file exists, the module has + // already been completely extracted. .partial files may be created when a + // module zip directory is extracted in place instead of being extracted to a + // temporary directory and renamed. + dir, err = DownloadDir(mod) + if err == nil { + return dir, nil + } else if dir == "" || !errors.Is(err, os.ErrNotExist) { + return "", err } // To avoid cluttering the cache with extraneous files, @@ -66,26 +74,24 @@ func download(mod module.Version, dir string) (err error) { // Invoke DownloadZip before locking the file. zipfile, err := DownloadZip(mod) if err != nil { - return err - } - - if cfg.CmdName != "mod download" { - fmt.Fprintf(os.Stderr, "go: extracting %s %s\n", mod.Path, mod.Version) + return "", err } unlock, err := lockVersion(mod) if err != nil { - return err + return "", err } defer unlock() // Check whether the directory was populated while we were waiting on the lock. - fi, err = os.Stat(dir) - if err == nil && fi.IsDir() { - return nil + _, dirErr := DownloadDir(mod) + if dirErr == nil { + return dir, nil } + _, dirExists := dirErr.(*DownloadDirPartialError) - // Clean up any remaining temporary directories from previous runs. + // Clean up any remaining temporary directories from previous runs, as well + // as partially extracted diectories created by future versions of cmd/go. // This is only safe to do because the lock file ensures that their writers // are no longer active. parentDir := filepath.Dir(dir) @@ -95,39 +101,94 @@ func download(mod module.Version, dir string) (err error) { RemoveAll(path) // best effort } } - - // Extract the zip file to a temporary directory, then rename it to the - // final path. That way, we can use the existence of the source directory to - // signal that it has been extracted successfully, and if someone deletes - // the entire directory (e.g. as an attempt to prune out file corruption) - // the module cache will still be left in a recoverable state. - if err := os.MkdirAll(parentDir, 0777); err != nil { - return err + if dirExists { + if err := RemoveAll(dir); err != nil { + return "", err + } } - tmpDir, err := ioutil.TempDir(parentDir, tmpPrefix) + + partialPath, err := CachePath(mod, "partial") if err != nil { - return err + return "", err } - defer func() { + if err := os.Remove(partialPath); err != nil && !os.IsNotExist(err) { + return "", err + } + + // Extract the module zip directory. + // + // By default, we extract to a temporary directory, then atomically rename to + // its final location. We use the existence of the source directory to signal + // that it has been extracted successfully (see DownloadDir). If someone + // deletes the entire directory (e.g., as an attempt to prune out file + // corruption), the module cache will still be left in a recoverable + // state. + // + // Unfortunately, os.Rename may fail with ERROR_ACCESS_DENIED on Windows if + // another process opens files in the temporary directory. This is partially + // mitigated by using robustio.Rename, which retries os.Rename for a short + // time. + // + // To avoid this error completely, if unzipInPlace is set, we instead create a + // .partial file (indicating the directory isn't fully extracted), then we + // extract the directory at its final location, then we delete the .partial + // file. This is not the default behavior because older versions of Go may + // simply stat the directory to check whether it exists without looking for a + // .partial file. If multiple versions run concurrently, the older version may + // assume a partially extracted directory is complete. + // TODO(golang.org/issue/36568): when these older versions are no longer + // supported, remove the old default behavior and the unzipInPlace flag. + if err := os.MkdirAll(parentDir, 0777); err != nil { + return "", err + } + + if unzipInPlace { + if err := ioutil.WriteFile(partialPath, nil, 0666); err != nil { + return "", err + } + if err := modzip.Unzip(dir, mod, zipfile); err != nil { + fmt.Fprintf(os.Stderr, "-> %s\n", err) + if rmErr := RemoveAll(dir); rmErr == nil { + os.Remove(partialPath) + } + return "", err + } + if err := os.Remove(partialPath); err != nil { + return "", err + } + } else { + tmpDir, err := ioutil.TempDir(parentDir, tmpPrefix) if err != nil { + return "", err + } + if err := modzip.Unzip(tmpDir, mod, zipfile); err != nil { + fmt.Fprintf(os.Stderr, "-> %s\n", err) RemoveAll(tmpDir) + return "", err + } + if err := robustio.Rename(tmpDir, dir); err != nil { + RemoveAll(tmpDir) + return "", err } - }() - - modpath := mod.Path + "@" + mod.Version - if err := Unzip(tmpDir, zipfile, modpath, 0); err != nil { - fmt.Fprintf(os.Stderr, "-> %s\n", err) - return err } - if err := os.Rename(tmpDir, dir); err != nil { - return err + if !cfg.ModCacheRW { + // Make dir read-only only *after* renaming it. + // os.Rename was observed to fail for read-only directories on macOS. + makeDirsReadOnly(dir) } + return dir, nil +} - // Make dir read-only only *after* renaming it. - // os.Rename was observed to fail for read-only directories on macOS. - makeDirsReadOnly(dir) - return nil +var unzipInPlace bool + +func init() { + for _, f := range strings.Split(os.Getenv("GODEBUG"), ",") { + if f == "modcacheunzipinplace=1" { + unzipInPlace = true + break + } + } } var downloadZipCache par.Cache @@ -205,13 +266,16 @@ func downloadZip(mod module.Version, zipfile string) (err error) { } }() - repo, err := Lookup(mod.Path) + err = TryProxies(func(proxy string) error { + repo, err := Lookup(proxy, mod.Path) + if err != nil { + return err + } + return repo.Zip(f, mod.Version) + }) if err != nil { return err } - if err := repo.Zip(f, mod.Version); err != nil { - return err - } // Double-check that the paths within the zip file are well-formed. // @@ -246,9 +310,11 @@ func downloadZip(mod module.Version, zipfile string) (err error) { if err != nil { return err } - checkOneSum(mod, hash) + if err := checkModSum(mod, hash); err != nil { + return err + } - if err := renameio.WriteFile(zipfile+"hash", []byte(hash)); err != nil { + if err := renameio.WriteFile(zipfile+"hash", []byte(hash), 0666); err != nil { return err } if err := os.Rename(f.Name(), zipfile); err != nil { @@ -260,6 +326,45 @@ func downloadZip(mod module.Version, zipfile string) (err error) { return nil } +// makeDirsReadOnly makes a best-effort attempt to remove write permissions for dir +// and its transitive contents. +func makeDirsReadOnly(dir string) { + type pathMode struct { + path string + mode os.FileMode + } + var dirs []pathMode // in lexical order + filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err == nil && info.Mode()&0222 != 0 { + if info.IsDir() { + dirs = append(dirs, pathMode{path, info.Mode()}) + } + } + return nil + }) + + // Run over list backward to chmod children before parents. + for i := len(dirs) - 1; i >= 0; i-- { + os.Chmod(dirs[i].path, dirs[i].mode&^0222) + } +} + +// RemoveAll removes a directory written by Download or Unzip, first applying +// any permission changes needed to do so. +func RemoveAll(dir string) error { + // Module cache has 0555 directories; make them writable in order to remove content. + filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return nil // ignore errors walking in file system + } + if info.IsDir() { + os.Chmod(path, 0777) + } + return nil + }) + return robustio.RemoveAll(dir) +} + var GoSumFile string // path to go.sum; set by package modload type modSum struct { @@ -278,21 +383,22 @@ var goSum struct { } // initGoSum initializes the go.sum data. -// It reports whether use of go.sum is now enabled. +// The boolean it returns reports whether the +// use of go.sum is now enabled. // The goSum lock must be held. -func initGoSum() bool { +func initGoSum() (bool, error) { if GoSumFile == "" { - return false + return false, nil } if goSum.m != nil { - return true + return true, nil } goSum.m = make(map[module.Version][]string) goSum.checked = make(map[modSum]bool) - data, err := ioutil.ReadFile(GoSumFile) + data, err := lockedfile.Read(GoSumFile) if err != nil && !os.IsNotExist(err) { - base.Fatalf("go: %v", err) + return false, err } goSum.enabled = true readGoSum(goSum.m, GoSumFile, data) @@ -300,17 +406,17 @@ func initGoSum() bool { // Add old go.modverify file. // We'll delete go.modverify in WriteGoSum. alt := strings.TrimSuffix(GoSumFile, ".sum") + ".modverify" - if data, err := ioutil.ReadFile(alt); err == nil { + if data, err := renameio.ReadFile(alt); err == nil { migrate := make(map[module.Version][]string) readGoSum(migrate, alt, data) for mod, sums := range migrate { for _, sum := range sums { - checkOneSumLocked(mod, sum) + addModSumLocked(mod, sum) } } goSum.modverify = alt } - return true + return true, nil } // emptyGoModHash is the hash of a 1-file tree containing a 0-length go.mod. @@ -320,7 +426,7 @@ const emptyGoModHash = "h1:G7mAYYxgmS0lVkHyy2hEOLQCFB0DlQFTMLWggykrydY=" // readGoSum parses data, which is the content of file, // and adds it to goSum.m. The goSum lock must be held. -func readGoSum(dst map[module.Version][]string, file string, data []byte) { +func readGoSum(dst map[module.Version][]string, file string, data []byte) error { lineno := 0 for len(data) > 0 { var line []byte @@ -337,7 +443,7 @@ func readGoSum(dst map[module.Version][]string, file string, data []byte) { continue } if len(f) != 3 { - base.Fatalf("go: malformed go.sum:\n%s:%d: wrong number of fields %v", file, lineno, len(f)) + return fmt.Errorf("malformed go.sum:\n%s:%d: wrong number of fields %v", file, lineno, len(f)) } if f[2] == emptyGoModHash { // Old bug; drop it. @@ -346,11 +452,12 @@ func readGoSum(dst map[module.Version][]string, file string, data []byte) { mod := module.Version{Path: f[0], Version: f[1]} dst[mod] = append(dst[mod], f[2]) } + return nil } -// checkSum checks the given module's checksum. -func checkSum(mod module.Version) { - if PkgMod == "" { +// checkMod checks the given module's checksum. +func checkMod(mod module.Version) { + if cfg.GOMODCACHE == "" { // Do not use current directory. return } @@ -358,22 +465,24 @@ func checkSum(mod module.Version) { // Do the file I/O before acquiring the go.sum lock. ziphash, err := CachePath(mod, "ziphash") if err != nil { - base.Fatalf("verifying %s@%s: %v", mod.Path, mod.Version, err) + base.Fatalf("verifying %v", module.VersionError(mod, err)) } - data, err := ioutil.ReadFile(ziphash) + data, err := renameio.ReadFile(ziphash) if err != nil { - if os.IsNotExist(err) { + if errors.Is(err, os.ErrNotExist) { // This can happen if someone does rm -rf GOPATH/src/cache/download. So it goes. return } - base.Fatalf("verifying %s@%s: %v", mod.Path, mod.Version, err) + base.Fatalf("verifying %v", module.VersionError(mod, err)) } h := strings.TrimSpace(string(data)) if !strings.HasPrefix(h, "h1:") { - base.Fatalf("verifying %s@%s: unexpected ziphash: %q", mod.Path, mod.Version, h) + base.Fatalf("verifying %v", module.VersionError(mod, fmt.Errorf("unexpected ziphash: %q", h))) } - checkOneSum(mod, h) + if err := checkModSum(mod, h); err != nil { + base.Fatalf("%s", err) + } } // goModSum returns the checksum for the go.mod contents. @@ -385,46 +494,108 @@ func goModSum(data []byte) (string, error) { // checkGoMod checks the given module's go.mod checksum; // data is the go.mod content. -func checkGoMod(path, version string, data []byte) { +func checkGoMod(path, version string, data []byte) error { h, err := goModSum(data) if err != nil { - base.Fatalf("verifying %s %s go.mod: %v", path, version, err) + return &module.ModuleError{Path: path, Version: version, Err: fmt.Errorf("verifying go.mod: %v", err)} } - checkOneSum(module.Version{Path: path, Version: version + "/go.mod"}, h) + return checkModSum(module.Version{Path: path, Version: version + "/go.mod"}, h) } -// checkOneSum checks that the recorded hash for mod is h. -func checkOneSum(mod module.Version, h string) { +// checkModSum checks that the recorded checksum for mod is h. +func checkModSum(mod module.Version, h string) error { + // We lock goSum when manipulating it, + // but we arrange to release the lock when calling checkSumDB, + // so that parallel calls to checkModHash can execute parallel calls + // to checkSumDB. + + // Check whether mod+h is listed in go.sum already. If so, we're done. goSum.mu.Lock() - defer goSum.mu.Unlock() - if initGoSum() { - checkOneSumLocked(mod, h) + inited, err := initGoSum() + if err != nil { + goSum.mu.Unlock() + return err + } + done := inited && haveModSumLocked(mod, h) + goSum.mu.Unlock() + + if done { + return nil + } + + // Not listed, so we want to add them. + // Consult checksum database if appropriate. + if useSumDB(mod) { + // Calls base.Fatalf if mismatch detected. + if err := checkSumDB(mod, h); err != nil { + return err + } } + + // Add mod+h to go.sum, if it hasn't appeared already. + if inited { + goSum.mu.Lock() + addModSumLocked(mod, h) + goSum.mu.Unlock() + } + return nil } -func checkOneSumLocked(mod module.Version, h string) { +// haveModSumLocked reports whether the pair mod,h is already listed in go.sum. +// If it finds a conflicting pair instead, it calls base.Fatalf. +// goSum.mu must be locked. +func haveModSumLocked(mod module.Version, h string) bool { goSum.checked[modSum{mod, h}] = true - for _, vh := range goSum.m[mod] { if h == vh { - return + return true } if strings.HasPrefix(vh, "h1:") { - base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\tgo.sum: %v", mod.Path, mod.Version, h, vh) + base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\tgo.sum: %v"+goSumMismatch, mod.Path, mod.Version, h, vh) } } + return false +} + +// addModSumLocked adds the pair mod,h to go.sum. +// goSum.mu must be locked. +func addModSumLocked(mod module.Version, h string) { + if haveModSumLocked(mod, h) { + return + } if len(goSum.m[mod]) > 0 { - fmt.Fprintf(os.Stderr, "warning: verifying %s@%s: unknown hashes in go.sum: %v; adding %v", mod.Path, mod.Version, strings.Join(goSum.m[mod], ", "), h) + fmt.Fprintf(os.Stderr, "warning: verifying %s@%s: unknown hashes in go.sum: %v; adding %v"+hashVersionMismatch, mod.Path, mod.Version, strings.Join(goSum.m[mod], ", "), h) } goSum.m[mod] = append(goSum.m[mod], h) goSum.dirty = true } +// checkSumDB checks the mod, h pair against the Go checksum database. +// It calls base.Fatalf if the hash is to be rejected. +func checkSumDB(mod module.Version, h string) error { + db, lines, err := lookupSumDB(mod) + if err != nil { + return module.VersionError(mod, fmt.Errorf("verifying module: %v", err)) + } + + have := mod.Path + " " + mod.Version + " " + h + prefix := mod.Path + " " + mod.Version + " h1:" + for _, line := range lines { + if line == have { + return nil + } + if strings.HasPrefix(line, prefix) { + return module.VersionError(mod, fmt.Errorf("verifying module: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+sumdbMismatch, h, db, line[len(prefix)-len("h1:"):])) + } + } + return nil +} + // Sum returns the checksum for the downloaded copy of the given module, // if present in the download cache. func Sum(mod module.Version) string { - if PkgMod == "" { + if cfg.GOMODCACHE == "" { // Do not use current directory. return "" } @@ -433,7 +604,7 @@ func Sum(mod module.Version) string { if err != nil { return "" } - data, err := ioutil.ReadFile(ziphash) + data, err := renameio.ReadFile(ziphash) if err != nil { return "" } @@ -455,60 +626,49 @@ func WriteGoSum() { // Don't bother opening the go.sum file if we don't have anything to add. return } + if cfg.BuildMod == "readonly" { + base.Fatalf("go: updates to go.sum needed, disabled by -mod=readonly") + } - // We want to avoid races between creating the lockfile and deleting it, but - // we also don't want to leave a permanent lockfile in the user's repository. - // - // On top of that, if we crash while writing go.sum, we don't want to lose the - // sums that were already present in the file, so it's important that we write - // the file by renaming rather than truncating — which means that we can't - // lock the go.sum file itself. - // - // Instead, we'll lock a distinguished file in the cache directory: that will - // only race if the user runs `go clean -modcache` concurrently with a command - // that updates go.sum, and that's already racy to begin with. - // - // We'll end up slightly over-synchronizing go.sum writes if the user runs a - // bunch of go commands that update sums in separate modules simultaneously, - // but that's unlikely to matter in practice. - - unlock := SideLock() - defer unlock() + // Make a best-effort attempt to acquire the side lock, only to exclude + // previous versions of the 'go' command from making simultaneous edits. + if unlock, err := SideLock(); err == nil { + defer unlock() + } - if !goSum.overwrite { - // Re-read the go.sum file to incorporate any sums added by other processes - // in the meantime. - data, err := ioutil.ReadFile(GoSumFile) - if err != nil && !os.IsNotExist(err) { - base.Fatalf("go: re-reading go.sum: %v", err) + err := lockedfile.Transform(GoSumFile, func(data []byte) ([]byte, error) { + if !goSum.overwrite { + // Incorporate any sums added by other processes in the meantime. + // Add only the sums that we actually checked: the user may have edited or + // truncated the file to remove erroneous hashes, and we shouldn't restore + // them without good reason. + goSum.m = make(map[module.Version][]string, len(goSum.m)) + readGoSum(goSum.m, GoSumFile, data) + for ms := range goSum.checked { + addModSumLocked(ms.mod, ms.sum) + goSum.dirty = true + } } - // Add only the sums that we actually checked: the user may have edited or - // truncated the file to remove erroneous hashes, and we shouldn't restore - // them without good reason. - goSum.m = make(map[module.Version][]string, len(goSum.m)) - readGoSum(goSum.m, GoSumFile, data) - for ms := range goSum.checked { - checkOneSumLocked(ms.mod, ms.sum) + var mods []module.Version + for m := range goSum.m { + mods = append(mods, m) } - } - - var mods []module.Version - for m := range goSum.m { - mods = append(mods, m) - } - module.Sort(mods) - var buf bytes.Buffer - for _, m := range mods { - list := goSum.m[m] - sort.Strings(list) - for _, h := range list { - fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h) + module.Sort(mods) + + var buf bytes.Buffer + for _, m := range mods { + list := goSum.m[m] + sort.Strings(list) + for _, h := range list { + fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h) + } } - } + return buf.Bytes(), nil + }) - if err := renameio.WriteFile(GoSumFile, buf.Bytes()); err != nil { - base.Fatalf("go: writing go.sum: %v", err) + if err != nil { + base.Fatalf("go: updating go.sum: %v", err) } goSum.checked = make(map[modSum]bool) @@ -524,7 +684,11 @@ func WriteGoSum() { func TrimGoSum(keep map[module.Version]bool) { goSum.mu.Lock() defer goSum.mu.Unlock() - if !initGoSum() { + inited, err := initGoSum() + if err != nil { + base.Fatalf("%s", err) + } + if !inited { return } @@ -539,3 +703,172 @@ func TrimGoSum(keep map[module.Version]bool) { } } } + +const goSumMismatch = ` + +SECURITY ERROR +This download does NOT match an earlier download recorded in go.sum. +The bits may have been replaced on the origin server, or an attacker may +have intercepted the download attempt. + +For more information, see 'go help module-auth'. +` + +const sumdbMismatch = ` + +SECURITY ERROR +This download does NOT match the one reported by the checksum server. +The bits may have been replaced on the origin server, or an attacker may +have intercepted the download attempt. + +For more information, see 'go help module-auth'. +` + +const hashVersionMismatch = ` + +SECURITY WARNING +This download is listed in go.sum, but using an unknown hash algorithm. +The download cannot be verified. + +For more information, see 'go help module-auth'. + +` + +var HelpModuleAuth = &base.Command{ + UsageLine: "module-auth", + Short: "module authentication using go.sum", + Long: ` +The go command tries to authenticate every downloaded module, +checking that the bits downloaded for a specific module version today +match bits downloaded yesterday. This ensures repeatable builds +and detects introduction of unexpected changes, malicious or not. + +In each module's root, alongside go.mod, the go command maintains +a file named go.sum containing the cryptographic checksums of the +module's dependencies. + +The form of each line in go.sum is three fields: + + [/go.mod] + +Each known module version results in two lines in the go.sum file. +The first line gives the hash of the module version's file tree. +The second line appends "/go.mod" to the version and gives the hash +of only the module version's (possibly synthesized) go.mod file. +The go.mod-only hash allows downloading and authenticating a +module version's go.mod file, which is needed to compute the +dependency graph, without also downloading all the module's source code. + +The hash begins with an algorithm prefix of the form "h:". +The only defined algorithm prefix is "h1:", which uses SHA-256. + +Module authentication failures + +The go command maintains a cache of downloaded packages and computes +and records the cryptographic checksum of each package at download time. +In normal operation, the go command checks the main module's go.sum file +against these precomputed checksums instead of recomputing them on +each command invocation. The 'go mod verify' command checks that +the cached copies of module downloads still match both their recorded +checksums and the entries in go.sum. + +In day-to-day development, the checksum of a given module version +should never change. Each time a dependency is used by a given main +module, the go command checks its local cached copy, freshly +downloaded or not, against the main module's go.sum. If the checksums +don't match, the go command reports the mismatch as a security error +and refuses to run the build. When this happens, proceed with caution: +code changing unexpectedly means today's build will not match +yesterday's, and the unexpected change may not be beneficial. + +If the go command reports a mismatch in go.sum, the downloaded code +for the reported module version does not match the one used in a +previous build of the main module. It is important at that point +to find out what the right checksum should be, to decide whether +go.sum is wrong or the downloaded code is wrong. Usually go.sum is right: +you want to use the same code you used yesterday. + +If a downloaded module is not yet included in go.sum and it is a publicly +available module, the go command consults the Go checksum database to fetch +the expected go.sum lines. If the downloaded code does not match those +lines, the go command reports the mismatch and exits. Note that the +database is not consulted for module versions already listed in go.sum. + +If a go.sum mismatch is reported, it is always worth investigating why +the code downloaded today differs from what was downloaded yesterday. + +The GOSUMDB environment variable identifies the name of checksum database +to use and optionally its public key and URL, as in: + + GOSUMDB="sum.golang.org" + GOSUMDB="sum.golang.org+" + GOSUMDB="sum.golang.org+ https://sum.golang.org" + +The go command knows the public key of sum.golang.org, and also that the name +sum.golang.google.cn (available inside mainland China) connects to the +sum.golang.org checksum database; use of any other database requires giving +the public key explicitly. +The URL defaults to "https://" followed by the database name. + +GOSUMDB defaults to "sum.golang.org", the Go checksum database run by Google. +See https://sum.golang.org/privacy for the service's privacy policy. + +If GOSUMDB is set to "off", or if "go get" is invoked with the -insecure flag, +the checksum database is not consulted, and all unrecognized modules are +accepted, at the cost of giving up the security guarantee of verified repeatable +downloads for all modules. A better way to bypass the checksum database +for specific modules is to use the GOPRIVATE or GONOSUMDB environment +variables. See 'go help module-private' for details. + +The 'go env -w' command (see 'go help env') can be used to set these variables +for future go command invocations. +`, +} + +var HelpModulePrivate = &base.Command{ + UsageLine: "module-private", + Short: "module configuration for non-public modules", + Long: ` +The go command defaults to downloading modules from the public Go module +mirror at proxy.golang.org. It also defaults to validating downloaded modules, +regardless of source, against the public Go checksum database at sum.golang.org. +These defaults work well for publicly available source code. + +The GOPRIVATE environment variable controls which modules the go command +considers to be private (not available publicly) and should therefore not use the +proxy or checksum database. The variable is a comma-separated list of +glob patterns (in the syntax of Go's path.Match) of module path prefixes. +For example, + + GOPRIVATE=*.corp.example.com,rsc.io/private + +causes the go command to treat as private any module with a path prefix +matching either pattern, including git.corp.example.com/xyzzy, rsc.io/private, +and rsc.io/private/quux. + +The GOPRIVATE environment variable may be used by other tools as well to +identify non-public modules. For example, an editor could use GOPRIVATE +to decide whether to hyperlink a package import to a godoc.org page. + +For fine-grained control over module download and validation, the GONOPROXY +and GONOSUMDB environment variables accept the same kind of glob list +and override GOPRIVATE for the specific decision of whether to use the proxy +and checksum database, respectively. + +For example, if a company ran a module proxy serving private modules, +users would configure go using: + + GOPRIVATE=*.corp.example.com + GOPROXY=proxy.example.com + GONOPROXY=none + +This would tell the go command and other tools that modules beginning with +a corp.example.com subdomain are private but that the company proxy should +be used for downloading both public and private modules, because +GONOPROXY has been set to a pattern that won't match any modules, +overriding GOPRIVATE. + +The 'go env -w' command (see 'go help env') can be used to set these variables +for future go command invocations. +`, +} diff --git a/cmd/go/_internal_/modfetch/insecure.go b/cmd/go/_internal_/modfetch/insecure.go new file mode 100644 index 0000000..6c5f5fd --- /dev/null +++ b/cmd/go/_internal_/modfetch/insecure.go @@ -0,0 +1,16 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modfetch + +import ( + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/get" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/str" +) + +// allowInsecure reports whether we are allowed to fetch this path in an insecure manner. +func allowInsecure(path string) bool { + return get.Insecure || str.GlobsMatchPath(cfg.GOINSECURE, path) +} diff --git a/cmd/go/_internal_/modfetch/key.go b/cmd/go/_internal_/modfetch/key.go new file mode 100644 index 0000000..06f9989 --- /dev/null +++ b/cmd/go/_internal_/modfetch/key.go @@ -0,0 +1,9 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modfetch + +var knownGOSUMDB = map[string]string{ + "sum.golang.org": "sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8", +} diff --git a/cmd/go/_internal_/modfetch/proxy.go b/cmd/go/_internal_/modfetch/proxy.go index ad48134..ba295b7 100644 --- a/cmd/go/_internal_/modfetch/proxy.go +++ b/cmd/go/_internal_/modfetch/proxy.go @@ -6,33 +6,32 @@ package modfetch import ( "encoding/json" + "errors" "fmt" "io" + "io/ioutil" "net/url" "os" + "path" + pathpkg "path" + "path/filepath" "strings" + "sync" "time" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch/codehost" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/module" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/semver" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/web" + + "golang.org/x/mod/module" + "golang.org/x/mod/semver" ) var HelpGoproxy = &base.Command{ UsageLine: "goproxy", Short: "module proxy protocol", Long: ` -The go command by default downloads modules from version control systems -directly, just as 'go get' always has. The GOPROXY environment variable allows -further control over the download source. If GOPROXY is unset, is the empty string, -or is the string "direct", downloads use the default direct connection to version -control systems. Setting GOPROXY to "off" disallows downloading modules from -any source. Otherwise, GOPROXY is expected to be the URL of a module proxy, -in which case the go command will fetch all modules from that proxy. -No matter the source of the modules, downloaded modules must match existing -entries in go.sum (see 'go help modules' for discussion of verification). - A Go module proxy is any web server that can respond to GET requests for URLs of a specified form. The requests have no query parameters, so even a site serving from a fixed file system (including a file:/// URL) @@ -40,8 +39,8 @@ can be a module proxy. The GET requests sent to a Go module proxy are: -GET $GOPROXY//@v/list returns a list of all known versions of the -given module, one per line. +GET $GOPROXY//@v/list returns a list of known versions of the given +module, one per line. GET $GOPROXY//@v/.info returns JSON-formatted metadata about that version of the given module. @@ -52,6 +51,21 @@ for that version of the given module. GET $GOPROXY//@v/.zip returns the zip archive for that version of the given module. +GET $GOPROXY//@latest returns JSON-formatted metadata about the +latest known version of the given module in the same format as +/@v/.info. The latest version should be the version of +the module the go command may use if /@v/list is empty or no +listed version is suitable. /@latest is optional and may not +be implemented by a module proxy. + +When resolving the latest version of a module, the go command will request +/@v/list, then, if no suitable versions are found, /@latest. +The go command prefers, in order: the semantically highest release version, +the semantically highest pre-release version, and the chronologically +most recent pseudo-version. In Go 1.12 and earlier, the go command considered +pseudo-versions in /@v/list to be pre-release versions, but this is +no longer true since Go 1.13. + To avoid problems when serving from case-sensitive file systems, the and elements are case-encoded, replacing every uppercase letter with an exclamation mark followed by the corresponding @@ -85,47 +99,241 @@ cached module versions with GOPROXY=https://example.com/proxy. `, } -var proxyURL = os.Getenv("GOPROXY") +var proxyOnce struct { + sync.Once + list []proxySpec + err error +} + +type proxySpec struct { + // url is the proxy URL or one of "off", "direct", "noproxy". + url string + + // fallBackOnError is true if a request should be attempted on the next proxy + // in the list after any error from this proxy. If fallBackOnError is false, + // the request will only be attempted on the next proxy if the error is + // equivalent to os.ErrNotFound, which is true for 404 and 410 responses. + fallBackOnError bool +} + +func proxyList() ([]proxySpec, error) { + proxyOnce.Do(func() { + if cfg.GONOPROXY != "" && cfg.GOPROXY != "direct" { + proxyOnce.list = append(proxyOnce.list, proxySpec{url: "noproxy"}) + } + + goproxy := cfg.GOPROXY + for goproxy != "" { + var url string + fallBackOnError := false + if i := strings.IndexAny(goproxy, ",|"); i >= 0 { + url = goproxy[:i] + fallBackOnError = goproxy[i] == '|' + goproxy = goproxy[i+1:] + } else { + url = goproxy + goproxy = "" + } + + url = strings.TrimSpace(url) + if url == "" { + continue + } + if url == "off" { + // "off" always fails hard, so can stop walking list. + proxyOnce.list = append(proxyOnce.list, proxySpec{url: "off"}) + break + } + if url == "direct" { + proxyOnce.list = append(proxyOnce.list, proxySpec{url: "direct"}) + // For now, "direct" is the end of the line. We may decide to add some + // sort of fallback behavior for them in the future, so ignore + // subsequent entries for forward-compatibility. + break + } + + // Single-word tokens are reserved for built-in behaviors, and anything + // containing the string ":/" or matching an absolute file path must be a + // complete URL. For all other paths, implicitly add "https://". + if strings.ContainsAny(url, ".:/") && !strings.Contains(url, ":/") && !filepath.IsAbs(url) && !path.IsAbs(url) { + url = "https://" + url + } + + // Check that newProxyRepo accepts the URL. + // It won't do anything with the path. + if _, err := newProxyRepo(url, "golang.org/x/text"); err != nil { + proxyOnce.err = err + return + } + + proxyOnce.list = append(proxyOnce.list, proxySpec{ + url: url, + fallBackOnError: fallBackOnError, + }) + } -func lookupProxy(path string) (Repo, error) { - if strings.Contains(proxyURL, ",") { - return nil, fmt.Errorf("invalid $GOPROXY setting: cannot have comma") + if len(proxyOnce.list) == 0 || + len(proxyOnce.list) == 1 && proxyOnce.list[0].url == "noproxy" { + // There were no proxies, other than the implicit "noproxy" added when + // GONOPROXY is set. This can happen if GOPROXY is a non-empty string + // like "," or " ". + proxyOnce.err = fmt.Errorf("GOPROXY list is not the empty string, but contains no entries") + } + }) + + return proxyOnce.list, proxyOnce.err +} + +// TryProxies iterates f over each configured proxy (including "noproxy" and +// "direct" if applicable) until f returns no error or until f returns an +// error that is not equivalent to os.ErrNotExist on a proxy configured +// not to fall back on errors. +// +// TryProxies then returns that final error. +// +// If GOPROXY is set to "off", TryProxies invokes f once with the argument +// "off". +func TryProxies(f func(proxy string) error) error { + proxies, err := proxyList() + if err != nil { + return err } - u, err := url.Parse(proxyURL) - if err != nil || u.Scheme != "http" && u.Scheme != "https" && u.Scheme != "file" { - // Don't echo $GOPROXY back in case it has user:password in it (sigh). - return nil, fmt.Errorf("invalid $GOPROXY setting: malformed URL or invalid scheme (must be http, https, file)") + if len(proxies) == 0 { + panic("GOPROXY list is empty") + } + + // We try to report the most helpful error to the user. "direct" and "noproxy" + // errors are best, followed by proxy errors other than ErrNotExist, followed + // by ErrNotExist. + // + // Note that errProxyOff, errNoproxy, and errUseProxy are equivalent to + // ErrNotExist. errUseProxy should only be returned if "noproxy" is the only + // proxy. errNoproxy should never be returned, since there should always be a + // more useful error from "noproxy" first. + const ( + notExistRank = iota + proxyRank + directRank + ) + var bestErr error + bestErrRank := notExistRank + for _, proxy := range proxies { + err := f(proxy.url) + if err == nil { + return nil + } + isNotExistErr := errors.Is(err, os.ErrNotExist) + + if proxy.url == "direct" || (proxy.url == "noproxy" && err != errUseProxy) { + bestErr = err + bestErrRank = directRank + } else if bestErrRank <= proxyRank && !isNotExistErr { + bestErr = err + bestErrRank = proxyRank + } else if bestErrRank == notExistRank { + bestErr = err + } + + if !proxy.fallBackOnError && !isNotExistErr { + break + } } - return newProxyRepo(u.String(), path) + return bestErr } type proxyRepo struct { - url string + url *url.URL path string } func newProxyRepo(baseURL, path string) (Repo, error) { - enc, err := module.EncodePath(path) + base, err := url.Parse(baseURL) if err != nil { return nil, err } - return &proxyRepo{strings.TrimSuffix(baseURL, "/") + "/" + pathEscape(enc), path}, nil + switch base.Scheme { + case "http", "https": + // ok + case "file": + if *base != (url.URL{Scheme: base.Scheme, Path: base.Path, RawPath: base.RawPath}) { + return nil, fmt.Errorf("invalid file:// proxy URL with non-path elements: %s", base.Redacted()) + } + case "": + return nil, fmt.Errorf("invalid proxy URL missing scheme: %s", base.Redacted()) + default: + return nil, fmt.Errorf("invalid proxy URL scheme (must be https, http, file): %s", base.Redacted()) + } + + enc, err := module.EscapePath(path) + if err != nil { + return nil, err + } + + base.Path = strings.TrimSuffix(base.Path, "/") + "/" + enc + base.RawPath = strings.TrimSuffix(base.RawPath, "/") + "/" + pathEscape(enc) + return &proxyRepo{base, path}, nil } func (p *proxyRepo) ModulePath() string { return p.path } -func (p *proxyRepo) Versions(prefix string) ([]string, error) { - var data []byte - err := webGetBytes(p.url+"/@v/list", &data) +// versionError returns err wrapped in a ModuleError for p.path. +func (p *proxyRepo) versionError(version string, err error) error { + if version != "" && version != module.CanonicalVersion(version) { + return &module.ModuleError{ + Path: p.path, + Err: &module.InvalidVersionError{ + Version: version, + Pseudo: IsPseudoVersion(version), + Err: err, + }, + } + } + + return &module.ModuleError{ + Path: p.path, + Version: version, + Err: err, + } +} + +func (p *proxyRepo) getBytes(path string) ([]byte, error) { + body, err := p.getBody(path) if err != nil { return nil, err } + defer body.Close() + return ioutil.ReadAll(body) +} + +func (p *proxyRepo) getBody(path string) (io.ReadCloser, error) { + fullPath := pathpkg.Join(p.url.Path, path) + + target := *p.url + target.Path = fullPath + target.RawPath = pathpkg.Join(target.RawPath, pathEscape(path)) + + resp, err := web.Get(web.DefaultSecurity, &target) + if err != nil { + return nil, err + } + if err := resp.Err(); err != nil { + resp.Body.Close() + return nil, err + } + return resp.Body, nil +} + +func (p *proxyRepo) Versions(prefix string) ([]string, error) { + data, err := p.getBytes("@v/list") + if err != nil { + return nil, p.versionError("", err) + } var list []string for _, line := range strings.Split(string(data), "\n") { f := strings.Fields(line) - if len(f) >= 1 && semver.IsValid(f[0]) && strings.HasPrefix(f[0], prefix) { + if len(f) >= 1 && semver.IsValid(f[0]) && strings.HasPrefix(f[0], prefix) && !IsPseudoVersion(f[0]) { list = append(list, f[0]) } } @@ -134,98 +342,140 @@ func (p *proxyRepo) Versions(prefix string) ([]string, error) { } func (p *proxyRepo) latest() (*RevInfo, error) { - var data []byte - err := webGetBytes(p.url+"/@v/list", &data) + data, err := p.getBytes("@v/list") if err != nil { - return nil, err + return nil, p.versionError("", err) } - var best time.Time - var bestVersion string + + var ( + bestTime time.Time + bestTimeIsFromPseudo bool + bestVersion string + ) + for _, line := range strings.Split(string(data), "\n") { f := strings.Fields(line) - if len(f) >= 2 && semver.IsValid(f[0]) { - ft, err := time.Parse(time.RFC3339, f[1]) - if err == nil && best.Before(ft) { - best = ft + if len(f) >= 1 && semver.IsValid(f[0]) { + // If the proxy includes timestamps, prefer the timestamp it reports. + // Otherwise, derive the timestamp from the pseudo-version. + var ( + ft time.Time + ftIsFromPseudo = false + ) + if len(f) >= 2 { + ft, _ = time.Parse(time.RFC3339, f[1]) + } else if IsPseudoVersion(f[0]) { + ft, _ = PseudoVersionTime(f[0]) + ftIsFromPseudo = true + } else { + // Repo.Latest promises that this method is only called where there are + // no tagged versions. Ignore any tagged versions that were added in the + // meantime. + continue + } + if bestTime.Before(ft) { + bestTime = ft + bestTimeIsFromPseudo = ftIsFromPseudo bestVersion = f[0] } } } if bestVersion == "" { - return nil, fmt.Errorf("no commits") + return nil, p.versionError("", codehost.ErrNoCommits) + } + + if bestTimeIsFromPseudo { + // We parsed bestTime from the pseudo-version, but that's in UTC and we're + // supposed to report the timestamp as reported by the VCS. + // Stat the selected version to canonicalize the timestamp. + // + // TODO(bcmills): Should we also stat other versions to ensure that we + // report the correct Name and Short for the revision? + return p.Stat(bestVersion) } - info := &RevInfo{ + + return &RevInfo{ Version: bestVersion, Name: bestVersion, Short: bestVersion, - Time: best, - } - return info, nil + Time: bestTime, + }, nil } func (p *proxyRepo) Stat(rev string) (*RevInfo, error) { - var data []byte - encRev, err := module.EncodeVersion(rev) + encRev, err := module.EscapeVersion(rev) if err != nil { - return nil, err + return nil, p.versionError(rev, err) } - err = webGetBytes(p.url+"/@v/"+pathEscape(encRev)+".info", &data) + data, err := p.getBytes("@v/" + encRev + ".info") if err != nil { - return nil, err + return nil, p.versionError(rev, err) } info := new(RevInfo) if err := json.Unmarshal(data, info); err != nil { - return nil, err + return nil, p.versionError(rev, err) + } + if info.Version != rev && rev == module.CanonicalVersion(rev) && module.Check(p.path, rev) == nil { + // If we request a correct, appropriate version for the module path, the + // proxy must return either exactly that version or an error — not some + // arbitrary other version. + return nil, p.versionError(rev, fmt.Errorf("proxy returned info for version %s instead of requested version", info.Version)) } return info, nil } func (p *proxyRepo) Latest() (*RevInfo, error) { - var data []byte - u := p.url + "/@latest" - err := webGetBytes(u, &data) + data, err := p.getBytes("@latest") if err != nil { - // TODO return err if not 404 + if !errors.Is(err, os.ErrNotExist) { + return nil, p.versionError("", err) + } return p.latest() } info := new(RevInfo) if err := json.Unmarshal(data, info); err != nil { - return nil, err + return nil, p.versionError("", err) } return info, nil } func (p *proxyRepo) GoMod(version string) ([]byte, error) { - var data []byte - encVer, err := module.EncodeVersion(version) + if version != module.CanonicalVersion(version) { + return nil, p.versionError(version, fmt.Errorf("internal error: version passed to GoMod is not canonical")) + } + + encVer, err := module.EscapeVersion(version) if err != nil { - return nil, err + return nil, p.versionError(version, err) } - err = webGetBytes(p.url+"/@v/"+pathEscape(encVer)+".mod", &data) + data, err := p.getBytes("@v/" + encVer + ".mod") if err != nil { - return nil, err + return nil, p.versionError(version, err) } return data, nil } func (p *proxyRepo) Zip(dst io.Writer, version string) error { - var body io.ReadCloser - encVer, err := module.EncodeVersion(version) + if version != module.CanonicalVersion(version) { + return p.versionError(version, fmt.Errorf("internal error: version passed to Zip is not canonical")) + } + + encVer, err := module.EscapeVersion(version) if err != nil { - return err + return p.versionError(version, err) } - err = webGetBody(p.url+"/@v/"+pathEscape(encVer)+".zip", &body) + body, err := p.getBody("@v/" + encVer + ".zip") if err != nil { - return err + return p.versionError(version, err) } defer body.Close() lr := &io.LimitedReader{R: body, N: codehost.MaxZipFile + 1} if _, err := io.Copy(dst, lr); err != nil { - return err + return p.versionError(version, err) } if lr.N <= 0 { - return fmt.Errorf("downloaded zip file too large") + return p.versionError(version, fmt.Errorf("downloaded zip file too large")) } return nil } diff --git a/cmd/go/_internal_/modfetch/pseudo.go b/cmd/go/_internal_/modfetch/pseudo.go index 7207763..f402cbe 100644 --- a/cmd/go/_internal_/modfetch/pseudo.go +++ b/cmd/go/_internal_/modfetch/pseudo.go @@ -35,13 +35,21 @@ package modfetch import ( - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/semver" + "errors" "fmt" - "regexp" "strings" "time" + + "github.com/dependabot/gomodules-extracted/_internal_/lazyregexp" + + "golang.org/x/mod/module" + "golang.org/x/mod/semver" ) +var pseudoVersionRE = lazyregexp.New(`^v[0-9]+\.(0\.0-|\d+\.\d+-([^+]*\.)?0\.)\d{14}-[A-Za-z0-9]+(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$`) + +const pseudoVersionTimestampFormat = "20060102150405" + // PseudoVersion returns a pseudo-version for the given major version ("v1") // preexisting older tagged version ("" or "v1.2.3" or "v1.2.3-pre"), revision time, // and revision identifier (usually a 12-byte commit hash prefix). @@ -49,8 +57,7 @@ func PseudoVersion(major, older string, t time.Time, rev string) string { if major == "" { major = "v0" } - major = strings.TrimSuffix(major, "-unstable") // make gopkg.in/macaroon-bakery.v2-unstable use "v2" - segment := fmt.Sprintf("%s-%s", t.UTC().Format("20060102150405"), rev) + segment := fmt.Sprintf("%s-%s", t.UTC().Format(pseudoVersionTimestampFormat), rev) build := semver.Build(older) older = semver.Canonical(older) if older == "" { @@ -62,15 +69,19 @@ func PseudoVersion(major, older string, t time.Time, rev string) string { // Form (2), (3). // Extract patch from vMAJOR.MINOR.PATCH - v := older[:len(older)] - i := strings.LastIndex(v, ".") + 1 - v, patch := v[:i], v[i:] - - // Increment PATCH by adding 1 to decimal: - // scan right to left turning 9s to 0s until you find a digit to increment. - // (Number might exceed int64, but math/big is overkill.) - digits := []byte(patch) - for i = len(digits) - 1; i >= 0 && digits[i] == '9'; i-- { + i := strings.LastIndex(older, ".") + 1 + v, patch := older[:i], older[i:] + + // Reassemble. + return v + incDecimal(patch) + "-0." + segment + build +} + +// incDecimal returns the decimal string incremented by 1. +func incDecimal(decimal string) string { + // Scan right to left turning 9s to 0s until you find a digit to increment. + digits := []byte(decimal) + i := len(digits) - 1 + for ; i >= 0 && digits[i] == '9'; i-- { digits[i] = '0' } if i >= 0 { @@ -80,13 +91,29 @@ func PseudoVersion(major, older string, t time.Time, rev string) string { digits[0] = '1' digits = append(digits, '0') } - patch = string(digits) - - // Reassemble. - return v + patch + "-0." + segment + build + return string(digits) } -var pseudoVersionRE = regexp.MustCompile(`^v[0-9]+\.(0\.0-|\d+\.\d+-([^+]*\.)?0\.)\d{14}-[A-Za-z0-9]+(\+incompatible)?$`) +// decDecimal returns the decimal string decremented by 1, or the empty string +// if the decimal is all zeroes. +func decDecimal(decimal string) string { + // Scan right to left turning 0s to 9s until you find a digit to decrement. + digits := []byte(decimal) + i := len(digits) - 1 + for ; i >= 0 && digits[i] == '0'; i-- { + digits[i] = '9' + } + if i < 0 { + // decimal is all zeros + return "" + } + if i == 0 && digits[i] == '1' && len(digits) > 1 { + digits = digits[1:] + } else { + digits[i]-- + } + return string(digits) +} // IsPseudoVersion reports whether v is a pseudo-version. func IsPseudoVersion(v string) bool { @@ -97,10 +124,17 @@ func IsPseudoVersion(v string) bool { // It returns an error if v is not a pseudo-version or if the time stamp // embedded in the pseudo-version is not a valid time. func PseudoVersionTime(v string) (time.Time, error) { - timestamp, _, err := parsePseudoVersion(v) + _, timestamp, _, _, err := parsePseudoVersion(v) + if err != nil { + return time.Time{}, err + } t, err := time.Parse("20060102150405", timestamp) if err != nil { - return time.Time{}, fmt.Errorf("pseudo-version with malformed time %s: %q", timestamp, v) + return time.Time{}, &module.InvalidVersionError{ + Version: v, + Pseudo: true, + Err: fmt.Errorf("malformed time %q", timestamp), + } } return t, nil } @@ -108,22 +142,99 @@ func PseudoVersionTime(v string) (time.Time, error) { // PseudoVersionRev returns the revision identifier of the pseudo-version v. // It returns an error if v is not a pseudo-version. func PseudoVersionRev(v string) (rev string, err error) { - _, rev, err = parsePseudoVersion(v) + _, _, rev, _, err = parsePseudoVersion(v) return } -func parsePseudoVersion(v string) (timestamp, rev string, err error) { +// PseudoVersionBase returns the canonical parent version, if any, upon which +// the pseudo-version v is based. +// +// If v has no parent version (that is, if it is "vX.0.0-[…]"), +// PseudoVersionBase returns the empty string and a nil error. +func PseudoVersionBase(v string) (string, error) { + base, _, _, build, err := parsePseudoVersion(v) + if err != nil { + return "", err + } + + switch pre := semver.Prerelease(base); pre { + case "": + // vX.0.0-yyyymmddhhmmss-abcdef123456 → "" + if build != "" { + // Pseudo-versions of the form vX.0.0-yyyymmddhhmmss-abcdef123456+incompatible + // are nonsensical: the "vX.0.0-" prefix implies that there is no parent tag, + // but the "+incompatible" suffix implies that the major version of + // the parent tag is not compatible with the module's import path. + // + // There are a few such entries in the index generated by proxy.golang.org, + // but we believe those entries were generated by the proxy itself. + return "", &module.InvalidVersionError{ + Version: v, + Pseudo: true, + Err: fmt.Errorf("lacks base version, but has build metadata %q", build), + } + } + return "", nil + + case "-0": + // vX.Y.(Z+1)-0.yyyymmddhhmmss-abcdef123456 → vX.Y.Z + // vX.Y.(Z+1)-0.yyyymmddhhmmss-abcdef123456+incompatible → vX.Y.Z+incompatible + base = strings.TrimSuffix(base, pre) + i := strings.LastIndexByte(base, '.') + if i < 0 { + panic("base from parsePseudoVersion missing patch number: " + base) + } + patch := decDecimal(base[i+1:]) + if patch == "" { + // vX.0.0-0 is invalid, but has been observed in the wild in the index + // generated by requests to proxy.golang.org. + // + // NOTE(bcmills): I cannot find a historical bug that accounts for + // pseudo-versions of this form, nor have I seen such versions in any + // actual go.mod files. If we find actual examples of this form and a + // reasonable theory of how they came into existence, it seems fine to + // treat them as equivalent to vX.0.0 (especially since the invalid + // pseudo-versions have lower precedence than the real ones). For now, we + // reject them. + return "", &module.InvalidVersionError{ + Version: v, + Pseudo: true, + Err: fmt.Errorf("version before %s would have negative patch number", base), + } + } + return base[:i+1] + patch + build, nil + + default: + // vX.Y.Z-pre.0.yyyymmddhhmmss-abcdef123456 → vX.Y.Z-pre + // vX.Y.Z-pre.0.yyyymmddhhmmss-abcdef123456+incompatible → vX.Y.Z-pre+incompatible + if !strings.HasSuffix(base, ".0") { + panic(`base from parsePseudoVersion missing ".0" before date: ` + base) + } + return strings.TrimSuffix(base, ".0") + build, nil + } +} + +var errPseudoSyntax = errors.New("syntax error") + +func parsePseudoVersion(v string) (base, timestamp, rev, build string, err error) { if !IsPseudoVersion(v) { - return "", "", fmt.Errorf("malformed pseudo-version %q", v) + return "", "", "", "", &module.InvalidVersionError{ + Version: v, + Pseudo: true, + Err: errPseudoSyntax, + } } - v = strings.TrimSuffix(v, "+incompatible") + build = semver.Build(v) + v = strings.TrimSuffix(v, build) j := strings.LastIndex(v, "-") v, rev = v[:j], v[j+1:] i := strings.LastIndex(v, "-") if j := strings.LastIndex(v, "."); j > i { + base = v[:j] // "vX.Y.Z-pre.0" or "vX.Y.(Z+1)-0" timestamp = v[j+1:] } else { + base = v[:i] // "vX.0.0" timestamp = v[i+1:] } - return timestamp, rev, nil + return base, timestamp, rev, build, nil } diff --git a/cmd/go/_internal_/modfetch/repo.go b/cmd/go/_internal_/modfetch/repo.go index 315efc1..b94916a 100644 --- a/cmd/go/_internal_/modfetch/repo.go +++ b/cmd/go/_internal_/modfetch/repo.go @@ -16,8 +16,10 @@ import ( "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/get" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch/codehost" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/par" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/semver" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/str" web "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/web" + + "golang.org/x/mod/semver" ) const traceRepo = false // trace all repo actions, for debugging @@ -32,7 +34,7 @@ type Repo interface { // Pseudo-versions are not included. // Versions should be returned sorted in semver order // (implementations can use SortVersions). - Versions(prefix string) (tags []string, err error) + Versions(prefix string) ([]string, error) // Stat returns information about the revision rev. // A revision can be any identifier known to the underlying service: @@ -53,7 +55,7 @@ type Repo interface { // A Rev describes a single revision in a module repository. type RevInfo struct { - Version string // version string + Version string // suggested version string for this revision Time time.Time // commit time // These fields are used for Stat of arbitrary rev, @@ -160,12 +162,6 @@ type RevInfo struct { // To avoid version control access except when absolutely necessary, // Lookup does not attempt to connect to the repository itself. // -// The Import function takes an import path found in source code and -// determines which module to add to the requirement list to satisfy -// that import. It checks successive truncations of the import path -// to determine possible modules and stops when it finds a module -// in which the latest version satisfies the import path. -// // The ImportRepoRev function is a variant of Import which is limited // to code in a source code repository at a particular revision identifier // (usually a commit hash or source code repository tag, not necessarily @@ -177,20 +173,32 @@ type RevInfo struct { var lookupCache par.Cache -// Lookup returns the module with the given module path. +type lookupCacheKey struct { + proxy, path string +} + +// Lookup returns the module with the given module path, +// fetched through the given proxy. +// +// The distinguished proxy "direct" indicates that the path should be fetched +// from its origin, and "noproxy" indicates that the patch should be fetched +// directly only if GONOPROXY matches the given path. +// +// For the distinguished proxy "off", Lookup always returns a non-nil error. +// // A successful return does not guarantee that the module // has any defined versions. -func Lookup(path string) (Repo, error) { +func Lookup(proxy, path string) (Repo, error) { if traceRepo { - defer logCall("Lookup(%q)", path)() + defer logCall("Lookup(%q, %q)", proxy, path)() } type cached struct { r Repo err error } - c := lookupCache.Do(path, func() interface{} { - r, err := lookup(path) + c := lookupCache.Do(lookupCacheKey{proxy, path}, func() interface{} { + r, err := lookup(proxy, path) if err == nil { if traceRepo { r = newLoggingRepo(r) @@ -204,25 +212,59 @@ func Lookup(path string) (Repo, error) { } // lookup returns the module with the given module path. -func lookup(path string) (r Repo, err error) { +func lookup(proxy, path string) (r Repo, err error) { if cfg.BuildMod == "vendor" { - return nil, fmt.Errorf("module lookup disabled by -mod=%s", cfg.BuildMod) + return nil, errLookupDisabled } - if proxyURL == "off" { - return nil, fmt.Errorf("module lookup disabled by GOPROXY=%s", proxyURL) + + if str.GlobsMatchPath(cfg.GONOPROXY, path) { + switch proxy { + case "noproxy", "direct": + return lookupDirect(path) + default: + return nil, errNoproxy + } } - if proxyURL != "" && proxyURL != "direct" { - return lookupProxy(path) + + switch proxy { + case "off": + return nil, errProxyOff + case "direct": + return lookupDirect(path) + case "noproxy": + return nil, errUseProxy + default: + return newProxyRepo(proxy, path) } +} + +type lookupDisabledError struct{} + +func (lookupDisabledError) Error() string { + if cfg.BuildModReason == "" { + return fmt.Sprintf("module lookup disabled by -mod=%s", cfg.BuildMod) + } + return fmt.Sprintf("module lookup disabled by -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) +} - security := web.Secure - if get.Insecure { +var errLookupDisabled error = lookupDisabledError{} + +var ( + errProxyOff = notExistErrorf("module lookup disabled by GOPROXY=off") + errNoproxy error = notExistErrorf("disabled by GOPRIVATE/GONOPROXY") + errUseProxy error = notExistErrorf("path does not match GOPRIVATE/GONOPROXY") +) + +func lookupDirect(path string) (Repo, error) { + security := web.SecureOnly + + if allowInsecure(path) { security = web.Insecure } rr, err := get.RepoRootForImportPath(path, get.PreferMod, security) if err != nil { // We don't know where to find code for a module with this path. - return nil, err + return nil, notExistError{err: err} } if rr.VCS == "mod" { @@ -260,8 +302,8 @@ func ImportRepoRev(path, rev string) (Repo, *RevInfo, error) { // Note: Because we are converting a code reference from a legacy // version control system, we ignore meta tags about modules // and use only direct source control entries (get.IgnoreMod). - security := web.Secure - if get.Insecure { + security := web.SecureOnly + if allowInsecure(path) { security = web.Insecure } rr, err := get.RepoRootForImportPath(path, get.IgnoreMod, security) @@ -288,7 +330,7 @@ func ImportRepoRev(path, rev string) (Repo, *RevInfo, error) { return nil, nil, err } - info, err := repo.(*codeRepo).convert(revInfo, "") + info, err := repo.(*codeRepo).convert(revInfo, rev) if err != nil { return nil, nil, err } @@ -364,3 +406,24 @@ func (l *loggingRepo) Zip(dst io.Writer, version string) error { defer logCall("Repo[%s]: Zip(%s, %q)", l.r.ModulePath(), dstName, version)() return l.r.Zip(dst, version) } + +// A notExistError is like os.ErrNotExist, but with a custom message +type notExistError struct { + err error +} + +func notExistErrorf(format string, args ...interface{}) error { + return notExistError{fmt.Errorf(format, args...)} +} + +func (e notExistError) Error() string { + return e.err.Error() +} + +func (notExistError) Is(target error) bool { + return target == os.ErrNotExist +} + +func (e notExistError) Unwrap() error { + return e.err +} diff --git a/cmd/go/_internal_/modfetch/sumdb.go b/cmd/go/_internal_/modfetch/sumdb.go new file mode 100644 index 0000000..37e3087 --- /dev/null +++ b/cmd/go/_internal_/modfetch/sumdb.go @@ -0,0 +1,280 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Go checksum database lookup + +// +build !cmd_go_bootstrap + +package modfetch + +import ( + "bytes" + "errors" + "fmt" + "io/ioutil" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/get" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/lockedfile" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/str" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/web" + + "golang.org/x/mod/module" + "golang.org/x/mod/sumdb" + "golang.org/x/mod/sumdb/note" +) + +// useSumDB reports whether to use the Go checksum database for the given module. +func useSumDB(mod module.Version) bool { + return cfg.GOSUMDB != "off" && !get.Insecure && !str.GlobsMatchPath(cfg.GONOSUMDB, mod.Path) +} + +// lookupSumDB returns the Go checksum database's go.sum lines for the given module, +// along with the name of the database. +func lookupSumDB(mod module.Version) (dbname string, lines []string, err error) { + dbOnce.Do(func() { + dbName, db, dbErr = dbDial() + }) + if dbErr != nil { + return "", nil, dbErr + } + lines, err = db.Lookup(mod.Path, mod.Version) + return dbName, lines, err +} + +var ( + dbOnce sync.Once + dbName string + db *sumdb.Client + dbErr error +) + +func dbDial() (dbName string, db *sumdb.Client, err error) { + // $GOSUMDB can be "key" or "key url", + // and the key can be a full verifier key + // or a host on our list of known keys. + + // Special case: sum.golang.google.cn + // is an alias, reachable inside mainland China, + // for sum.golang.org. If there are more + // of these we should add a map like knownGOSUMDB. + gosumdb := cfg.GOSUMDB + if gosumdb == "sum.golang.google.cn" { + gosumdb = "sum.golang.org https://sum.golang.google.cn" + } + + key := strings.Fields(gosumdb) + if len(key) >= 1 { + if k := knownGOSUMDB[key[0]]; k != "" { + key[0] = k + } + } + if len(key) == 0 { + return "", nil, fmt.Errorf("missing GOSUMDB") + } + if len(key) > 2 { + return "", nil, fmt.Errorf("invalid GOSUMDB: too many fields") + } + vkey, err := note.NewVerifier(key[0]) + if err != nil { + return "", nil, fmt.Errorf("invalid GOSUMDB: %v", err) + } + name := vkey.Name() + + // No funny business in the database name. + direct, err := url.Parse("https://" + name) + if err != nil || strings.HasSuffix(name, "/") || *direct != (url.URL{Scheme: "https", Host: direct.Host, Path: direct.Path, RawPath: direct.RawPath}) || direct.RawPath != "" || direct.Host == "" { + return "", nil, fmt.Errorf("invalid sumdb name (must be host[/path]): %s %+v", name, *direct) + } + + // Determine how to get to database. + var base *url.URL + if len(key) >= 2 { + // Use explicit alternate URL listed in $GOSUMDB, + // bypassing both the default URL derivation and any proxies. + u, err := url.Parse(key[1]) + if err != nil { + return "", nil, fmt.Errorf("invalid GOSUMDB URL: %v", err) + } + base = u + } + + return name, sumdb.NewClient(&dbClient{key: key[0], name: name, direct: direct, base: base}), nil +} + +type dbClient struct { + key string + name string + direct *url.URL + + once sync.Once + base *url.URL + baseErr error +} + +func (c *dbClient) ReadRemote(path string) ([]byte, error) { + c.once.Do(c.initBase) + if c.baseErr != nil { + return nil, c.baseErr + } + + var data []byte + start := time.Now() + targ := web.Join(c.base, path) + data, err := web.GetBytes(targ) + if false { + fmt.Fprintf(os.Stderr, "%.3fs %s\n", time.Since(start).Seconds(), targ.Redacted()) + } + return data, err +} + +// initBase determines the base URL for connecting to the database. +// Determining the URL requires sending network traffic to proxies, +// so this work is delayed until we need to download something from +// the database. If everything we need is in the local cache and +// c.ReadRemote is never called, we will never do this work. +func (c *dbClient) initBase() { + if c.base != nil { + return + } + + // Try proxies in turn until we find out how to connect to this database. + // + // Before accessing any checksum database URL using a proxy, the proxy + // client should first fetch /sumdb//supported. + // + // If that request returns a successful (HTTP 200) response, then the proxy + // supports proxying checksum database requests. In that case, the client + // should use the proxied access method only, never falling back to a direct + // connection to the database. + // + // If the /sumdb//supported check fails with a “not found” (HTTP + // 404) or “gone” (HTTP 410) response, or if the proxy is configured to fall + // back on errors, the client will try the next proxy. If there are no + // proxies left or if the proxy is "direct" or "off", the client should + // connect directly to that database. + // + // Any other response is treated as the database being unavailable. + // + // See https://golang.org/design/25530-sumdb#proxying-a-checksum-database. + err := TryProxies(func(proxy string) error { + switch proxy { + case "noproxy": + return errUseProxy + case "direct", "off": + return errProxyOff + default: + proxyURL, err := url.Parse(proxy) + if err != nil { + return err + } + if _, err := web.GetBytes(web.Join(proxyURL, "sumdb/"+c.name+"/supported")); err != nil { + return err + } + // Success! This proxy will help us. + c.base = web.Join(proxyURL, "sumdb/"+c.name) + return nil + } + }) + if errors.Is(err, os.ErrNotExist) { + // No proxies, or all proxies failed (with 404, 410, or were were allowed + // to fall back), or we reached an explicit "direct" or "off". + c.base = c.direct + } else if err != nil { + c.baseErr = err + } +} + +// ReadConfig reads the key from c.key +// and otherwise reads the config (a latest tree head) from GOPATH/pkg/sumdb/. +func (c *dbClient) ReadConfig(file string) (data []byte, err error) { + if file == "key" { + return []byte(c.key), nil + } + + if cfg.SumdbDir == "" { + return nil, errors.New("could not locate sumdb file: missing $GOPATH") + } + targ := filepath.Join(cfg.SumdbDir, file) + data, err = lockedfile.Read(targ) + if errors.Is(err, os.ErrNotExist) { + // Treat non-existent as empty, to bootstrap the "latest" file + // the first time we connect to a given database. + return []byte{}, nil + } + return data, err +} + +// WriteConfig rewrites the latest tree head. +func (*dbClient) WriteConfig(file string, old, new []byte) error { + if file == "key" { + // Should not happen. + return fmt.Errorf("cannot write key") + } + if cfg.SumdbDir == "" { + return errors.New("could not locate sumdb file: missing $GOPATH") + } + targ := filepath.Join(cfg.SumdbDir, file) + os.MkdirAll(filepath.Dir(targ), 0777) + f, err := lockedfile.Edit(targ) + if err != nil { + return err + } + defer f.Close() + data, err := ioutil.ReadAll(f) + if err != nil { + return err + } + if len(data) > 0 && !bytes.Equal(data, old) { + return sumdb.ErrWriteConflict + } + if _, err := f.Seek(0, 0); err != nil { + return err + } + if err := f.Truncate(0); err != nil { + return err + } + if _, err := f.Write(new); err != nil { + return err + } + return f.Close() +} + +// ReadCache reads cached lookups or tiles from +// GOPATH/pkg/mod/cache/download/sumdb, +// which will be deleted by "go clean -modcache". +func (*dbClient) ReadCache(file string) ([]byte, error) { + targ := filepath.Join(cfg.GOMODCACHE, "cache/download/sumdb", file) + data, err := lockedfile.Read(targ) + // lockedfile.Write does not atomically create the file with contents. + // There is a moment between file creation and locking the file for writing, + // during which the empty file can be locked for reading. + // Treat observing an empty file as file not found. + if err == nil && len(data) == 0 { + err = &os.PathError{Op: "read", Path: targ, Err: os.ErrNotExist} + } + return data, err +} + +// WriteCache updates cached lookups or tiles. +func (*dbClient) WriteCache(file string, data []byte) { + targ := filepath.Join(cfg.GOMODCACHE, "cache/download/sumdb", file) + os.MkdirAll(filepath.Dir(targ), 0777) + lockedfile.Write(targ, bytes.NewReader(data), 0666) +} + +func (*dbClient) Log(msg string) { + // nothing for now +} + +func (*dbClient) SecurityError(msg string) { + base.Fatalf("%s", msg) +} diff --git a/cmd/go/_internal_/modfetch/unzip.go b/cmd/go/_internal_/modfetch/unzip.go deleted file mode 100644 index 1572c25..0000000 --- a/cmd/go/_internal_/modfetch/unzip.go +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package modfetch - -import ( - "archive/zip" - "fmt" - "io" - "io/ioutil" - "os" - "path" - "path/filepath" - "strings" - - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch/codehost" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/module" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/str" -) - -func Unzip(dir, zipfile, prefix string, maxSize int64) error { - // TODO(bcmills): The maxSize parameter is invariantly 0. Remove it. - if maxSize == 0 { - maxSize = codehost.MaxZipFile - } - - // Directory can exist, but must be empty. - files, _ := ioutil.ReadDir(dir) - if len(files) > 0 { - return fmt.Errorf("target directory %v exists and is not empty", dir) - } - if err := os.MkdirAll(dir, 0777); err != nil { - return err - } - - f, err := os.Open(zipfile) - if err != nil { - return err - } - defer f.Close() - info, err := f.Stat() - if err != nil { - return err - } - - z, err := zip.NewReader(f, info.Size()) - if err != nil { - return fmt.Errorf("unzip %v: %s", zipfile, err) - } - - foldPath := make(map[string]string) - var checkFold func(string) error - checkFold = func(name string) error { - fold := str.ToFold(name) - if foldPath[fold] == name { - return nil - } - dir := path.Dir(name) - if dir != "." { - if err := checkFold(dir); err != nil { - return err - } - } - if foldPath[fold] == "" { - foldPath[fold] = name - return nil - } - other := foldPath[fold] - return fmt.Errorf("unzip %v: case-insensitive file name collision: %q and %q", zipfile, other, name) - } - - // Check total size, valid file names. - var size int64 - for _, zf := range z.File { - if !str.HasPathPrefix(zf.Name, prefix) { - return fmt.Errorf("unzip %v: unexpected file name %s", zipfile, zf.Name) - } - if zf.Name == prefix || strings.HasSuffix(zf.Name, "/") { - continue - } - name := zf.Name[len(prefix)+1:] - if err := module.CheckFilePath(name); err != nil { - return fmt.Errorf("unzip %v: %v", zipfile, err) - } - if err := checkFold(name); err != nil { - return err - } - if path.Clean(zf.Name) != zf.Name || strings.HasPrefix(zf.Name[len(prefix)+1:], "/") { - return fmt.Errorf("unzip %v: invalid file name %s", zipfile, zf.Name) - } - s := int64(zf.UncompressedSize64) - if s < 0 || maxSize-size < s { - return fmt.Errorf("unzip %v: content too large", zipfile) - } - size += s - } - - // Unzip, enforcing sizes checked earlier. - for _, zf := range z.File { - if zf.Name == prefix || strings.HasSuffix(zf.Name, "/") { - continue - } - name := zf.Name[len(prefix):] - dst := filepath.Join(dir, name) - if err := os.MkdirAll(filepath.Dir(dst), 0777); err != nil { - return err - } - w, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0444) - if err != nil { - return fmt.Errorf("unzip %v: %v", zipfile, err) - } - r, err := zf.Open() - if err != nil { - w.Close() - return fmt.Errorf("unzip %v: %v", zipfile, err) - } - lr := &io.LimitedReader{R: r, N: int64(zf.UncompressedSize64) + 1} - _, err = io.Copy(w, lr) - r.Close() - if err != nil { - w.Close() - return fmt.Errorf("unzip %v: %v", zipfile, err) - } - if err := w.Close(); err != nil { - return fmt.Errorf("unzip %v: %v", zipfile, err) - } - if lr.N <= 0 { - return fmt.Errorf("unzip %v: content too large", zipfile) - } - } - - return nil -} - -// makeDirsReadOnly makes a best-effort attempt to remove write permissions for dir -// and its transitive contents. -func makeDirsReadOnly(dir string) { - type pathMode struct { - path string - mode os.FileMode - } - var dirs []pathMode // in lexical order - filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err == nil && info.Mode()&0222 != 0 { - if info.IsDir() { - dirs = append(dirs, pathMode{path, info.Mode()}) - } - } - return nil - }) - - // Run over list backward to chmod children before parents. - for i := len(dirs) - 1; i >= 0; i-- { - os.Chmod(dirs[i].path, dirs[i].mode&^0222) - } -} - -// RemoveAll removes a directory written by Download or Unzip, first applying -// any permission changes needed to do so. -func RemoveAll(dir string) error { - // Module cache has 0555 directories; make them writable in order to remove content. - filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return nil // ignore errors walking in file system - } - if info.IsDir() { - os.Chmod(path, 0777) - } - return nil - }) - return os.RemoveAll(dir) -} diff --git a/cmd/go/_internal_/modfetch/web.go b/cmd/go/_internal_/modfetch/web.go deleted file mode 100644 index 9eafc28..0000000 --- a/cmd/go/_internal_/modfetch/web.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !cmd_go_bootstrap - -package modfetch - -import ( - "io" - - web "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/web2" -) - -// webGetGoGet fetches a go-get=1 URL and returns the body in *body. -// It allows non-200 responses, as usual for these URLs. -func webGetGoGet(url string, body *io.ReadCloser) error { - return web.Get(url, web.Non200OK(), web.Body(body)) -} - -// webGetBytes returns the body returned by an HTTP GET, as a []byte. -// It insists on a 200 response. -func webGetBytes(url string, body *[]byte) error { - return web.Get(url, web.ReadAllBody(body)) -} - -// webGetBody returns the body returned by an HTTP GET, as a io.ReadCloser. -// It insists on a 200 response. -func webGetBody(url string, body *io.ReadCloser) error { - return web.Get(url, web.Body(body)) -} diff --git a/cmd/go/_internal_/modfile/gopkgin.go b/cmd/go/_internal_/modfile/gopkgin.go deleted file mode 100644 index c94b384..0000000 --- a/cmd/go/_internal_/modfile/gopkgin.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// TODO: Figure out what gopkg.in should do. - -package modfile - -import "strings" - -// ParseGopkgIn splits gopkg.in import paths into their constituent parts -func ParseGopkgIn(path string) (root, repo, major, subdir string, ok bool) { - if !strings.HasPrefix(path, "gopkg.in/") { - return - } - f := strings.Split(path, "/") - if len(f) >= 2 { - if elem, v, ok := dotV(f[1]); ok { - root = strings.Join(f[:2], "/") - repo = "github.com/go-" + elem + "/" + elem - major = v - subdir = strings.Join(f[2:], "/") - return root, repo, major, subdir, true - } - } - if len(f) >= 3 { - if elem, v, ok := dotV(f[2]); ok { - root = strings.Join(f[:3], "/") - repo = "github.com/" + f[1] + "/" + elem - major = v - subdir = strings.Join(f[3:], "/") - return root, repo, major, subdir, true - } - } - return -} - -func dotV(name string) (elem, v string, ok bool) { - i := len(name) - 1 - for i >= 0 && '0' <= name[i] && name[i] <= '9' { - i-- - } - if i <= 2 || i+1 >= len(name) || name[i-1] != '.' || name[i] != 'v' || name[i+1] == '0' && len(name) != i+2 { - return "", "", false - } - return name[:i-1], name[i:], true -} diff --git a/cmd/go/_internal_/modfile/print.go b/cmd/go/_internal_/modfile/print.go deleted file mode 100644 index 863c984..0000000 --- a/cmd/go/_internal_/modfile/print.go +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Module file printer. - -package modfile - -import ( - "bytes" - "fmt" - "strings" -) - -func Format(f *FileSyntax) []byte { - pr := &printer{} - pr.file(f) - return pr.Bytes() -} - -// A printer collects the state during printing of a file or expression. -type printer struct { - bytes.Buffer // output buffer - comment []Comment // pending end-of-line comments - margin int // left margin (indent), a number of tabs -} - -// printf prints to the buffer. -func (p *printer) printf(format string, args ...interface{}) { - fmt.Fprintf(p, format, args...) -} - -// indent returns the position on the current line, in bytes, 0-indexed. -func (p *printer) indent() int { - b := p.Bytes() - n := 0 - for n < len(b) && b[len(b)-1-n] != '\n' { - n++ - } - return n -} - -// newline ends the current line, flushing end-of-line comments. -func (p *printer) newline() { - if len(p.comment) > 0 { - p.printf(" ") - for i, com := range p.comment { - if i > 0 { - p.trim() - p.printf("\n") - for i := 0; i < p.margin; i++ { - p.printf("\t") - } - } - p.printf("%s", strings.TrimSpace(com.Token)) - } - p.comment = p.comment[:0] - } - - p.trim() - p.printf("\n") - for i := 0; i < p.margin; i++ { - p.printf("\t") - } -} - -// trim removes trailing spaces and tabs from the current line. -func (p *printer) trim() { - // Remove trailing spaces and tabs from line we're about to end. - b := p.Bytes() - n := len(b) - for n > 0 && (b[n-1] == '\t' || b[n-1] == ' ') { - n-- - } - p.Truncate(n) -} - -// file formats the given file into the print buffer. -func (p *printer) file(f *FileSyntax) { - for _, com := range f.Before { - p.printf("%s", strings.TrimSpace(com.Token)) - p.newline() - } - - for i, stmt := range f.Stmt { - switch x := stmt.(type) { - case *CommentBlock: - // comments already handled - p.expr(x) - - default: - p.expr(x) - p.newline() - } - - for _, com := range stmt.Comment().After { - p.printf("%s", strings.TrimSpace(com.Token)) - p.newline() - } - - if i+1 < len(f.Stmt) { - p.newline() - } - } -} - -func (p *printer) expr(x Expr) { - // Emit line-comments preceding this expression. - if before := x.Comment().Before; len(before) > 0 { - // Want to print a line comment. - // Line comments must be at the current margin. - p.trim() - if p.indent() > 0 { - // There's other text on the line. Start a new line. - p.printf("\n") - } - // Re-indent to margin. - for i := 0; i < p.margin; i++ { - p.printf("\t") - } - for _, com := range before { - p.printf("%s", strings.TrimSpace(com.Token)) - p.newline() - } - } - - switch x := x.(type) { - default: - panic(fmt.Errorf("printer: unexpected type %T", x)) - - case *CommentBlock: - // done - - case *LParen: - p.printf("(") - case *RParen: - p.printf(")") - - case *Line: - sep := "" - for _, tok := range x.Token { - p.printf("%s%s", sep, tok) - sep = " " - } - - case *LineBlock: - for _, tok := range x.Token { - p.printf("%s ", tok) - } - p.expr(&x.LParen) - p.margin++ - for _, l := range x.Line { - p.newline() - p.expr(l) - } - p.margin-- - p.newline() - p.expr(&x.RParen) - } - - // Queue end-of-line comments for printing when we - // reach the end of the line. - p.comment = append(p.comment, x.Comment().Suffix...) -} diff --git a/cmd/go/_internal_/modfile/read.go b/cmd/go/_internal_/modfile/read.go deleted file mode 100644 index 2adbe71..0000000 --- a/cmd/go/_internal_/modfile/read.go +++ /dev/null @@ -1,869 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Module file parser. -// This is a simplified copy of Google's buildifier parser. - -package modfile - -import ( - "bytes" - "fmt" - "os" - "strconv" - "strings" - "unicode" - "unicode/utf8" -) - -// A Position describes the position between two bytes of input. -type Position struct { - Line int // line in input (starting at 1) - LineRune int // rune in line (starting at 1) - Byte int // byte in input (starting at 0) -} - -// add returns the position at the end of s, assuming it starts at p. -func (p Position) add(s string) Position { - p.Byte += len(s) - if n := strings.Count(s, "\n"); n > 0 { - p.Line += n - s = s[strings.LastIndex(s, "\n")+1:] - p.LineRune = 1 - } - p.LineRune += utf8.RuneCountInString(s) - return p -} - -// An Expr represents an input element. -type Expr interface { - // Span returns the start and end position of the expression, - // excluding leading or trailing comments. - Span() (start, end Position) - - // Comment returns the comments attached to the expression. - // This method would normally be named 'Comments' but that - // would interfere with embedding a type of the same name. - Comment() *Comments -} - -// A Comment represents a single // comment. -type Comment struct { - Start Position - Token string // without trailing newline - Suffix bool // an end of line (not whole line) comment -} - -// Comments collects the comments associated with an expression. -type Comments struct { - Before []Comment // whole-line comments before this expression - Suffix []Comment // end-of-line comments after this expression - - // For top-level expressions only, After lists whole-line - // comments following the expression. - After []Comment -} - -// Comment returns the receiver. This isn't useful by itself, but -// a Comments struct is embedded into all the expression -// implementation types, and this gives each of those a Comment -// method to satisfy the Expr interface. -func (c *Comments) Comment() *Comments { - return c -} - -// A FileSyntax represents an entire go.mod file. -type FileSyntax struct { - Name string // file path - Comments - Stmt []Expr -} - -func (x *FileSyntax) Span() (start, end Position) { - if len(x.Stmt) == 0 { - return - } - start, _ = x.Stmt[0].Span() - _, end = x.Stmt[len(x.Stmt)-1].Span() - return start, end -} - -func (x *FileSyntax) addLine(hint Expr, tokens ...string) *Line { - if hint == nil { - // If no hint given, add to the last statement of the given type. - Loop: - for i := len(x.Stmt) - 1; i >= 0; i-- { - stmt := x.Stmt[i] - switch stmt := stmt.(type) { - case *Line: - if stmt.Token != nil && stmt.Token[0] == tokens[0] { - hint = stmt - break Loop - } - case *LineBlock: - if stmt.Token[0] == tokens[0] { - hint = stmt - break Loop - } - } - } - } - - if hint != nil { - for i, stmt := range x.Stmt { - switch stmt := stmt.(type) { - case *Line: - if stmt == hint { - // Convert line to line block. - stmt.InBlock = true - block := &LineBlock{Token: stmt.Token[:1], Line: []*Line{stmt}} - stmt.Token = stmt.Token[1:] - x.Stmt[i] = block - new := &Line{Token: tokens[1:], InBlock: true} - block.Line = append(block.Line, new) - return new - } - case *LineBlock: - if stmt == hint { - new := &Line{Token: tokens[1:], InBlock: true} - stmt.Line = append(stmt.Line, new) - return new - } - for j, line := range stmt.Line { - if line == hint { - // Add new line after hint. - stmt.Line = append(stmt.Line, nil) - copy(stmt.Line[j+2:], stmt.Line[j+1:]) - new := &Line{Token: tokens[1:], InBlock: true} - stmt.Line[j+1] = new - return new - } - } - } - } - } - - new := &Line{Token: tokens} - x.Stmt = append(x.Stmt, new) - return new -} - -func (x *FileSyntax) updateLine(line *Line, tokens ...string) { - if line.InBlock { - tokens = tokens[1:] - } - line.Token = tokens -} - -func (x *FileSyntax) removeLine(line *Line) { - line.Token = nil -} - -// Cleanup cleans up the file syntax x after any edit operations. -// To avoid quadratic behavior, removeLine marks the line as dead -// by setting line.Token = nil but does not remove it from the slice -// in which it appears. After edits have all been indicated, -// calling Cleanup cleans out the dead lines. -func (x *FileSyntax) Cleanup() { - w := 0 - for _, stmt := range x.Stmt { - switch stmt := stmt.(type) { - case *Line: - if stmt.Token == nil { - continue - } - case *LineBlock: - ww := 0 - for _, line := range stmt.Line { - if line.Token != nil { - stmt.Line[ww] = line - ww++ - } - } - if ww == 0 { - continue - } - if ww == 1 { - // Collapse block into single line. - line := &Line{ - Comments: Comments{ - Before: commentsAdd(stmt.Before, stmt.Line[0].Before), - Suffix: commentsAdd(stmt.Line[0].Suffix, stmt.Suffix), - After: commentsAdd(stmt.Line[0].After, stmt.After), - }, - Token: stringsAdd(stmt.Token, stmt.Line[0].Token), - } - x.Stmt[w] = line - w++ - continue - } - stmt.Line = stmt.Line[:ww] - } - x.Stmt[w] = stmt - w++ - } - x.Stmt = x.Stmt[:w] -} - -func commentsAdd(x, y []Comment) []Comment { - return append(x[:len(x):len(x)], y...) -} - -func stringsAdd(x, y []string) []string { - return append(x[:len(x):len(x)], y...) -} - -// A CommentBlock represents a top-level block of comments separate -// from any rule. -type CommentBlock struct { - Comments - Start Position -} - -func (x *CommentBlock) Span() (start, end Position) { - return x.Start, x.Start -} - -// A Line is a single line of tokens. -type Line struct { - Comments - Start Position - Token []string - InBlock bool - End Position -} - -func (x *Line) Span() (start, end Position) { - return x.Start, x.End -} - -// A LineBlock is a factored block of lines, like -// -// require ( -// "x" -// "y" -// ) -// -type LineBlock struct { - Comments - Start Position - LParen LParen - Token []string - Line []*Line - RParen RParen -} - -func (x *LineBlock) Span() (start, end Position) { - return x.Start, x.RParen.Pos.add(")") -} - -// An LParen represents the beginning of a parenthesized line block. -// It is a place to store suffix comments. -type LParen struct { - Comments - Pos Position -} - -func (x *LParen) Span() (start, end Position) { - return x.Pos, x.Pos.add(")") -} - -// An RParen represents the end of a parenthesized line block. -// It is a place to store whole-line (before) comments. -type RParen struct { - Comments - Pos Position -} - -func (x *RParen) Span() (start, end Position) { - return x.Pos, x.Pos.add(")") -} - -// An input represents a single input file being parsed. -type input struct { - // Lexing state. - filename string // name of input file, for errors - complete []byte // entire input - remaining []byte // remaining input - token []byte // token being scanned - lastToken string // most recently returned token, for error messages - pos Position // current input position - comments []Comment // accumulated comments - endRule int // position of end of current rule - - // Parser state. - file *FileSyntax // returned top-level syntax tree - parseError error // error encountered during parsing - - // Comment assignment state. - pre []Expr // all expressions, in preorder traversal - post []Expr // all expressions, in postorder traversal -} - -func newInput(filename string, data []byte) *input { - return &input{ - filename: filename, - complete: data, - remaining: data, - pos: Position{Line: 1, LineRune: 1, Byte: 0}, - } -} - -// parse parses the input file. -func parse(file string, data []byte) (f *FileSyntax, err error) { - in := newInput(file, data) - // The parser panics for both routine errors like syntax errors - // and for programmer bugs like array index errors. - // Turn both into error returns. Catching bug panics is - // especially important when processing many files. - defer func() { - if e := recover(); e != nil { - if e == in.parseError { - err = in.parseError - } else { - err = fmt.Errorf("%s:%d:%d: internal error: %v", in.filename, in.pos.Line, in.pos.LineRune, e) - } - } - }() - - // Invoke the parser. - in.parseFile() - if in.parseError != nil { - return nil, in.parseError - } - in.file.Name = in.filename - - // Assign comments to nearby syntax. - in.assignComments() - - return in.file, nil -} - -// Error is called to report an error. -// The reason s is often "syntax error". -// Error does not return: it panics. -func (in *input) Error(s string) { - if s == "syntax error" && in.lastToken != "" { - s += " near " + in.lastToken - } - in.parseError = fmt.Errorf("%s:%d:%d: %v", in.filename, in.pos.Line, in.pos.LineRune, s) - panic(in.parseError) -} - -// eof reports whether the input has reached end of file. -func (in *input) eof() bool { - return len(in.remaining) == 0 -} - -// peekRune returns the next rune in the input without consuming it. -func (in *input) peekRune() int { - if len(in.remaining) == 0 { - return 0 - } - r, _ := utf8.DecodeRune(in.remaining) - return int(r) -} - -// peekPrefix reports whether the remaining input begins with the given prefix. -func (in *input) peekPrefix(prefix string) bool { - // This is like bytes.HasPrefix(in.remaining, []byte(prefix)) - // but without the allocation of the []byte copy of prefix. - for i := 0; i < len(prefix); i++ { - if i >= len(in.remaining) || in.remaining[i] != prefix[i] { - return false - } - } - return true -} - -// readRune consumes and returns the next rune in the input. -func (in *input) readRune() int { - if len(in.remaining) == 0 { - in.Error("internal lexer error: readRune at EOF") - } - r, size := utf8.DecodeRune(in.remaining) - in.remaining = in.remaining[size:] - if r == '\n' { - in.pos.Line++ - in.pos.LineRune = 1 - } else { - in.pos.LineRune++ - } - in.pos.Byte += size - return int(r) -} - -type symType struct { - pos Position - endPos Position - text string -} - -// startToken marks the beginning of the next input token. -// It must be followed by a call to endToken, once the token has -// been consumed using readRune. -func (in *input) startToken(sym *symType) { - in.token = in.remaining - sym.text = "" - sym.pos = in.pos -} - -// endToken marks the end of an input token. -// It records the actual token string in sym.text if the caller -// has not done that already. -func (in *input) endToken(sym *symType) { - if sym.text == "" { - tok := string(in.token[:len(in.token)-len(in.remaining)]) - sym.text = tok - in.lastToken = sym.text - } - sym.endPos = in.pos -} - -// lex is called from the parser to obtain the next input token. -// It returns the token value (either a rune like '+' or a symbolic token _FOR) -// and sets val to the data associated with the token. -// For all our input tokens, the associated data is -// val.Pos (the position where the token begins) -// and val.Token (the input string corresponding to the token). -func (in *input) lex(sym *symType) int { - // Skip past spaces, stopping at non-space or EOF. - countNL := 0 // number of newlines we've skipped past - for !in.eof() { - // Skip over spaces. Count newlines so we can give the parser - // information about where top-level blank lines are, - // for top-level comment assignment. - c := in.peekRune() - if c == ' ' || c == '\t' || c == '\r' { - in.readRune() - continue - } - - // Comment runs to end of line. - if in.peekPrefix("//") { - in.startToken(sym) - - // Is this comment the only thing on its line? - // Find the last \n before this // and see if it's all - // spaces from there to here. - i := bytes.LastIndex(in.complete[:in.pos.Byte], []byte("\n")) - suffix := len(bytes.TrimSpace(in.complete[i+1:in.pos.Byte])) > 0 - in.readRune() - in.readRune() - - // Consume comment. - for len(in.remaining) > 0 && in.readRune() != '\n' { - } - in.endToken(sym) - - sym.text = strings.TrimRight(sym.text, "\n") - in.lastToken = "comment" - - // If we are at top level (not in a statement), hand the comment to - // the parser as a _COMMENT token. The grammar is written - // to handle top-level comments itself. - if !suffix { - // Not in a statement. Tell parser about top-level comment. - return _COMMENT - } - - // Otherwise, save comment for later attachment to syntax tree. - if countNL > 1 { - in.comments = append(in.comments, Comment{sym.pos, "", false}) - } - in.comments = append(in.comments, Comment{sym.pos, sym.text, suffix}) - countNL = 1 - return _EOL - } - - if in.peekPrefix("/*") { - in.Error(fmt.Sprintf("mod files must use // comments (not /* */ comments)")) - } - - // Found non-space non-comment. - break - } - - // Found the beginning of the next token. - in.startToken(sym) - defer in.endToken(sym) - - // End of file. - if in.eof() { - in.lastToken = "EOF" - return _EOF - } - - // Punctuation tokens. - switch c := in.peekRune(); c { - case '\n': - in.readRune() - return c - - case '(': - in.readRune() - return c - - case ')': - in.readRune() - return c - - case '"', '`': // quoted string - quote := c - in.readRune() - for { - if in.eof() { - in.pos = sym.pos - in.Error("unexpected EOF in string") - } - if in.peekRune() == '\n' { - in.Error("unexpected newline in string") - } - c := in.readRune() - if c == quote { - break - } - if c == '\\' && quote != '`' { - if in.eof() { - in.pos = sym.pos - in.Error("unexpected EOF in string") - } - in.readRune() - } - } - in.endToken(sym) - return _STRING - } - - // Checked all punctuation. Must be identifier token. - if c := in.peekRune(); !isIdent(c) { - in.Error(fmt.Sprintf("unexpected input character %#q", c)) - } - - // Scan over identifier. - for isIdent(in.peekRune()) { - if in.peekPrefix("//") { - break - } - if in.peekPrefix("/*") { - in.Error(fmt.Sprintf("mod files must use // comments (not /* */ comments)")) - } - in.readRune() - } - return _IDENT -} - -// isIdent reports whether c is an identifier rune. -// We treat nearly all runes as identifier runes. -func isIdent(c int) bool { - return c != 0 && !unicode.IsSpace(rune(c)) -} - -// Comment assignment. -// We build two lists of all subexpressions, preorder and postorder. -// The preorder list is ordered by start location, with outer expressions first. -// The postorder list is ordered by end location, with outer expressions last. -// We use the preorder list to assign each whole-line comment to the syntax -// immediately following it, and we use the postorder list to assign each -// end-of-line comment to the syntax immediately preceding it. - -// order walks the expression adding it and its subexpressions to the -// preorder and postorder lists. -func (in *input) order(x Expr) { - if x != nil { - in.pre = append(in.pre, x) - } - switch x := x.(type) { - default: - panic(fmt.Errorf("order: unexpected type %T", x)) - case nil: - // nothing - case *LParen, *RParen: - // nothing - case *CommentBlock: - // nothing - case *Line: - // nothing - case *FileSyntax: - for _, stmt := range x.Stmt { - in.order(stmt) - } - case *LineBlock: - in.order(&x.LParen) - for _, l := range x.Line { - in.order(l) - } - in.order(&x.RParen) - } - if x != nil { - in.post = append(in.post, x) - } -} - -// assignComments attaches comments to nearby syntax. -func (in *input) assignComments() { - const debug = false - - // Generate preorder and postorder lists. - in.order(in.file) - - // Split into whole-line comments and suffix comments. - var line, suffix []Comment - for _, com := range in.comments { - if com.Suffix { - suffix = append(suffix, com) - } else { - line = append(line, com) - } - } - - if debug { - for _, c := range line { - fmt.Fprintf(os.Stderr, "LINE %q :%d:%d #%d\n", c.Token, c.Start.Line, c.Start.LineRune, c.Start.Byte) - } - } - - // Assign line comments to syntax immediately following. - for _, x := range in.pre { - start, _ := x.Span() - if debug { - fmt.Printf("pre %T :%d:%d #%d\n", x, start.Line, start.LineRune, start.Byte) - } - xcom := x.Comment() - for len(line) > 0 && start.Byte >= line[0].Start.Byte { - if debug { - fmt.Fprintf(os.Stderr, "ASSIGN LINE %q #%d\n", line[0].Token, line[0].Start.Byte) - } - xcom.Before = append(xcom.Before, line[0]) - line = line[1:] - } - } - - // Remaining line comments go at end of file. - in.file.After = append(in.file.After, line...) - - if debug { - for _, c := range suffix { - fmt.Fprintf(os.Stderr, "SUFFIX %q :%d:%d #%d\n", c.Token, c.Start.Line, c.Start.LineRune, c.Start.Byte) - } - } - - // Assign suffix comments to syntax immediately before. - for i := len(in.post) - 1; i >= 0; i-- { - x := in.post[i] - - start, end := x.Span() - if debug { - fmt.Printf("post %T :%d:%d #%d :%d:%d #%d\n", x, start.Line, start.LineRune, start.Byte, end.Line, end.LineRune, end.Byte) - } - - // Do not assign suffix comments to end of line block or whole file. - // Instead assign them to the last element inside. - switch x.(type) { - case *FileSyntax: - continue - } - - // Do not assign suffix comments to something that starts - // on an earlier line, so that in - // - // x ( y - // z ) // comment - // - // we assign the comment to z and not to x ( ... ). - if start.Line != end.Line { - continue - } - xcom := x.Comment() - for len(suffix) > 0 && end.Byte <= suffix[len(suffix)-1].Start.Byte { - if debug { - fmt.Fprintf(os.Stderr, "ASSIGN SUFFIX %q #%d\n", suffix[len(suffix)-1].Token, suffix[len(suffix)-1].Start.Byte) - } - xcom.Suffix = append(xcom.Suffix, suffix[len(suffix)-1]) - suffix = suffix[:len(suffix)-1] - } - } - - // We assigned suffix comments in reverse. - // If multiple suffix comments were appended to the same - // expression node, they are now in reverse. Fix that. - for _, x := range in.post { - reverseComments(x.Comment().Suffix) - } - - // Remaining suffix comments go at beginning of file. - in.file.Before = append(in.file.Before, suffix...) -} - -// reverseComments reverses the []Comment list. -func reverseComments(list []Comment) { - for i, j := 0, len(list)-1; i < j; i, j = i+1, j-1 { - list[i], list[j] = list[j], list[i] - } -} - -func (in *input) parseFile() { - in.file = new(FileSyntax) - var sym symType - var cb *CommentBlock - for { - tok := in.lex(&sym) - switch tok { - case '\n': - if cb != nil { - in.file.Stmt = append(in.file.Stmt, cb) - cb = nil - } - case _COMMENT: - if cb == nil { - cb = &CommentBlock{Start: sym.pos} - } - com := cb.Comment() - com.Before = append(com.Before, Comment{Start: sym.pos, Token: sym.text}) - case _EOF: - if cb != nil { - in.file.Stmt = append(in.file.Stmt, cb) - } - return - default: - in.parseStmt(&sym) - if cb != nil { - in.file.Stmt[len(in.file.Stmt)-1].Comment().Before = cb.Before - cb = nil - } - } - } -} - -func (in *input) parseStmt(sym *symType) { - start := sym.pos - end := sym.endPos - token := []string{sym.text} - for { - tok := in.lex(sym) - switch tok { - case '\n', _EOF, _EOL: - in.file.Stmt = append(in.file.Stmt, &Line{ - Start: start, - Token: token, - End: end, - }) - return - case '(': - in.file.Stmt = append(in.file.Stmt, in.parseLineBlock(start, token, sym)) - return - default: - token = append(token, sym.text) - end = sym.endPos - } - } -} - -func (in *input) parseLineBlock(start Position, token []string, sym *symType) *LineBlock { - x := &LineBlock{ - Start: start, - Token: token, - LParen: LParen{Pos: sym.pos}, - } - var comments []Comment - for { - tok := in.lex(sym) - switch tok { - case _EOL: - // ignore - case '\n': - if len(comments) == 0 && len(x.Line) > 0 || len(comments) > 0 && comments[len(comments)-1].Token != "" { - comments = append(comments, Comment{}) - } - case _COMMENT: - comments = append(comments, Comment{Start: sym.pos, Token: sym.text}) - case _EOF: - in.Error(fmt.Sprintf("syntax error (unterminated block started at %s:%d:%d)", in.filename, x.Start.Line, x.Start.LineRune)) - case ')': - x.RParen.Before = comments - x.RParen.Pos = sym.pos - tok = in.lex(sym) - if tok != '\n' && tok != _EOF && tok != _EOL { - in.Error("syntax error (expected newline after closing paren)") - } - return x - default: - l := in.parseLine(sym) - x.Line = append(x.Line, l) - l.Comment().Before = comments - comments = nil - } - } -} - -func (in *input) parseLine(sym *symType) *Line { - start := sym.pos - end := sym.endPos - token := []string{sym.text} - for { - tok := in.lex(sym) - switch tok { - case '\n', _EOF, _EOL: - return &Line{ - Start: start, - Token: token, - End: end, - InBlock: true, - } - default: - token = append(token, sym.text) - end = sym.endPos - } - } -} - -const ( - _EOF = -(1 + iota) - _EOL - _IDENT - _STRING - _COMMENT -) - -var ( - slashSlash = []byte("//") - moduleStr = []byte("module") -) - -// ModulePath returns the module path from the gomod file text. -// If it cannot find a module path, it returns an empty string. -// It is tolerant of unrelated problems in the go.mod file. -func ModulePath(mod []byte) string { - for len(mod) > 0 { - line := mod - mod = nil - if i := bytes.IndexByte(line, '\n'); i >= 0 { - line, mod = line[:i], line[i+1:] - } - if i := bytes.Index(line, slashSlash); i >= 0 { - line = line[:i] - } - line = bytes.TrimSpace(line) - if !bytes.HasPrefix(line, moduleStr) { - continue - } - line = line[len(moduleStr):] - n := len(line) - line = bytes.TrimSpace(line) - if len(line) == n || len(line) == 0 { - continue - } - - if line[0] == '"' || line[0] == '`' { - p, err := strconv.Unquote(string(line)) - if err != nil { - return "" // malformed quoted string or multiline module path - } - return p - } - - return string(line) - } - return "" // missing module path -} diff --git a/cmd/go/_internal_/modfile/rule.go b/cmd/go/_internal_/modfile/rule.go deleted file mode 100644 index 85a9fe3..0000000 --- a/cmd/go/_internal_/modfile/rule.go +++ /dev/null @@ -1,740 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package modfile - -import ( - "bytes" - "errors" - "fmt" - "path/filepath" - "regexp" - "sort" - "strconv" - "strings" - "unicode" - - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/module" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/semver" -) - -// A File is the parsed, interpreted form of a go.mod file. -type File struct { - Module *Module - Go *Go - Require []*Require - Exclude []*Exclude - Replace []*Replace - - Syntax *FileSyntax -} - -// A Module is the module statement. -type Module struct { - Mod module.Version - Syntax *Line -} - -// A Go is the go statement. -type Go struct { - Version string // "1.23" - Syntax *Line -} - -// A Require is a single require statement. -type Require struct { - Mod module.Version - Indirect bool // has "// indirect" comment - Syntax *Line -} - -// An Exclude is a single exclude statement. -type Exclude struct { - Mod module.Version - Syntax *Line -} - -// A Replace is a single replace statement. -type Replace struct { - Old module.Version - New module.Version - Syntax *Line -} - -func (f *File) AddModuleStmt(path string) error { - if f.Syntax == nil { - f.Syntax = new(FileSyntax) - } - if f.Module == nil { - f.Module = &Module{ - Mod: module.Version{Path: path}, - Syntax: f.Syntax.addLine(nil, "module", AutoQuote(path)), - } - } else { - f.Module.Mod.Path = path - f.Syntax.updateLine(f.Module.Syntax, "module", AutoQuote(path)) - } - return nil -} - -func (f *File) AddComment(text string) { - if f.Syntax == nil { - f.Syntax = new(FileSyntax) - } - f.Syntax.Stmt = append(f.Syntax.Stmt, &CommentBlock{ - Comments: Comments{ - Before: []Comment{ - { - Token: text, - }, - }, - }, - }) -} - -type VersionFixer func(path, version string) (string, error) - -// Parse parses the data, reported in errors as being from file, -// into a File struct. It applies fix, if non-nil, to canonicalize all module versions found. -func Parse(file string, data []byte, fix VersionFixer) (*File, error) { - return parseToFile(file, data, fix, true) -} - -// ParseLax is like Parse but ignores unknown statements. -// It is used when parsing go.mod files other than the main module, -// under the theory that most statement types we add in the future will -// only apply in the main module, like exclude and replace, -// and so we get better gradual deployments if old go commands -// simply ignore those statements when found in go.mod files -// in dependencies. -func ParseLax(file string, data []byte, fix VersionFixer) (*File, error) { - return parseToFile(file, data, fix, false) -} - -func parseToFile(file string, data []byte, fix VersionFixer, strict bool) (*File, error) { - fs, err := parse(file, data) - if err != nil { - return nil, err - } - f := &File{ - Syntax: fs, - } - - var errs bytes.Buffer - for _, x := range fs.Stmt { - switch x := x.(type) { - case *Line: - f.add(&errs, x, x.Token[0], x.Token[1:], fix, strict) - - case *LineBlock: - if len(x.Token) > 1 { - if strict { - fmt.Fprintf(&errs, "%s:%d: unknown block type: %s\n", file, x.Start.Line, strings.Join(x.Token, " ")) - } - continue - } - switch x.Token[0] { - default: - if strict { - fmt.Fprintf(&errs, "%s:%d: unknown block type: %s\n", file, x.Start.Line, strings.Join(x.Token, " ")) - } - continue - case "module", "require", "exclude", "replace": - for _, l := range x.Line { - f.add(&errs, l, x.Token[0], l.Token, fix, strict) - } - } - } - } - - if errs.Len() > 0 { - return nil, errors.New(strings.TrimRight(errs.String(), "\n")) - } - return f, nil -} - -var GoVersionRE = regexp.MustCompile(`([1-9][0-9]*)\.(0|[1-9][0-9]*)`) - -func (f *File) add(errs *bytes.Buffer, line *Line, verb string, args []string, fix VersionFixer, strict bool) { - // If strict is false, this module is a dependency. - // We ignore all unknown directives as well as main-module-only - // directives like replace and exclude. It will work better for - // forward compatibility if we can depend on modules that have unknown - // statements (presumed relevant only when acting as the main module) - // and simply ignore those statements. - if !strict { - switch verb { - case "module", "require", "go": - // want these even for dependency go.mods - default: - return - } - } - - switch verb { - default: - fmt.Fprintf(errs, "%s:%d: unknown directive: %s\n", f.Syntax.Name, line.Start.Line, verb) - - case "go": - if f.Go != nil { - fmt.Fprintf(errs, "%s:%d: repeated go statement\n", f.Syntax.Name, line.Start.Line) - return - } - if len(args) != 1 || !GoVersionRE.MatchString(args[0]) { - fmt.Fprintf(errs, "%s:%d: usage: go 1.23\n", f.Syntax.Name, line.Start.Line) - return - } - f.Go = &Go{Syntax: line} - f.Go.Version = args[0] - case "module": - if f.Module != nil { - fmt.Fprintf(errs, "%s:%d: repeated module statement\n", f.Syntax.Name, line.Start.Line) - return - } - f.Module = &Module{Syntax: line} - if len(args) != 1 { - - fmt.Fprintf(errs, "%s:%d: usage: module module/path [version]\n", f.Syntax.Name, line.Start.Line) - return - } - s, err := parseString(&args[0]) - if err != nil { - fmt.Fprintf(errs, "%s:%d: invalid quoted string: %v\n", f.Syntax.Name, line.Start.Line, err) - return - } - f.Module.Mod = module.Version{Path: s} - case "require", "exclude": - if len(args) != 2 { - fmt.Fprintf(errs, "%s:%d: usage: %s module/path v1.2.3\n", f.Syntax.Name, line.Start.Line, verb) - return - } - s, err := parseString(&args[0]) - if err != nil { - fmt.Fprintf(errs, "%s:%d: invalid quoted string: %v\n", f.Syntax.Name, line.Start.Line, err) - return - } - old := args[1] - v, err := parseVersion(s, &args[1], fix) - if err != nil { - fmt.Fprintf(errs, "%s:%d: invalid module version %q: %v\n", f.Syntax.Name, line.Start.Line, old, err) - return - } - pathMajor, err := modulePathMajor(s) - if err != nil { - fmt.Fprintf(errs, "%s:%d: %v\n", f.Syntax.Name, line.Start.Line, err) - return - } - if !module.MatchPathMajor(v, pathMajor) { - if pathMajor == "" { - pathMajor = "v0 or v1" - } - fmt.Fprintf(errs, "%s:%d: invalid module: %s should be %s, not %s (%s)\n", f.Syntax.Name, line.Start.Line, s, pathMajor, semver.Major(v), v) - return - } - if verb == "require" { - f.Require = append(f.Require, &Require{ - Mod: module.Version{Path: s, Version: v}, - Syntax: line, - Indirect: isIndirect(line), - }) - } else { - f.Exclude = append(f.Exclude, &Exclude{ - Mod: module.Version{Path: s, Version: v}, - Syntax: line, - }) - } - case "replace": - arrow := 2 - if len(args) >= 2 && args[1] == "=>" { - arrow = 1 - } - if len(args) < arrow+2 || len(args) > arrow+3 || args[arrow] != "=>" { - fmt.Fprintf(errs, "%s:%d: usage: %s module/path [v1.2.3] => other/module v1.4\n\t or %s module/path [v1.2.3] => ../local/directory\n", f.Syntax.Name, line.Start.Line, verb, verb) - return - } - s, err := parseString(&args[0]) - if err != nil { - fmt.Fprintf(errs, "%s:%d: invalid quoted string: %v\n", f.Syntax.Name, line.Start.Line, err) - return - } - pathMajor, err := modulePathMajor(s) - if err != nil { - fmt.Fprintf(errs, "%s:%d: %v\n", f.Syntax.Name, line.Start.Line, err) - return - } - var v string - if arrow == 2 { - old := args[1] - v, err = parseVersion(s, &args[1], fix) - if err != nil { - fmt.Fprintf(errs, "%s:%d: invalid module version %v: %v\n", f.Syntax.Name, line.Start.Line, old, err) - return - } - if !module.MatchPathMajor(v, pathMajor) { - if pathMajor == "" { - pathMajor = "v0 or v1" - } - fmt.Fprintf(errs, "%s:%d: invalid module: %s should be %s, not %s (%s)\n", f.Syntax.Name, line.Start.Line, s, pathMajor, semver.Major(v), v) - return - } - } - ns, err := parseString(&args[arrow+1]) - if err != nil { - fmt.Fprintf(errs, "%s:%d: invalid quoted string: %v\n", f.Syntax.Name, line.Start.Line, err) - return - } - nv := "" - if len(args) == arrow+2 { - if !IsDirectoryPath(ns) { - fmt.Fprintf(errs, "%s:%d: replacement module without version must be directory path (rooted or starting with ./ or ../)\n", f.Syntax.Name, line.Start.Line) - return - } - if filepath.Separator == '/' && strings.Contains(ns, `\`) { - fmt.Fprintf(errs, "%s:%d: replacement directory appears to be Windows path (on a non-windows system)\n", f.Syntax.Name, line.Start.Line) - return - } - } - if len(args) == arrow+3 { - old := args[arrow+1] - nv, err = parseVersion(ns, &args[arrow+2], fix) - if err != nil { - fmt.Fprintf(errs, "%s:%d: invalid module version %v: %v\n", f.Syntax.Name, line.Start.Line, old, err) - return - } - if IsDirectoryPath(ns) { - fmt.Fprintf(errs, "%s:%d: replacement module directory path %q cannot have version\n", f.Syntax.Name, line.Start.Line, ns) - return - } - } - f.Replace = append(f.Replace, &Replace{ - Old: module.Version{Path: s, Version: v}, - New: module.Version{Path: ns, Version: nv}, - Syntax: line, - }) - } -} - -// isIndirect reports whether line has a "// indirect" comment, -// meaning it is in go.mod only for its effect on indirect dependencies, -// so that it can be dropped entirely once the effective version of the -// indirect dependency reaches the given minimum version. -func isIndirect(line *Line) bool { - if len(line.Suffix) == 0 { - return false - } - f := strings.Fields(line.Suffix[0].Token) - return (len(f) == 2 && f[1] == "indirect" || len(f) > 2 && f[1] == "indirect;") && f[0] == "//" -} - -// setIndirect sets line to have (or not have) a "// indirect" comment. -func setIndirect(line *Line, indirect bool) { - if isIndirect(line) == indirect { - return - } - if indirect { - // Adding comment. - if len(line.Suffix) == 0 { - // New comment. - line.Suffix = []Comment{{Token: "// indirect", Suffix: true}} - return - } - // Insert at beginning of existing comment. - com := &line.Suffix[0] - space := " " - if len(com.Token) > 2 && com.Token[2] == ' ' || com.Token[2] == '\t' { - space = "" - } - com.Token = "// indirect;" + space + com.Token[2:] - return - } - - // Removing comment. - f := strings.Fields(line.Suffix[0].Token) - if len(f) == 2 { - // Remove whole comment. - line.Suffix = nil - return - } - - // Remove comment prefix. - com := &line.Suffix[0] - i := strings.Index(com.Token, "indirect;") - com.Token = "//" + com.Token[i+len("indirect;"):] -} - -// IsDirectoryPath reports whether the given path should be interpreted -// as a directory path. Just like on the go command line, relative paths -// and rooted paths are directory paths; the rest are module paths. -func IsDirectoryPath(ns string) bool { - // Because go.mod files can move from one system to another, - // we check all known path syntaxes, both Unix and Windows. - return strings.HasPrefix(ns, "./") || strings.HasPrefix(ns, "../") || strings.HasPrefix(ns, "/") || - strings.HasPrefix(ns, `.\`) || strings.HasPrefix(ns, `..\`) || strings.HasPrefix(ns, `\`) || - len(ns) >= 2 && ('A' <= ns[0] && ns[0] <= 'Z' || 'a' <= ns[0] && ns[0] <= 'z') && ns[1] == ':' -} - -// MustQuote reports whether s must be quoted in order to appear as -// a single token in a go.mod line. -func MustQuote(s string) bool { - for _, r := range s { - if !unicode.IsPrint(r) || r == ' ' || r == '"' || r == '\'' || r == '`' { - return true - } - } - return s == "" || strings.Contains(s, "//") || strings.Contains(s, "/*") -} - -// AutoQuote returns s or, if quoting is required for s to appear in a go.mod, -// the quotation of s. -func AutoQuote(s string) string { - if MustQuote(s) { - return strconv.Quote(s) - } - return s -} - -func parseString(s *string) (string, error) { - t := *s - if strings.HasPrefix(t, `"`) { - var err error - if t, err = strconv.Unquote(t); err != nil { - return "", err - } - } else if strings.ContainsAny(t, "\"'`") { - // Other quotes are reserved both for possible future expansion - // and to avoid confusion. For example if someone types 'x' - // we want that to be a syntax error and not a literal x in literal quotation marks. - return "", fmt.Errorf("unquoted string cannot contain quote") - } - *s = AutoQuote(t) - return t, nil -} - -func parseVersion(path string, s *string, fix VersionFixer) (string, error) { - t, err := parseString(s) - if err != nil { - return "", err - } - if fix != nil { - var err error - t, err = fix(path, t) - if err != nil { - return "", err - } - } - if v := module.CanonicalVersion(t); v != "" { - *s = v - return *s, nil - } - return "", fmt.Errorf("version must be of the form v1.2.3") -} - -func modulePathMajor(path string) (string, error) { - _, major, ok := module.SplitPathVersion(path) - if !ok { - return "", fmt.Errorf("invalid module path") - } - return major, nil -} - -func (f *File) Format() ([]byte, error) { - return Format(f.Syntax), nil -} - -// Cleanup cleans up the file f after any edit operations. -// To avoid quadratic behavior, modifications like DropRequire -// clear the entry but do not remove it from the slice. -// Cleanup cleans out all the cleared entries. -func (f *File) Cleanup() { - w := 0 - for _, r := range f.Require { - if r.Mod.Path != "" { - f.Require[w] = r - w++ - } - } - f.Require = f.Require[:w] - - w = 0 - for _, x := range f.Exclude { - if x.Mod.Path != "" { - f.Exclude[w] = x - w++ - } - } - f.Exclude = f.Exclude[:w] - - w = 0 - for _, r := range f.Replace { - if r.Old.Path != "" { - f.Replace[w] = r - w++ - } - } - f.Replace = f.Replace[:w] - - f.Syntax.Cleanup() -} - -func (f *File) AddGoStmt(version string) error { - if !GoVersionRE.MatchString(version) { - return fmt.Errorf("invalid language version string %q", version) - } - if f.Go == nil { - f.Go = &Go{ - Version: version, - Syntax: f.Syntax.addLine(nil, "go", version), - } - } else { - f.Go.Version = version - f.Syntax.updateLine(f.Go.Syntax, "go", version) - } - return nil -} - -func (f *File) AddRequire(path, vers string) error { - need := true - for _, r := range f.Require { - if r.Mod.Path == path { - if need { - r.Mod.Version = vers - f.Syntax.updateLine(r.Syntax, "require", AutoQuote(path), vers) - need = false - } else { - f.Syntax.removeLine(r.Syntax) - *r = Require{} - } - } - } - - if need { - f.AddNewRequire(path, vers, false) - } - return nil -} - -func (f *File) AddNewRequire(path, vers string, indirect bool) { - line := f.Syntax.addLine(nil, "require", AutoQuote(path), vers) - setIndirect(line, indirect) - f.Require = append(f.Require, &Require{module.Version{Path: path, Version: vers}, indirect, line}) -} - -func (f *File) SetRequire(req []*Require) { - need := make(map[string]string) - indirect := make(map[string]bool) - for _, r := range req { - need[r.Mod.Path] = r.Mod.Version - indirect[r.Mod.Path] = r.Indirect - } - - for _, r := range f.Require { - if v, ok := need[r.Mod.Path]; ok { - r.Mod.Version = v - r.Indirect = indirect[r.Mod.Path] - } - } - - var newStmts []Expr - for _, stmt := range f.Syntax.Stmt { - switch stmt := stmt.(type) { - case *LineBlock: - if len(stmt.Token) > 0 && stmt.Token[0] == "require" { - var newLines []*Line - for _, line := range stmt.Line { - if p, err := parseString(&line.Token[0]); err == nil && need[p] != "" { - line.Token[1] = need[p] - delete(need, p) - setIndirect(line, indirect[p]) - newLines = append(newLines, line) - } - } - if len(newLines) == 0 { - continue // drop stmt - } - stmt.Line = newLines - } - - case *Line: - if len(stmt.Token) > 0 && stmt.Token[0] == "require" { - if p, err := parseString(&stmt.Token[1]); err == nil && need[p] != "" { - stmt.Token[2] = need[p] - delete(need, p) - setIndirect(stmt, indirect[p]) - } else { - continue // drop stmt - } - } - } - newStmts = append(newStmts, stmt) - } - f.Syntax.Stmt = newStmts - - for path, vers := range need { - f.AddNewRequire(path, vers, indirect[path]) - } - f.SortBlocks() -} - -func (f *File) DropRequire(path string) error { - for _, r := range f.Require { - if r.Mod.Path == path { - f.Syntax.removeLine(r.Syntax) - *r = Require{} - } - } - return nil -} - -func (f *File) AddExclude(path, vers string) error { - var hint *Line - for _, x := range f.Exclude { - if x.Mod.Path == path && x.Mod.Version == vers { - return nil - } - if x.Mod.Path == path { - hint = x.Syntax - } - } - - f.Exclude = append(f.Exclude, &Exclude{Mod: module.Version{Path: path, Version: vers}, Syntax: f.Syntax.addLine(hint, "exclude", AutoQuote(path), vers)}) - return nil -} - -func (f *File) DropExclude(path, vers string) error { - for _, x := range f.Exclude { - if x.Mod.Path == path && x.Mod.Version == vers { - f.Syntax.removeLine(x.Syntax) - *x = Exclude{} - } - } - return nil -} - -func (f *File) AddReplace(oldPath, oldVers, newPath, newVers string) error { - need := true - old := module.Version{Path: oldPath, Version: oldVers} - new := module.Version{Path: newPath, Version: newVers} - tokens := []string{"replace", AutoQuote(oldPath)} - if oldVers != "" { - tokens = append(tokens, oldVers) - } - tokens = append(tokens, "=>", AutoQuote(newPath)) - if newVers != "" { - tokens = append(tokens, newVers) - } - - var hint *Line - for _, r := range f.Replace { - if r.Old.Path == oldPath && (oldVers == "" || r.Old.Version == oldVers) { - if need { - // Found replacement for old; update to use new. - r.New = new - f.Syntax.updateLine(r.Syntax, tokens...) - need = false - continue - } - // Already added; delete other replacements for same. - f.Syntax.removeLine(r.Syntax) - *r = Replace{} - } - if r.Old.Path == oldPath { - hint = r.Syntax - } - } - if need { - f.Replace = append(f.Replace, &Replace{Old: old, New: new, Syntax: f.Syntax.addLine(hint, tokens...)}) - } - return nil -} - -func (f *File) DropReplace(oldPath, oldVers string) error { - for _, r := range f.Replace { - if r.Old.Path == oldPath && r.Old.Version == oldVers { - f.Syntax.removeLine(r.Syntax) - *r = Replace{} - } - } - return nil -} - -func (f *File) SortBlocks() { - f.removeDups() // otherwise sorting is unsafe - - for _, stmt := range f.Syntax.Stmt { - block, ok := stmt.(*LineBlock) - if !ok { - continue - } - sort.Slice(block.Line, func(i, j int) bool { - li := block.Line[i] - lj := block.Line[j] - for k := 0; k < len(li.Token) && k < len(lj.Token); k++ { - if li.Token[k] != lj.Token[k] { - return li.Token[k] < lj.Token[k] - } - } - return len(li.Token) < len(lj.Token) - }) - } -} - -func (f *File) removeDups() { - have := make(map[module.Version]bool) - kill := make(map[*Line]bool) - for _, x := range f.Exclude { - if have[x.Mod] { - kill[x.Syntax] = true - continue - } - have[x.Mod] = true - } - var excl []*Exclude - for _, x := range f.Exclude { - if !kill[x.Syntax] { - excl = append(excl, x) - } - } - f.Exclude = excl - - have = make(map[module.Version]bool) - // Later replacements take priority over earlier ones. - for i := len(f.Replace) - 1; i >= 0; i-- { - x := f.Replace[i] - if have[x.Old] { - kill[x.Syntax] = true - continue - } - have[x.Old] = true - } - var repl []*Replace - for _, x := range f.Replace { - if !kill[x.Syntax] { - repl = append(repl, x) - } - } - f.Replace = repl - - var stmts []Expr - for _, stmt := range f.Syntax.Stmt { - switch stmt := stmt.(type) { - case *Line: - if kill[stmt] { - continue - } - case *LineBlock: - var lines []*Line - for _, line := range stmt.Line { - if !kill[line] { - lines = append(lines, line) - } - } - stmt.Line = lines - if len(lines) == 0 { - continue - } - } - stmts = append(stmts, stmt) - } - f.Syntax.Stmt = stmts -} diff --git a/cmd/go/_internal_/modinfo/info.go b/cmd/go/_internal_/modinfo/info.go index e5a8406..5dcbbcc 100644 --- a/cmd/go/_internal_/modinfo/info.go +++ b/cmd/go/_internal_/modinfo/info.go @@ -20,8 +20,8 @@ type ModulePublic struct { Indirect bool `json:",omitempty"` // module is only indirectly needed by main module Dir string `json:",omitempty"` // directory holding local copy of files, if any GoMod string `json:",omitempty"` // path to go.mod file describing module, if any - Error *ModuleError `json:",omitempty"` // error loading module GoVersion string `json:",omitempty"` // go version used in module + Error *ModuleError `json:",omitempty"` // error loading module } type ModuleError struct { diff --git a/cmd/go/_internal_/modload/build.go b/cmd/go/_internal_/modload/build.go index 41d221d..bef3860 100644 --- a/cmd/go/_internal_/modload/build.go +++ b/cmd/go/_internal_/modload/build.go @@ -6,12 +6,6 @@ package modload import ( "bytes" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modinfo" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/module" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/search" "encoding/hex" "fmt" "github.com/dependabot/gomodules-extracted/_internal_/goroot" @@ -19,6 +13,15 @@ import ( "path/filepath" "runtime/debug" "strings" + + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modinfo" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/search" + + "golang.org/x/mod/module" + "golang.org/x/mod/semver" ) var ( @@ -38,18 +41,23 @@ func findStandardImportPath(path string) string { if goroot.IsStandardPackage(cfg.GOROOT, cfg.BuildContext.Compiler, path) { return filepath.Join(cfg.GOROOT, "src", path) } - if goroot.IsStandardPackage(cfg.GOROOT, cfg.BuildContext.Compiler, "vendor/"+path) { - return filepath.Join(cfg.GOROOT, "src/vendor", path) - } } return "" } +// PackageModuleInfo returns information about the module that provides +// a given package. If modules are not enabled or if the package is in the +// standard library or if the package was not successfully loaded with +// ImportPaths or a similar loading function, nil is returned. func PackageModuleInfo(pkgpath string) *modinfo.ModulePublic { if isStandardImportPath(pkgpath) || !Enabled() { return nil } - return moduleInfo(findModule(pkgpath, pkgpath), true) + m, ok := findModule(pkgpath) + if !ok { + return nil + } + return moduleInfo(m, true) } func ModuleInfo(path string) *modinfo.ModulePublic { @@ -77,13 +85,15 @@ func ModuleInfo(path string) *modinfo.ModulePublic { // addUpdate fills in m.Update if an updated version is available. func addUpdate(m *modinfo.ModulePublic) { - if m.Version != "" { - if info, err := Query(m.Path, "latest", Allowed); err == nil && info.Version != m.Version { - m.Update = &modinfo.ModulePublic{ - Path: m.Path, - Version: info.Version, - Time: &info.Time, - } + if m.Version == "" { + return + } + + if info, err := Query(m.Path, "upgrade", m.Version, Allowed); err == nil && semver.Compare(info.Version, m.Version) > 0 { + m.Update = &modinfo.ModulePublic{ + Path: m.Path, + Version: info.Version, + Time: &info.Time, } } } @@ -102,7 +112,7 @@ func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { } if HasModRoot() { info.Dir = ModRoot() - info.GoMod = filepath.Join(info.Dir, "go.mod") + info.GoMod = ModFilePath() if modFile.Go != nil { info.GoVersion = modFile.Go.Version } @@ -119,15 +129,10 @@ func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { info.GoVersion = loaded.goVersion[m.Path] } - if cfg.BuildMod == "vendor" { - info.Dir = filepath.Join(ModRoot(), "vendor", m.Path) - return info - } - - // complete fills in the extra fields in m. - complete := func(m *modinfo.ModulePublic) { + // completeFromModCache fills in the extra fields in m using the module cache. + completeFromModCache := func(m *modinfo.ModulePublic) { if m.Version != "" { - if q, err := Query(m.Path, m.Version, nil); err != nil { + if q, err := Query(m.Path, m.Version, "", nil); err != nil { m.Error = &modinfo.ModuleError{Err: err.Error()} } else { m.Version = q.Version @@ -143,21 +148,27 @@ func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { } dir, err := modfetch.DownloadDir(mod) if err == nil { - if info, err := os.Stat(dir); err == nil && info.IsDir() { - m.Dir = dir - } + m.Dir = dir } } } if !fromBuildList { - complete(info) + completeFromModCache(info) // Will set m.Error in vendor mode. return info } r := Replacement(m) if r.Path == "" { - complete(info) + if cfg.BuildMod == "vendor" { + // It's tempting to fill in the "Dir" field to point within the vendor + // directory, but that would be misleading: the vendor directory contains + // a flattened package tree, not complete modules, and it can even + // interleave packages from different modules if one module path is a + // prefix of the other. + } else { + completeFromModCache(info) + } return info } @@ -176,23 +187,30 @@ func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { } else { info.Replace.Dir = filepath.Join(ModRoot(), r.Path) } + info.Replace.GoMod = filepath.Join(info.Replace.Dir, "go.mod") + } + if cfg.BuildMod != "vendor" { + completeFromModCache(info.Replace) + info.Dir = info.Replace.Dir + info.GoMod = info.Replace.GoMod } - complete(info.Replace) - info.Dir = info.Replace.Dir - info.GoMod = filepath.Join(info.Dir, "go.mod") return info } +// PackageBuildInfo returns a string containing module version information +// for modules providing packages named by path and deps. path and deps must +// name packages that were resolved successfully with ImportPaths or one of +// the Load functions. func PackageBuildInfo(path string, deps []string) string { if isStandardImportPath(path) || !Enabled() { return "" } - target := findModule(path, path) + target := mustFindModule(path, path) mdeps := make(map[module.Version]bool) for _, dep := range deps { if !isStandardImportPath(dep) { - mdeps[findModule(path, dep)] = true + mdeps[mustFindModule(path, dep)] = true } } var mods []module.Version @@ -204,32 +222,34 @@ func PackageBuildInfo(path string, deps []string) string { var buf bytes.Buffer fmt.Fprintf(&buf, "path\t%s\n", path) - tv := target.Version - if tv == "" { - tv = "(devel)" - } - fmt.Fprintf(&buf, "mod\t%s\t%s\t%s\n", target.Path, tv, modfetch.Sum(target)) - for _, mod := range mods { - mv := mod.Version + + writeEntry := func(token string, m module.Version) { + mv := m.Version if mv == "" { mv = "(devel)" } - r := Replacement(mod) - h := "" - if r.Path == "" { - h = "\t" + modfetch.Sum(mod) - } - fmt.Fprintf(&buf, "dep\t%s\t%s%s\n", mod.Path, mod.Version, h) - if r.Path != "" { - fmt.Fprintf(&buf, "=>\t%s\t%s\t%s\n", r.Path, r.Version, modfetch.Sum(r)) + fmt.Fprintf(&buf, "%s\t%s\t%s", token, m.Path, mv) + if r := Replacement(m); r.Path == "" { + fmt.Fprintf(&buf, "\t%s\n", modfetch.Sum(m)) + } else { + fmt.Fprintf(&buf, "\n=>\t%s\t%s\t%s\n", r.Path, r.Version, modfetch.Sum(r)) } } + + writeEntry("mod", target) + for _, mod := range mods { + writeEntry("dep", mod) + } + return buf.String() } -// findModule returns the module containing the package at path, -// needed to build the package at target. -func findModule(target, path string) module.Version { +// mustFindModule is like findModule, but it calls base.Fatalf if the +// module can't be found. +// +// TODO(jayconrod): remove this. Callers should use findModule and return +// errors instead of relying on base.Fatalf. +func mustFindModule(target, path string) module.Version { pkg, ok := loaded.pkgCache.Get(path).(*loadPkg) if ok { if pkg.err != nil { @@ -249,22 +269,48 @@ func findModule(target, path string) module.Version { panic("unreachable") } -func ModInfoProg(info string) []byte { - // Inject a variable with the debug information as runtime/debug.modinfo, +// findModule searches for the module that contains the package at path. +// If the package was loaded with ImportPaths or one of the other loading +// functions, its containing module and true are returned. Otherwise, +// module.Version{} and false are returend. +func findModule(path string) (module.Version, bool) { + if pkg, ok := loaded.pkgCache.Get(path).(*loadPkg); ok { + return pkg.mod, pkg.mod != module.Version{} + } + if path == "command-line-arguments" { + return Target, true + } + return module.Version{}, false +} + +func ModInfoProg(info string, isgccgo bool) []byte { + // Inject a variable with the debug information as runtime.modinfo, // but compile it in package main so that it is specific to the binary. - // // The variable must be a literal so that it will have the correct value // before the initializer for package main runs. // - // We also want the value to be present even if runtime/debug.modinfo is - // otherwise unused in the rest of the program. Reading it in an init function - // suffices for now. + // The runtime startup code refers to the variable, which keeps it live + // in all binaries. + // + // Note: we use an alternate recipe below for gccgo (based on an + // init function) due to the fact that gccgo does not support + // applying a "//go:linkname" directive to a variable. This has + // drawbacks in that other packages may want to look at the module + // info in their init functions (see issue 29628), which won't + // work for gccgo. See also issue 30344. - return []byte(fmt.Sprintf(`package main + if !isgccgo { + return []byte(fmt.Sprintf(`package main import _ "unsafe" -//go:linkname __debug_modinfo__ runtime/debug.modinfo +//go:linkname __debug_modinfo__ runtime.modinfo var __debug_modinfo__ = %q -var keepalive_modinfo = __debug_modinfo__ -func init() { keepalive_modinfo = __debug_modinfo__ } `, string(infoStart)+info+string(infoEnd))) + } else { + return []byte(fmt.Sprintf(`package main +import _ "unsafe" +//go:linkname __set_debug_modinfo__ runtime.setmodinfo +func __set_debug_modinfo__(string) +func init() { __set_debug_modinfo__(%q) } + `, string(infoStart)+info+string(infoEnd))) + } } diff --git a/cmd/go/_internal_/modload/help.go b/cmd/go/_internal_/modload/help.go index 0a67b75..98a6113 100644 --- a/cmd/go/_internal_/modload/help.go +++ b/cmd/go/_internal_/modload/help.go @@ -19,34 +19,29 @@ including recording and resolving dependencies on other modules. Modules replace the old GOPATH-based approach to specifying which source files are used in a given build. -Preliminary module support - -Go 1.11 includes preliminary support for Go modules, -including a new module-aware 'go get' command. -We intend to keep revising this support, while preserving compatibility, -until it can be declared official (no longer preliminary), -and then at a later point we may remove support for work -in GOPATH and the old 'go get' command. - -The quickest way to take advantage of the new Go 1.11 module support -is to check out your repository into a directory outside GOPATH/src, -create a go.mod file (described in the next section) there, and run +Module support + +The go command includes support for Go modules. Module-aware mode is active +by default whenever a go.mod file is found in the current directory or in +any parent directory. + +The quickest way to take advantage of module support is to check out your +repository, create a go.mod file (described in the next section) there, and run go commands from within that file tree. -For more fine-grained control, the module support in Go 1.11 respects +For more fine-grained control, the go command continues to respect a temporary environment variable, GO111MODULE, which can be set to one of three string values: off, on, or auto (the default). -If GO111MODULE=off, then the go command never uses the -new module support. Instead it looks in vendor directories and GOPATH -to find dependencies; we now refer to this as "GOPATH mode." If GO111MODULE=on, then the go command requires the use of modules, -never consulting GOPATH. We refer to this as the command being -module-aware or running in "module-aware mode". -If GO111MODULE=auto or is unset, then the go command enables or -disables module support based on the current directory. -Module support is enabled only when the current directory is outside -GOPATH/src and itself contains a go.mod file or is below a directory -containing a go.mod file. +never consulting GOPATH. We refer to this as the command +being module-aware or running in "module-aware mode". +If GO111MODULE=off, then the go command never uses +module support. Instead it looks in vendor directories and GOPATH +to find dependencies; we now refer to this as "GOPATH mode." +If GO111MODULE=auto or is unset, then the go command enables or disables +module support based on the current directory. +Module support is enabled only when the current directory contains a +go.mod file or is below a directory containing a go.mod file. In module-aware mode, GOPATH no longer defines the meaning of imports during a build, but it still stores downloaded dependencies (in GOPATH/pkg/mod) @@ -181,9 +176,20 @@ not need updates, such as in a continuous integration and testing system. The "go get" command remains permitted to update go.mod even with -mod=readonly, and the "go mod" commands do not take the -mod flag (or any other build flags). -If invoked with -mod=vendor, the go command assumes that the vendor -directory holds the correct copies of dependencies and ignores -the dependency descriptions in go.mod. +If invoked with -mod=vendor, the go command loads packages from the main +module's vendor directory instead of downloading modules to and loading packages +from the module cache. The go command assumes the vendor directory holds +correct copies of dependencies, and it does not compute the set of required +module versions from go.mod files. However, the go command does check that +vendor/modules.txt (generated by 'go mod vendor') contains metadata consistent +with go.mod. + +If invoked with -mod=mod, the go command loads modules from the module cache +even if there is a vendor directory present. + +If the go command is not invoked with a -mod flag and the vendor directory +is present and the "go" version in go.mod is 1.14 or higher, the go command +will act as if it were invoked with -mod=vendor. Pseudo-versions @@ -237,12 +243,25 @@ evaluates to the available tagged version nearest to the comparison target The string "latest" matches the latest available tagged version, or else the underlying source repository's latest untagged revision. -A revision identifier for the underlying source repository, -such as a commit hash prefix, revision tag, or branch name, -selects that specific code revision. If the revision is -also tagged with a semantic version, the query evaluates to -that semantic version. Otherwise the query evaluates to a -pseudo-version for the commit. +The string "upgrade" is like "latest", but if the module is +currently required at a later version than the version "latest" +would select (for example, a newer pre-release version), "upgrade" +will select the later version instead. + +The string "patch" matches the latest available tagged version +of a module with the same major and minor version numbers as the +currently required version. If no version is currently required, +"patch" is equivalent to "latest". + +A revision identifier for the underlying source repository, such as +a commit hash prefix, revision tag, or branch name, selects that +specific code revision. If the revision is also tagged with a +semantic version, the query evaluates to that semantic version. +Otherwise the query evaluates to a pseudo-version for the commit. +Note that branches and tags with names that are matched by other +query syntax cannot be selected this way. For example, the query +"v2" means the latest version starting with "v2", not the branch +named "v2". All queries prefer release versions to pre-release versions. For example, " 0 { - var buf bytes.Buffer - fmt.Fprintf(&buf, "ambiguous import: found %s in multiple modules:", path) - for i, m := range mods { - fmt.Fprintf(&buf, "\n\t%s", m.Path) - if m.Version != "" { - fmt.Fprintf(&buf, " %s", m.Version) - } - fmt.Fprintf(&buf, " (%s)", dirs[i]) - } - return module.Version{}, "", errors.New(buf.String()) + return module.Version{}, "", &AmbiguousImportError{importPath: path, Dirs: dirs, Modules: mods} } // Look up module containing the package, for addition to the build list. // Goal is to determine the module, download it to dir, and return m, dir, ErrMissing. if cfg.BuildMod == "readonly" { - return module.Version{}, "", fmt.Errorf("import lookup disabled by -mod=%s", cfg.BuildMod) + var queryErr error + if !pathIsStd { + if cfg.BuildModReason == "" { + queryErr = fmt.Errorf("import lookup disabled by -mod=%s", cfg.BuildMod) + } else { + queryErr = fmt.Errorf("import lookup disabled by -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) + } + } + return module.Version{}, "", &ImportMissingError{Path: path, QueryErr: queryErr} + } + if modRoot == "" && !allowMissingModuleImports { + return module.Version{}, "", &ImportMissingError{ + Path: path, + QueryErr: errors.New("working directory is not part of a module"), + } } // Not on build list. @@ -139,7 +216,12 @@ func Import(path string) (m module.Version, dir string, err error) { latest := map[string]string{} // path -> version for _, r := range modFile.Replace { if maybeInModule(path, r.Old.Path) { - latest[r.Old.Path] = semver.Max(r.Old.Version, latest[r.Old.Path]) + // Don't use semver.Max here; need to preserve +incompatible suffix. + v := latest[r.Old.Path] + if semver.Compare(r.Old.Version, v) > 0 { + v = r.Old.Version + } + latest[r.Old.Path] = v } } @@ -171,21 +253,69 @@ func Import(path string) (m module.Version, dir string, err error) { // Report fetch error as above. return module.Version{}, "", err } - _, ok := dirInModule(path, m.Path, root, isLocal) - if ok { - return m, "", &ImportMissingError{ImportPath: path, Module: m} + if _, ok, err := dirInModule(path, m.Path, root, isLocal); err != nil { + return m, "", err + } else if ok { + return m, "", &ImportMissingError{Path: path, Module: m} + } + } + if len(mods) > 0 && module.CheckPath(path) != nil { + // The package path is not valid to fetch remotely, + // so it can only exist if in a replaced module, + // and we know from the above loop that it is not. + return module.Version{}, "", &PackageNotInModuleError{ + Mod: mods[0], + Query: "latest", + Pattern: path, + Replacement: Replacement(mods[0]), } } } - m, _, err = QueryPackage(path, "latest", Allowed) + if pathIsStd { + // This package isn't in the standard library, isn't in any module already + // in the build list, and isn't in any other module that the user has + // shimmed in via a "replace" directive. + // Moreover, the import path is reserved for the standard library, so + // QueryPackage cannot possibly find a module containing this package. + // + // Instead of trying QueryPackage, report an ImportMissingError immediately. + return module.Version{}, "", &ImportMissingError{Path: path} + } + + fmt.Fprintf(os.Stderr, "go: finding module for package %s\n", path) + + candidates, err := QueryPackage(path, "latest", Allowed) if err != nil { - if _, ok := err.(*codehost.VCSError); ok { + if errors.Is(err, os.ErrNotExist) { + // Return "cannot find module providing package […]" instead of whatever + // low-level error QueryPackage produced. + return module.Version{}, "", &ImportMissingError{Path: path, QueryErr: err} + } else { return module.Version{}, "", err } - return module.Version{}, "", &ImportMissingError{ImportPath: path} } - return m, "", &ImportMissingError{ImportPath: path, Module: m} + m = candidates[0].Mod + newMissingVersion := "" + for _, c := range candidates { + cm := c.Mod + for _, bm := range buildList { + if bm.Path == cm.Path && semver.Compare(bm.Version, cm.Version) > 0 { + // QueryPackage proposed that we add module cm to provide the package, + // but we already depend on a newer version of that module (and we don't + // have the package). + // + // This typically happens when a package is present at the "@latest" + // version (e.g., v1.0.0) of a module, but we have a newer version + // of the same module in the build list (e.g., v1.0.1-beta), and + // the package is not present there. + m = cm + newMissingVersion = bm.Version + break + } + } + } + return m, "", &ImportMissingError{Path: path, Module: m, newMissingVersion: newMissingVersion} } // maybeInModule reports whether, syntactically, @@ -196,19 +326,29 @@ func maybeInModule(path, mpath string) bool { len(path) > len(mpath) && path[len(mpath)] == '/' && path[:len(mpath)] == mpath } -var haveGoModCache, haveGoFilesCache par.Cache +var ( + haveGoModCache par.Cache // dir → bool + haveGoFilesCache par.Cache // dir → goFilesEntry +) + +type goFilesEntry struct { + haveGoFiles bool + err error +} // dirInModule locates the directory that would hold the package named by the given path, // if it were in the module with module path mpath and root mdir. // If path is syntactically not within mpath, // or if mdir is a local file tree (isLocal == true) and the directory // that would hold path is in a sub-module (covered by a go.mod below mdir), -// dirInModule returns "", false. +// dirInModule returns "", false, nil. // // Otherwise, dirInModule returns the name of the directory where // Go source files would be expected, along with a boolean indicating // whether there are in fact Go source files in that directory. -func dirInModule(path, mpath, mdir string, isLocal bool) (dir string, haveGoFiles bool) { +// A non-nil error indicates that the existence of the directory and/or +// source files could not be determined, for example due to a permission error. +func dirInModule(path, mpath, mdir string, isLocal bool) (dir string, haveGoFiles bool, err error) { // Determine where to expect the package. if path == mpath { dir = mdir @@ -217,7 +357,7 @@ func dirInModule(path, mpath, mdir string, isLocal bool) (dir string, haveGoFile } else if len(path) > len(mpath) && path[len(mpath)] == '/' && path[:len(mpath)] == mpath { dir = filepath.Join(mdir, path[len(mpath)+1:]) } else { - return "", false + return "", false, nil } // Check that there aren't other modules in the way. @@ -229,12 +369,12 @@ func dirInModule(path, mpath, mdir string, isLocal bool) (dir string, haveGoFile if isLocal { for d := dir; d != mdir && len(d) > len(mdir); { haveGoMod := haveGoModCache.Do(d, func() interface{} { - _, err := os.Stat(filepath.Join(d, "go.mod")) - return err == nil + fi, err := os.Stat(filepath.Join(d, "go.mod")) + return err == nil && !fi.IsDir() }).(bool) if haveGoMod { - return "", false + return "", false, nil } parent := filepath.Dir(d) if parent == d { @@ -251,23 +391,58 @@ func dirInModule(path, mpath, mdir string, isLocal bool) (dir string, haveGoFile // Are there Go source files in the directory? // We don't care about build tags, not even "+build ignore". // We're just looking for a plausible directory. - haveGoFiles = haveGoFilesCache.Do(dir, func() interface{} { - f, err := os.Open(dir) - if err != nil { - return false + res := haveGoFilesCache.Do(dir, func() interface{} { + ok, err := isDirWithGoFiles(dir) + return goFilesEntry{haveGoFiles: ok, err: err} + }).(goFilesEntry) + + return dir, res.haveGoFiles, res.err +} + +func isDirWithGoFiles(dir string) (bool, error) { + f, err := os.Open(dir) + if err != nil { + if os.IsNotExist(err) { + return false, nil } - defer f.Close() - names, _ := f.Readdirnames(-1) - for _, name := range names { - if strings.HasSuffix(name, ".go") { - info, err := os.Stat(filepath.Join(dir, name)) - if err == nil && info.Mode().IsRegular() { - return true + return false, err + } + defer f.Close() + + names, firstErr := f.Readdirnames(-1) + if firstErr != nil { + if fi, err := f.Stat(); err == nil && !fi.IsDir() { + return false, nil + } + + // Rewrite the error from ReadDirNames to include the path if not present. + // See https://golang.org/issue/38923. + var pe *os.PathError + if !errors.As(firstErr, &pe) { + firstErr = &os.PathError{Op: "readdir", Path: dir, Err: firstErr} + } + } + + for _, name := range names { + if strings.HasSuffix(name, ".go") { + info, err := os.Stat(filepath.Join(dir, name)) + if err == nil && info.Mode().IsRegular() { + // If any .go source file exists, the package exists regardless of + // errors for other source files. Leave further error reporting for + // later. + return true, nil + } + if firstErr == nil { + if os.IsNotExist(err) { + // If the file was concurrently deleted, or was a broken symlink, + // convert the error to an opaque error instead of one matching + // os.IsNotExist. + err = errors.New(err.Error()) } + firstErr = err } } - return false - }).(bool) + } - return dir, haveGoFiles + return false, firstErr } diff --git a/cmd/go/_internal_/modload/init.go b/cmd/go/_internal_/modload/init.go index 18468a5..31568f9 100644 --- a/cmd/go/_internal_/modload/init.go +++ b/cmd/go/_internal_/modload/init.go @@ -6,46 +6,56 @@ package modload import ( "bytes" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cache" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/load" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modconv" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch/codehost" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfile" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/module" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/mvs" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/renameio" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/search" "encoding/json" + "errors" "fmt" "go/build" + "github.com/dependabot/gomodules-extracted/_internal_/lazyregexp" "io/ioutil" "os" "path" "path/filepath" - "regexp" "runtime/debug" "strconv" "strings" + + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cache" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/load" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/lockedfile" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modconv" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/mvs" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/search" + + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" + "golang.org/x/mod/semver" ) var ( - cwd string // TODO(bcmills): Is this redundant with base.Cwd? - MustUseModules = mustUseModules() + mustUseModules = false initialized bool - modRoot string - modFile *modfile.File - modFileData []byte - excluded map[module.Version]bool - Target module.Version + modRoot string + Target module.Version + + // targetPrefix is the path prefix for packages in Target, without a trailing + // slash. For most modules, targetPrefix is just Target.Path, but the + // standard-library module "std" has an empty prefix. + targetPrefix string + + // targetInGorootSrc caches whether modRoot is within GOROOT/src. + // The "std" module is special within GOROOT/src, but not otherwise. + targetInGorootSrc bool gopath string CmdModInit bool // running 'go mod init' CmdModModule string // module argument for 'go mod init' + + allowMissingModuleImports bool ) // ModFile returns the parsed go.mod file. @@ -69,18 +79,6 @@ func BinDir() string { return filepath.Join(gopath, "bin") } -// mustUseModules reports whether we are invoked as vgo -// (as opposed to go). -// If so, we only support builds with go.mod files. -func mustUseModules() bool { - name := os.Args[0] - name = name[strings.LastIndex(name, "/")+1:] - name = name[strings.LastIndex(name, `\`)+1:] - return strings.HasPrefix(name, "vgo") -} - -var inGOPATH bool // running in GOPATH/src - // Init determines whether module mode is enabled, locates the root of the // current module (if any), sets environment variables for Git subprocesses, and // configures the cfg, codehost, load, modfetch, and search packages for use @@ -91,18 +89,20 @@ func Init() { } initialized = true - env := os.Getenv("GO111MODULE") + // Keep in sync with WillBeEnabled. We perform extra validation here, and + // there are lots of diagnostics and side effects, so we can't use + // WillBeEnabled directly. + env := cfg.Getenv("GO111MODULE") switch env { default: base.Fatalf("go: unknown environment setting GO111MODULE=%s", env) - case "", "auto": - // leave MustUseModules alone + case "auto", "": + mustUseModules = false case "on": - MustUseModules = true + mustUseModules = true case "off": - if !MustUseModules { - return - } + mustUseModules = false + return } // Disable any prompting for passwords by Git. @@ -132,42 +132,17 @@ func Init() { os.Setenv("GIT_SSH_COMMAND", "ssh -o ControlMaster=no") } - var err error - cwd, err = os.Getwd() - if err != nil { - base.Fatalf("go: %v", err) - } - - inGOPATH = false - for _, gopath := range filepath.SplitList(cfg.BuildContext.GOPATH) { - if gopath == "" { - continue - } - if search.InDir(cwd, filepath.Join(gopath, "src")) != "" { - inGOPATH = true - break - } - } - - if inGOPATH && !MustUseModules { - if CmdModInit { - die() // Don't init a module that we're just going to ignore. - } - // No automatic enabling in GOPATH. - if root, _ := FindModuleRoot(cwd, "", false); root != "" { - cfg.GoModInGOPATH = filepath.Join(root, "go.mod") - } - return - } - if CmdModInit { // Running 'go mod init': go.mod will be created in current directory. - modRoot = cwd + modRoot = base.Cwd } else { - modRoot, _ = FindModuleRoot(cwd, "", MustUseModules) + modRoot = findModuleRoot(base.Cwd) if modRoot == "" { - if !MustUseModules { - // GO111MODULE is 'auto' (or unset), and we can't find a module root. + if cfg.ModFile != "" { + base.Fatalf("go: cannot find main module, but -modfile was set.\n\t-modfile cannot be used to set the module root directory.") + } + if !mustUseModules { + // GO111MODULE is 'auto', and we can't find a module root. // Stay in GOPATH mode. return } @@ -181,6 +156,9 @@ func Init() { fmt.Fprintf(os.Stderr, "go: warning: ignoring go.mod in system temp root %v\n", os.TempDir()) } } + if cfg.ModFile != "" && !strings.HasSuffix(cfg.ModFile, ".mod") { + base.Fatalf("go: -modfile=%s: file does not have .mod extension", cfg.ModFile) + } // We're in module mode. Install the hooks to make it work. @@ -199,17 +177,6 @@ func Init() { base.Fatalf("$GOPATH/go.mod exists but should not") } - oldSrcMod := filepath.Join(list[0], "src/mod") - pkgMod := filepath.Join(list[0], "pkg/mod") - infoOld, errOld := os.Stat(oldSrcMod) - _, errMod := os.Stat(pkgMod) - if errOld == nil && infoOld.IsDir() && errMod != nil && os.IsNotExist(errMod) { - os.Rename(oldSrcMod, pkgMod) - } - - modfetch.PkgMod = pkgMod - codehost.WorkRoot = filepath.Join(pkgMod, "cache/vcs") - cfg.ModulesEnabled = true load.ModBinDir = BinDir load.ModLookup = Lookup @@ -223,41 +190,77 @@ func Init() { if modRoot == "" { // We're in module mode, but not inside a module. // - // If the command is 'go get' or 'go list' and all of the args are in the - // same existing module, we could use that module's download directory in - // the module cache as the module root, applying any replacements and/or - // exclusions specified by that module. However, that would leave us in a - // strange state: we want 'go get' to be consistent with 'go list', and 'go - // list' should be able to operate on multiple modules. Moreover, the 'get' - // target might specify relative file paths (e.g. in the same repository) as - // replacements, and we would not be able to apply those anyway: we would - // need to either error out or ignore just those replacements, when a build - // from an empty module could proceed without error. + // Commands like 'go build', 'go run', 'go list' have no go.mod file to + // read or write. They would need to find and download the latest versions + // of a potentially large number of modules with no way to save version + // information. We can succeed slowly (but not reproducibly), but that's + // not usually a good experience. // - // Instead, we'll operate as though we're in some ephemeral external module, - // ignoring all replacements and exclusions uniformly. - - // Normally we check sums using the go.sum file from the main module, but - // without a main module we do not have an authoritative go.sum file. + // Instead, we forbid resolving import paths to modules other than std and + // cmd. Users may still build packages specified with .go files on the + // command line, but they'll see an error if those files import anything + // outside std. // - // TODO(bcmills): In Go 1.13, check sums when outside the main module. + // This can be overridden by calling AllowMissingModuleImports. + // For example, 'go get' does this, since it is expected to resolve paths. // - // One possible approach is to merge the go.sum files from all of the - // modules we download: that doesn't protect us against bad top-level - // modules, but it at least ensures consistency for transitive dependencies. + // See golang.org/issue/32027. } else { - modfetch.GoSumFile = filepath.Join(modRoot, "go.sum") + modfetch.GoSumFile = strings.TrimSuffix(ModFilePath(), ".mod") + ".sum" search.SetModRoot(modRoot) } } func init() { load.ModInit = Init +} - // Set modfetch.PkgMod unconditionally, so that go clean -modcache can run even without modules enabled. - if list := filepath.SplitList(cfg.BuildContext.GOPATH); len(list) > 0 && list[0] != "" { - modfetch.PkgMod = filepath.Join(list[0], "pkg/mod") +// WillBeEnabled checks whether modules should be enabled but does not +// initialize modules by installing hooks. If Init has already been called, +// WillBeEnabled returns the same result as Enabled. +// +// This function is needed to break a cycle. The main package needs to know +// whether modules are enabled in order to install the module or GOPATH version +// of 'go get', but Init reads the -modfile flag in 'go get', so it shouldn't +// be called until the command is installed and flags are parsed. Instead of +// calling Init and Enabled, the main package can call this function. +func WillBeEnabled() bool { + if modRoot != "" || mustUseModules { + return true } + if initialized { + return false + } + + // Keep in sync with Init. Init does extra validation and prints warnings or + // exits, so it can't call this function directly. + env := cfg.Getenv("GO111MODULE") + switch env { + case "on": + return true + case "auto", "": + break + default: + return false + } + + if CmdModInit { + // Running 'go mod init': go.mod will be created in current directory. + return true + } + if modRoot := findModuleRoot(base.Cwd); modRoot == "" { + // GO111MODULE is 'auto', and we can't find a module root. + // Stay in GOPATH mode. + return false + } else if search.InDir(modRoot, os.TempDir()) == "." { + // If you create /tmp/go.mod for experimenting, + // then any tests that create work directories under /tmp + // will find it and get modules when they're not expecting them. + // It's a bit of a peculiar thing to disallow but quite mysterious + // when it happens. See golang.org/issue/26708. + return false + } + return true } // Enabled reports whether modules are (or must be) enabled. @@ -266,7 +269,7 @@ func init() { // (usually through MustModRoot). func Enabled() bool { Init() - return modRoot != "" || MustUseModules + return modRoot != "" || mustUseModules } // ModRoot returns the root of the main module. @@ -286,6 +289,20 @@ func HasModRoot() bool { return modRoot != "" } +// ModFilePath returns the effective path of the go.mod file. Normally, this +// "go.mod" in the directory returned by ModRoot, but the -modfile flag may +// change its location. ModFilePath calls base.Fatalf if there is no main +// module, even if -modfile is set. +func ModFilePath() string { + if !HasModRoot() { + die() + } + if cfg.ModFile != "" { + return cfg.ModFile + } + return filepath.Join(modRoot, "go.mod") +} + // printStackInDie causes die to print a stack trace. // // It is enabled by the testgo tag, and helps to diagnose paths that @@ -296,17 +313,28 @@ func die() { if printStackInDie { debug.PrintStack() } - if os.Getenv("GO111MODULE") == "off" { + if cfg.Getenv("GO111MODULE") == "off" { base.Fatalf("go: modules disabled by GO111MODULE=off; see 'go help modules'") } - if inGOPATH && !MustUseModules { - base.Fatalf("go: modules disabled inside GOPATH/src by GO111MODULE=auto; see 'go help modules'") + if dir, name := findAltConfig(base.Cwd); dir != "" { + rel, err := filepath.Rel(base.Cwd, dir) + if err != nil { + rel = dir + } + cdCmd := "" + if rel != "." { + cdCmd = fmt.Sprintf("cd %s && ", rel) + } + base.Fatalf("go: cannot find main module, but found %s in %s\n\tto create a module there, run:\n\t%sgo mod init", name, dir, cdCmd) } base.Fatalf("go: cannot find main module; see 'go help modules'") } // InitMod sets Target and, if there is a main module, parses the initial build // list from its go.mod file, creating and populating that file if needed. +// +// As a side-effect, InitMod sets a default for cfg.BuildMod if it does not +// already have an explicit value. func InitMod() { if len(buildList) > 0 { return @@ -315,6 +343,7 @@ func InitMod() { Init() if modRoot == "" { Target = module.Version{Path: "command-line-arguments"} + targetPrefix = "command-line-arguments" buildList = []module.Version{Target} return } @@ -327,29 +356,24 @@ func InitMod() { return } - gomod := filepath.Join(modRoot, "go.mod") - data, err := ioutil.ReadFile(gomod) + gomod := ModFilePath() + data, err := lockedfile.Read(gomod) if err != nil { - if os.IsNotExist(err) { - legacyModInit() - modFileToBuildList() - WriteGoMod() - return - } base.Fatalf("go: %v", err) } - f, err := modfile.Parse(gomod, data, fixVersion) + var fixed bool + f, err := modfile.Parse(gomod, data, fixVersion(&fixed)) if err != nil { // Errors returned by modfile.Parse begin with file:line. base.Fatalf("go: errors parsing go.mod:\n%s\n", err) } modFile = f - modFileData = data + index = indexModFile(data, f, fixed) if len(f.Syntax.Stmt) == 0 || f.Module == nil { // Empty mod file. Must add module path. - path, err := FindModulePath(modRoot) + path, err := findModulePath(modRoot) if err != nil { base.Fatalf("go: %v", err) } @@ -362,17 +386,83 @@ func InitMod() { legacyModInit() } - excluded = make(map[module.Version]bool) - for _, x := range f.Exclude { - excluded[x.Mod] = true - } modFileToBuildList() - WriteGoMod() + setDefaultBuildMod() + if cfg.BuildMod == "vendor" { + readVendorList() + checkVendorConsistency() + } else { + // TODO(golang.org/issue/33326): if cfg.BuildMod != "readonly"? + WriteGoMod() + } +} + +// fixVersion returns a modfile.VersionFixer implemented using the Query function. +// +// It resolves commit hashes and branch names to versions, +// canonicalizes versions that appeared in early vgo drafts, +// and does nothing for versions that already appear to be canonical. +// +// The VersionFixer sets 'fixed' if it ever returns a non-canonical version. +func fixVersion(fixed *bool) modfile.VersionFixer { + return func(path, vers string) (resolved string, err error) { + defer func() { + if err == nil && resolved != vers { + *fixed = true + } + }() + + // Special case: remove the old -gopkgin- hack. + if strings.HasPrefix(path, "gopkg.in/") && strings.Contains(vers, "-gopkgin-") { + vers = vers[strings.Index(vers, "-gopkgin-")+len("-gopkgin-"):] + } + + // fixVersion is called speculatively on every + // module, version pair from every go.mod file. + // Avoid the query if it looks OK. + _, pathMajor, ok := module.SplitPathVersion(path) + if !ok { + return "", &module.ModuleError{ + Path: path, + Err: &module.InvalidVersionError{ + Version: vers, + Err: fmt.Errorf("malformed module path %q", path), + }, + } + } + if vers != "" && module.CanonicalVersion(vers) == vers { + if err := module.CheckPathMajor(vers, pathMajor); err == nil { + return vers, nil + } + } + + info, err := Query(path, vers, "", nil) + if err != nil { + return "", err + } + return info.Version, nil + } +} + +// AllowMissingModuleImports allows import paths to be resolved to modules +// when there is no module root. Normally, this is forbidden because it's slow +// and there's no way to make the result reproducible, but some commands +// like 'go get' are expected to do this. +func AllowMissingModuleImports() { + allowMissingModuleImports = true } // modFileToBuildList initializes buildList from the modFile. func modFileToBuildList() { Target = modFile.Module.Mod + targetPrefix = Target.Path + if rel := search.InDir(base.Cwd, cfg.GOROOTsrc); rel != "" { + targetInGorootSrc = true + if Target.Path == "std" { + targetPrefix = "" + } + } + list := []module.Version{Target} for _, r := range modFile.Require { list = append(list, r.Mod) @@ -380,24 +470,63 @@ func modFileToBuildList() { buildList = list } -// Allowed reports whether module m is allowed (not excluded) by the main module's go.mod. -func Allowed(m module.Version) bool { - return !excluded[m] +// setDefaultBuildMod sets a default value for cfg.BuildMod +// if it is currently empty. +func setDefaultBuildMod() { + if cfg.BuildMod != "" { + // Don't override an explicit '-mod=' argument. + return + } + cfg.BuildMod = "mod" + if cfg.CmdName == "get" || strings.HasPrefix(cfg.CmdName, "mod ") { + // Don't set -mod implicitly for commands whose purpose is to + // manipulate the build list. + return + } + if modRoot == "" { + return + } + + if fi, err := os.Stat(filepath.Join(modRoot, "vendor")); err == nil && fi.IsDir() { + modGo := "unspecified" + if index.goVersion != "" { + if semver.Compare("v"+index.goVersion, "v1.14") >= 0 { + // The Go version is at least 1.14, and a vendor directory exists. + // Set -mod=vendor by default. + cfg.BuildMod = "vendor" + cfg.BuildModReason = "Go version in go.mod is at least 1.14 and vendor directory exists." + return + } else { + modGo = index.goVersion + } + } + + // Since a vendor directory exists, we have a non-trivial reason for + // choosing -mod=mod, although it probably won't be used for anything. + // Record the reason anyway for consistency. + // It may be overridden if we switch to mod=readonly below. + cfg.BuildModReason = fmt.Sprintf("Go version in go.mod is %s.", modGo) + } + + p := ModFilePath() + if fi, err := os.Stat(p); err == nil && !hasWritePerm(p, fi) { + cfg.BuildMod = "readonly" + cfg.BuildModReason = "go.mod file is read-only." + } } func legacyModInit() { if modFile == nil { - path, err := FindModulePath(modRoot) + path, err := findModulePath(modRoot) if err != nil { base.Fatalf("go: %v", err) } fmt.Fprintf(os.Stderr, "go: creating new go.mod: module %s\n", path) modFile = new(modfile.File) modFile.AddModuleStmt(path) + addGoStmt() // Add the go directive before converted module requirements. } - addGoStmt() - for _, name := range altConfigs { cfg := filepath.Join(modRoot, name) data, err := ioutil.ReadFile(cfg) @@ -420,15 +549,12 @@ func legacyModInit() { } } -// InitGoStmt adds a go statement, unless there already is one. -func InitGoStmt() { - if modFile.Go == nil { - addGoStmt() - } -} - -// addGoStmt adds a go statement referring to the current version. +// addGoStmt adds a go directive to the go.mod file if it does not already include one. +// The 'go' version added, if any, is the latest version supported by this toolchain. func addGoStmt() { + if modFile.Go != nil && modFile.Go.Version != "" { + return + } tags := build.Default.ReleaseTags version := tags[len(tags)-1] if !strings.HasPrefix(version, "go") || !modfile.GoVersionRE.MatchString(version[2:]) { @@ -454,19 +580,16 @@ var altConfigs = []string{ ".git/config", } -// Exported only for testing. -func FindModuleRoot(dir, limit string, legacyConfigOK bool) (root, file string) { +func findModuleRoot(dir string) (root string) { + if dir == "" { + panic("dir not set") + } dir = filepath.Clean(dir) - dir1 := dir - limit = filepath.Clean(limit) // Look for enclosing go.mod. for { if fi, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil && !fi.IsDir() { - return dir, "go.mod" - } - if dir == limit { - break + return dir } d := filepath.Dir(dir) if d == dir { @@ -474,37 +597,49 @@ func FindModuleRoot(dir, limit string, legacyConfigOK bool) (root, file string) } dir = d } + return "" +} - // Failing that, look for enclosing alternate version config. - if legacyConfigOK { - dir = dir1 - for { - for _, name := range altConfigs { - if fi, err := os.Stat(filepath.Join(dir, name)); err == nil && !fi.IsDir() { - return dir, name - } - } - if dir == limit { - break - } - d := filepath.Dir(dir) - if d == dir { - break +func findAltConfig(dir string) (root, name string) { + if dir == "" { + panic("dir not set") + } + dir = filepath.Clean(dir) + if rel := search.InDir(dir, cfg.BuildContext.GOROOT); rel != "" { + // Don't suggest creating a module from $GOROOT/.git/config + // or a config file found in any parent of $GOROOT (see #34191). + return "", "" + } + for { + for _, name := range altConfigs { + if fi, err := os.Stat(filepath.Join(dir, name)); err == nil && !fi.IsDir() { + return dir, name } - dir = d } + d := filepath.Dir(dir) + if d == dir { + break + } + dir = d } - return "", "" } -// Exported only for testing. -func FindModulePath(dir string) (string, error) { +func findModulePath(dir string) (string, error) { if CmdModModule != "" { // Running go mod init x/y/z; return x/y/z. + if err := module.CheckImportPath(CmdModModule); err != nil { + return "", err + } return CmdModModule, nil } + // TODO(bcmills): once we have located a plausible module path, we should + // query version control (if available) to verify that it matches the major + // version of the most recent tag. + // See https://golang.org/issue/29433, https://golang.org/issue/27009, and + // https://golang.org/issue/31549. + // Cast about for import comments, // first in top-level directory, then in subdirectories. list, _ := ioutil.ReadDir(dir) @@ -554,18 +689,19 @@ func FindModulePath(dir string) (string, error) { } } - // Look for .git/config with github origin as last resort. - data, _ = ioutil.ReadFile(filepath.Join(dir, ".git/config")) - if m := gitOriginRE.FindSubmatch(data); m != nil { - return "github.com/" + string(m[1]), nil - } + msg := `cannot determine module path for source directory %s (outside GOPATH, module path must be specified) + +Example usage: + 'go mod init example.com/m' to initialize a v0 or v1 module + 'go mod init example.com/m/v2' to initialize a v2 module - return "", fmt.Errorf("cannot determine module path for source directory %s (outside GOPATH, no import comments)", dir) +Run 'go help mod init' for more information. +` + return "", fmt.Errorf(msg, dir) } var ( - gitOriginRE = regexp.MustCompile(`(?m)^\[remote "origin"\]\r?\n\turl = (?:https://github.com/|git@github.com:|gh:)([^/]+/[^/]+?)(\.git)?\r?\n`) - importCommentRE = regexp.MustCompile(`(?m)^package[ \t]+[^ \t\r\n/]+[ \t]+//[ \t]+import[ \t]+(\"[^"]+\")[ \t]*\r?\n`) + importCommentRE = lazyregexp.New(`(?m)^package[ \t]+[^ \t\r\n/]+[ \t]+//[ \t]+import[ \t]+(\"[^"]+\")[ \t]*\r?\n`) ) func findImportComment(file string) string { @@ -599,16 +735,17 @@ func AllowWriteGoMod() { allowWriteGoMod = true } -// MinReqs returns a Reqs with minimal dependencies of Target, +// MinReqs returns a Reqs with minimal additional dependencies of Target, // as will be written to go.mod. func MinReqs() mvs.Reqs { - var direct []string + var retain []string for _, m := range buildList[1:] { - if loaded.direct[m.Path] { - direct = append(direct, m.Path) + _, explicit := index.require[m] + if explicit || loaded.direct[m.Path] { + retain = append(retain, m.Path) } } - min, err := mvs.Req(Target, buildList, direct, Reqs()) + min, err := mvs.Req(Target, retain, Reqs()) if err != nil { base.Fatalf("go: %v", err) } @@ -629,6 +766,10 @@ func WriteGoMod() { return } + if cfg.BuildMod != "readonly" { + addGoStmt() + } + if loaded != nil { reqs := MinReqs() min, err := reqs.Required(Target) @@ -644,76 +785,67 @@ func WriteGoMod() { } modFile.SetRequire(list) } + modFile.Cleanup() - modFile.Cleanup() // clean file after edits - new, err := modFile.Format() - if err != nil { - base.Fatalf("go: %v", err) + dirty := index.modFileIsDirty(modFile) + if dirty && cfg.BuildMod == "readonly" { + // If we're about to fail due to -mod=readonly, + // prefer to report a dirty go.mod over a dirty go.sum + if cfg.BuildModReason != "" { + base.Fatalf("go: updates to go.mod needed, disabled by -mod=readonly\n\t(%s)", cfg.BuildModReason) + } else { + base.Fatalf("go: updates to go.mod needed, disabled by -mod=readonly") + } } - // Always update go.sum, even if we didn't change go.mod: we may have // downloaded modules that we didn't have before. modfetch.WriteGoSum() - if bytes.Equal(new, modFileData) { - // We don't need to modify go.mod from what we read previously. + if !dirty && cfg.CmdName != "mod tidy" { + // The go.mod file has the same semantic content that it had before + // (but not necessarily the same exact bytes). // Ignore any intervening edits. return } - if cfg.BuildMod == "readonly" { - base.Fatalf("go: updates to go.mod needed, disabled by -mod=readonly") + + new, err := modFile.Format() + if err != nil { + base.Fatalf("go: %v", err) + } + defer func() { + // At this point we have determined to make the go.mod file on disk equal to new. + index = indexModFile(new, modFile, false) + }() + + // Make a best-effort attempt to acquire the side lock, only to exclude + // previous versions of the 'go' command from making simultaneous edits. + if unlock, err := modfetch.SideLock(); err == nil { + defer unlock() } - unlock := modfetch.SideLock() - defer unlock() + errNoChange := errors.New("no update needed") - file := filepath.Join(modRoot, "go.mod") - old, err := ioutil.ReadFile(file) - if !bytes.Equal(old, modFileData) { + err = lockedfile.Transform(ModFilePath(), func(old []byte) ([]byte, error) { if bytes.Equal(old, new) { - // Some other process wrote the same go.mod file that we were about to write. - modFileData = new - return - } - if err != nil { - base.Fatalf("go: can't determine whether go.mod has changed: %v", err) + // The go.mod file is already equal to new, possibly as the result of some + // other process. + return nil, errNoChange } - // The contents of the go.mod file have changed. In theory we could add all - // of the new modules to the build list, recompute, and check whether any - // module in *our* build list got bumped to a different version, but that's - // a lot of work for marginal benefit. Instead, fail the command: if users - // want to run concurrent commands, they need to start with a complete, - // consistent module definition. - base.Fatalf("go: updates to go.mod needed, but contents have changed") - } - - if err := renameio.WriteFile(file, new); err != nil { - base.Fatalf("error writing go.mod: %v", err) - } - modFileData = new -} - -func fixVersion(path, vers string) (string, error) { - // Special case: remove the old -gopkgin- hack. - if strings.HasPrefix(path, "gopkg.in/") && strings.Contains(vers, "-gopkgin-") { - vers = vers[strings.Index(vers, "-gopkgin-")+len("-gopkgin-"):] - } + if index != nil && !bytes.Equal(old, index.data) { + // The contents of the go.mod file have changed. In theory we could add all + // of the new modules to the build list, recompute, and check whether any + // module in *our* build list got bumped to a different version, but that's + // a lot of work for marginal benefit. Instead, fail the command: if users + // want to run concurrent commands, they need to start with a complete, + // consistent module definition. + return nil, fmt.Errorf("existing contents have changed since last read") + } - // fixVersion is called speculatively on every - // module, version pair from every go.mod file. - // Avoid the query if it looks OK. - _, pathMajor, ok := module.SplitPathVersion(path) - if !ok { - return "", fmt.Errorf("malformed module path: %s", path) - } - if vers != "" && module.CanonicalVersion(vers) == vers && module.MatchPathMajor(vers, pathMajor) { - return vers, nil - } + return new, nil + }) - info, err := Query(path, vers, nil) - if err != nil { - return "", err + if err != nil && err != errNoChange { + base.Fatalf("go: updating go.mod: %v", err) } - return info.Version, nil } diff --git a/cmd/go/_internal_/modload/list.go b/cmd/go/_internal_/modload/list.go index bd92fee..12fae9c 100644 --- a/cmd/go/_internal_/modload/list.go +++ b/cmd/go/_internal_/modload/list.go @@ -5,15 +5,18 @@ package modload import ( + "errors" "fmt" "os" "strings" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modinfo" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/module" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/par" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/search" + + "golang.org/x/mod/module" ) func ListModules(args []string, listU, listVersions bool) []*modinfo.ModulePublic { @@ -54,19 +57,30 @@ func listModules(args []string, listVersions bool) []*modinfo.ModulePublic { if search.IsRelativePath(arg) { base.Fatalf("go: cannot use relative path %s to specify module", arg) } + if !HasModRoot() && (arg == "all" || strings.Contains(arg, "...")) { + base.Fatalf("go: cannot match %q: working directory is not part of a module", arg) + } if i := strings.Index(arg, "@"); i >= 0 { - info, err := Query(arg[:i], arg[i+1:], nil) + path := arg[:i] + vers := arg[i+1:] + var current string + for _, m := range buildList { + if m.Path == path { + current = m.Version + break + } + } + + info, err := Query(path, vers, current, nil) if err != nil { mods = append(mods, &modinfo.ModulePublic{ - Path: arg[:i], - Version: arg[i+1:], - Error: &modinfo.ModuleError{ - Err: err.Error(), - }, + Path: path, + Version: vers, + Error: modinfoError(path, vers, err), }) continue } - mods = append(mods, moduleInfo(module.Version{Path: arg[:i], Version: info.Version}, false)) + mods = append(mods, moduleInfo(module.Version{Path: path, Version: info.Version}, false)) continue } @@ -101,18 +115,31 @@ func listModules(args []string, listVersions bool) []*modinfo.ModulePublic { // Don't make the user provide an explicit '@latest' when they're // explicitly asking what the available versions are. // Instead, resolve the module, even if it isn't an existing dependency. - info, err := Query(arg, "latest", nil) + info, err := Query(arg, "latest", "", nil) if err == nil { mods = append(mods, moduleInfo(module.Version{Path: arg, Version: info.Version}, false)) - continue + } else { + mods = append(mods, &modinfo.ModulePublic{ + Path: arg, + Error: modinfoError(arg, "", err), + }) } + continue + } + if cfg.BuildMod == "vendor" { + // In vendor mode, we can't determine whether a missing module is “a + // known dependency” because the module graph is incomplete. + // Give a more explicit error message. + mods = append(mods, &modinfo.ModulePublic{ + Path: arg, + Error: modinfoError(arg, "", errors.New("can't resolve module using the vendor directory\n\t(Use -mod=mod or -mod=readonly to bypass.)")), + }) + } else { + mods = append(mods, &modinfo.ModulePublic{ + Path: arg, + Error: modinfoError(arg, "", errors.New("not a known dependency")), + }) } - mods = append(mods, &modinfo.ModulePublic{ - Path: arg, - Error: &modinfo.ModuleError{ - Err: fmt.Sprintf("module %q is not a known dependency", arg), - }, - }) } else { fmt.Fprintf(os.Stderr, "warning: pattern %q matched no module dependencies\n", arg) } @@ -121,3 +148,21 @@ func listModules(args []string, listVersions bool) []*modinfo.ModulePublic { return mods } + +// modinfoError wraps an error to create an error message in +// modinfo.ModuleError with minimal redundancy. +func modinfoError(path, vers string, err error) *modinfo.ModuleError { + var nerr *NoMatchingVersionError + var merr *module.ModuleError + if errors.As(err, &nerr) { + // NoMatchingVersionError contains the query, so we don't mention the + // query again in ModuleError. + err = &module.ModuleError{Path: path, Err: err} + } else if !errors.As(err, &merr) { + // If the error does not contain path and version, wrap it in a + // module.ModuleError. + err = &module.ModuleError{Path: path, Version: vers, Err: err} + } + + return &modinfo.ModuleError{Err: err.Error()} +} diff --git a/cmd/go/_internal_/modload/load.go b/cmd/go/_internal_/modload/load.go index bd3f6d7..b7fabe2 100644 --- a/cmd/go/_internal_/modload/load.go +++ b/cmd/go/_internal_/modload/load.go @@ -6,28 +6,25 @@ package modload import ( "bytes" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/imports" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/mvs" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/par" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/search" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/str" "errors" "fmt" "go/build" - "io/ioutil" "os" "path" + pathpkg "path" "path/filepath" "sort" "strings" - "sync" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/imports" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfile" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/module" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/mvs" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/par" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/search" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/semver" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/str" + "golang.org/x/mod/module" ) // buildList is the list of modules to use for building packages. @@ -50,37 +47,26 @@ var buildList []module.Version var loaded *loader // ImportPaths returns the set of packages matching the args (patterns), -// adding modules to the build list as needed to satisfy new imports. +// on the target platform. Modules may be added to the build list +// to satisfy new imports. func ImportPaths(patterns []string) []*search.Match { - InitMod() - - var matches []*search.Match - for _, pattern := range search.CleanPatterns(patterns) { - m := &search.Match{ - Pattern: pattern, - Literal: !strings.Contains(pattern, "...") && !search.IsMetaPackage(pattern), - } - if m.Literal { - m.Pkgs = []string{pattern} - } - matches = append(matches, m) - } + matches := ImportPathsQuiet(patterns, imports.Tags()) + search.WarnUnmatched(matches) + return matches +} - fsDirs := make([][]string, len(matches)) - loaded = newLoader() - updateMatches := func(iterating bool) { - for i, m := range matches { +// ImportPathsQuiet is like ImportPaths but does not warn about patterns with +// no matches. It also lets the caller specify a set of build tags to match +// packages. The build tags should typically be imports.Tags() or +// imports.AnyTags(); a nil map has no special meaning. +func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { + updateMatches := func(matches []*search.Match, iterating bool) { + for _, m := range matches { switch { - case build.IsLocalImport(m.Pattern) || filepath.IsAbs(m.Pattern): + case m.IsLocal(): // Evaluate list of file system directories on first iteration. - if fsDirs[i] == nil { - var dirs []string - if m.Literal { - dirs = []string{m.Pattern} - } else { - dirs = search.MatchPackagesInFS(m.Pattern).Pkgs - } - fsDirs[i] = dirs + if m.Dirs == nil { + matchLocalDirs(m) } // Make a copy of the directory list and translate to import paths. @@ -89,95 +75,87 @@ func ImportPaths(patterns []string) []*search.Match { // from not being in the build list to being in it and back as // the exact version of a particular module increases during // the loader iterations. - m.Pkgs = str.StringList(fsDirs[i]) - for j, pkg := range m.Pkgs { - dir := pkg - if !filepath.IsAbs(dir) { - dir = filepath.Join(cwd, pkg) - } else { - dir = filepath.Clean(dir) - } - - // Note: The checks for @ here are just to avoid misinterpreting - // the module cache directories (formerly GOPATH/src/mod/foo@v1.5.2/bar). - // It's not strictly necessary but helpful to keep the checks. - if modRoot != "" && dir == modRoot { - pkg = Target.Path - } else if modRoot != "" && strings.HasPrefix(dir, modRoot+string(filepath.Separator)) && !strings.Contains(dir[len(modRoot):], "@") { - suffix := filepath.ToSlash(dir[len(modRoot):]) - if strings.HasPrefix(suffix, "/vendor/") { - // TODO getmode vendor check - pkg = strings.TrimPrefix(suffix, "/vendor/") - } else { - pkg = Target.Path + suffix + m.Pkgs = m.Pkgs[:0] + for _, dir := range m.Dirs { + pkg, err := resolveLocalPackage(dir) + if err != nil { + if !m.IsLiteral() && (err == errPkgIsBuiltin || err == errPkgIsGorootSrc) { + continue // Don't include "builtin" or GOROOT/src in wildcard patterns. } - } else if sub := search.InDir(dir, cfg.GOROOTsrc); sub != "" && !strings.Contains(sub, "@") { - pkg = filepath.ToSlash(sub) - } else if path := pathInModuleCache(dir); path != "" { - pkg = path - } else { - pkg = "" + + // If we're outside of a module, ensure that the failure mode + // indicates that. + ModRoot() + if !iterating { - ModRoot() - base.Errorf("go: directory %s outside available modules", base.ShortPath(dir)) - } - } - info, err := os.Stat(dir) - if err != nil || !info.IsDir() { - // If the directory is local but does not exist, don't return it - // while loader is iterating, since this would trigger a fetch. - // After loader is done iterating, we still need to return the - // path, so that "go list -e" produces valid output. - if iterating { - pkg = "" + m.AddError(err) } + continue } - m.Pkgs[j] = pkg + m.Pkgs = append(m.Pkgs, pkg) } - case strings.Contains(m.Pattern, "..."): - m.Pkgs = matchPackages(m.Pattern, loaded.tags, true, buildList) + case m.IsLiteral(): + m.Pkgs = []string{m.Pattern()} + + case strings.Contains(m.Pattern(), "..."): + m.Errs = m.Errs[:0] + matchPackages(m, loaded.tags, includeStd, buildList) - case m.Pattern == "all": + case m.Pattern() == "all": loaded.testAll = true if iterating { // Enumerate the packages in the main module. // We'll load the dependencies as we find them. - m.Pkgs = matchPackages("...", loaded.tags, false, []module.Version{Target}) + m.Errs = m.Errs[:0] + matchPackages(m, loaded.tags, omitStd, []module.Version{Target}) } else { // Starting with the packages in the main module, // enumerate the full list of "all". m.Pkgs = loaded.computePatternAll(m.Pkgs) } - case search.IsMetaPackage(m.Pattern): // std, cmd - if len(m.Pkgs) == 0 { - m.Pkgs = search.MatchPackages(m.Pattern).Pkgs + case m.Pattern() == "std" || m.Pattern() == "cmd": + if m.Pkgs == nil { + m.MatchPackages() // Locate the packages within GOROOT/src. } + + default: + panic(fmt.Sprintf("internal error: modload missing case for pattern %s", m.Pattern())) } } } + InitMod() + + var matches []*search.Match + for _, pattern := range search.CleanPatterns(patterns) { + matches = append(matches, search.NewMatch(pattern)) + } + + loaded = newLoader(tags) loaded.load(func() []string { var roots []string - updateMatches(true) + updateMatches(matches, true) for _, m := range matches { - for _, pkg := range m.Pkgs { - if pkg != "" { - roots = append(roots, pkg) - } - } + roots = append(roots, m.Pkgs...) } return roots }) // One last pass to finalize wildcards. - updateMatches(false) + updateMatches(matches, false) + checkMultiplePaths() + WriteGoMod() + + return matches +} - // A given module path may be used as itself or as a replacement for another - // module, but not both at the same time. Otherwise, the aliasing behavior is - // too subtle (see https://golang.org/issue/26607), and we don't want to - // commit to a specific behavior at this point. +// checkMultiplePaths verifies that a given module path is used as itself +// or as a replacement for another module, but not both at the same time. +// +// (See https://golang.org/issue/26607 and https://golang.org/issue/34650.) +func checkMultiplePaths() { firstPath := make(map[module.Version]string, len(buildList)) for _, mod := range buildList { src := mod @@ -191,17 +169,157 @@ func ImportPaths(patterns []string) []*search.Match { } } base.ExitIfErrors() - WriteGoMod() +} - search.WarnUnmatched(matches) - return matches +// matchLocalDirs is like m.MatchDirs, but tries to avoid scanning directories +// outside of the standard library and active modules. +func matchLocalDirs(m *search.Match) { + if !m.IsLocal() { + panic(fmt.Sprintf("internal error: resolveLocalDirs on non-local pattern %s", m.Pattern())) + } + + if i := strings.Index(m.Pattern(), "..."); i >= 0 { + // The pattern is local, but it is a wildcard. Its packages will + // only resolve to paths if they are inside of the standard + // library, the main module, or some dependency of the main + // module. Verify that before we walk the filesystem: a filesystem + // walk in a directory like /var or /etc can be very expensive! + dir := filepath.Dir(filepath.Clean(m.Pattern()[:i+3])) + absDir := dir + if !filepath.IsAbs(dir) { + absDir = filepath.Join(base.Cwd, dir) + } + if search.InDir(absDir, cfg.GOROOTsrc) == "" && search.InDir(absDir, ModRoot()) == "" && pathInModuleCache(absDir) == "" { + m.Dirs = []string{} + m.AddError(fmt.Errorf("directory prefix %s outside available modules", base.ShortPath(absDir))) + return + } + } + + m.MatchDirs() +} + +// resolveLocalPackage resolves a filesystem path to a package path. +func resolveLocalPackage(dir string) (string, error) { + var absDir string + if filepath.IsAbs(dir) { + absDir = filepath.Clean(dir) + } else { + absDir = filepath.Join(base.Cwd, dir) + } + + bp, err := cfg.BuildContext.ImportDir(absDir, 0) + if err != nil && (bp == nil || len(bp.IgnoredGoFiles) == 0) { + // golang.org/issue/32917: We should resolve a relative path to a + // package path only if the relative path actually contains the code + // for that package. + // + // If the named directory does not exist or contains no Go files, + // the package does not exist. + // Other errors may affect package loading, but not resolution. + if _, err := os.Stat(absDir); err != nil { + if os.IsNotExist(err) { + // Canonicalize OS-specific errors to errDirectoryNotFound so that error + // messages will be easier for users to search for. + return "", &os.PathError{Op: "stat", Path: absDir, Err: errDirectoryNotFound} + } + return "", err + } + if _, noGo := err.(*build.NoGoError); noGo { + // A directory that does not contain any Go source files — even ignored + // ones! — is not a Go package, and we can't resolve it to a package + // path because that path could plausibly be provided by some other + // module. + // + // Any other error indicates that the package “exists” (at least in the + // sense that it cannot exist in any other module), but has some other + // problem (such as a syntax error). + return "", err + } + } + + if modRoot != "" && absDir == modRoot { + if absDir == cfg.GOROOTsrc { + return "", errPkgIsGorootSrc + } + return targetPrefix, nil + } + + // Note: The checks for @ here are just to avoid misinterpreting + // the module cache directories (formerly GOPATH/src/mod/foo@v1.5.2/bar). + // It's not strictly necessary but helpful to keep the checks. + if modRoot != "" && strings.HasPrefix(absDir, modRoot+string(filepath.Separator)) && !strings.Contains(absDir[len(modRoot):], "@") { + suffix := filepath.ToSlash(absDir[len(modRoot):]) + if strings.HasPrefix(suffix, "/vendor/") { + if cfg.BuildMod != "vendor" { + return "", fmt.Errorf("without -mod=vendor, directory %s has no package path", absDir) + } + + readVendorList() + pkg := strings.TrimPrefix(suffix, "/vendor/") + if _, ok := vendorPkgModule[pkg]; !ok { + return "", fmt.Errorf("directory %s is not a package listed in vendor/modules.txt", absDir) + } + return pkg, nil + } + + if targetPrefix == "" { + pkg := strings.TrimPrefix(suffix, "/") + if pkg == "builtin" { + // "builtin" is a pseudo-package with a real source file. + // It's not included in "std", so it shouldn't resolve from "." + // within module "std" either. + return "", errPkgIsBuiltin + } + return pkg, nil + } + + pkg := targetPrefix + suffix + if _, ok, err := dirInModule(pkg, targetPrefix, modRoot, true); err != nil { + return "", err + } else if !ok { + return "", &PackageNotInModuleError{Mod: Target, Pattern: pkg} + } + return pkg, nil + } + + if sub := search.InDir(absDir, cfg.GOROOTsrc); sub != "" && sub != "." && !strings.Contains(sub, "@") { + pkg := filepath.ToSlash(sub) + if pkg == "builtin" { + return "", errPkgIsBuiltin + } + return pkg, nil + } + + pkg := pathInModuleCache(absDir) + if pkg == "" { + return "", fmt.Errorf("directory %s outside available modules", base.ShortPath(absDir)) + } + return pkg, nil } +var ( + errDirectoryNotFound = errors.New("directory not found") + errPkgIsGorootSrc = errors.New("GOROOT/src is not an importable package") + errPkgIsBuiltin = errors.New(`"builtin" is a pseudo-package, not an importable package`) +) + // pathInModuleCache returns the import path of the directory dir, // if dir is in the module cache copy of a module in our build list. func pathInModuleCache(dir string) string { for _, m := range buildList[1:] { - root, err := modfetch.DownloadDir(m) + var root string + var err error + if repl := Replacement(m); repl.Path != "" && repl.Version == "" { + root = repl.Path + if !filepath.IsAbs(root) { + root = filepath.Join(ModRoot(), root) + } + } else if repl.Path != "" { + root, err = modfetch.DownloadDir(repl) + } else { + root, err = modfetch.DownloadDir(m) + } if err != nil { continue } @@ -215,27 +333,18 @@ func pathInModuleCache(dir string) string { return "" } -// warnPattern returns list, the result of matching pattern, -// but if list is empty then first it prints a warning about -// the pattern not matching any packages. -func warnPattern(pattern string, list []string) []string { - if len(list) == 0 { - fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern) - } - return list -} - // ImportFromFiles adds modules to the build list as needed // to satisfy the imports in the named Go source files. func ImportFromFiles(gofiles []string) { InitMod() - imports, testImports, err := imports.ScanFiles(gofiles, imports.Tags()) + tags := imports.Tags() + imports, testImports, err := imports.ScanFiles(gofiles, tags) if err != nil { base.Fatalf("go: %v", err) } - loaded = newLoader() + loaded = newLoader(tags) loaded.load(func() []string { var roots []string roots = append(roots, imports...) @@ -253,20 +362,20 @@ func DirImportPath(dir string) string { } if !filepath.IsAbs(dir) { - dir = filepath.Join(cwd, dir) + dir = filepath.Join(base.Cwd, dir) } else { dir = filepath.Clean(dir) } if dir == modRoot { - return Target.Path + return targetPrefix } if strings.HasPrefix(dir, modRoot+string(filepath.Separator)) { suffix := filepath.ToSlash(dir[len(modRoot):]) if strings.HasPrefix(suffix, "/vendor/") { return strings.TrimPrefix(suffix, "/vendor/") } - return Target.Path + suffix + return targetPrefix + suffix } return "." } @@ -284,7 +393,7 @@ func LoadBuildList() []module.Version { } func ReloadBuildList() []module.Version { - loaded = newLoader() + loaded = newLoader(imports.Tags()) loaded.load(func() []string { return nil }) return buildList } @@ -310,35 +419,43 @@ func LoadVendor() []string { func loadAll(testAll bool) []string { InitMod() - loaded = newLoader() + loaded = newLoader(imports.AnyTags()) loaded.isALL = true - loaded.tags = anyTags loaded.testAll = testAll if !testAll { loaded.testRoots = true } - all := TargetPackages() - loaded.load(func() []string { return all }) + all := TargetPackages("...") + loaded.load(func() []string { return all.Pkgs }) + checkMultiplePaths() WriteGoMod() var paths []string for _, pkg := range loaded.pkgs { - if e, ok := pkg.err.(*ImportMissingError); ok && e.Module.Path == "" { - continue // Package doesn't actually exist. + if pkg.err != nil { + base.Errorf("%s: %v", pkg.stackText(), pkg.err) + continue } paths = append(paths, pkg.path) } + for _, err := range all.Errs { + base.Errorf("%v", err) + } + base.ExitIfErrors() return paths } -// anyTags is a special tags map that satisfies nearly all build tag expressions. -// Only "ignore" and malformed build tag requirements are considered false. -var anyTags = map[string]bool{"*": true} - -// TargetPackages returns the list of packages in the target (top-level) module, -// under all build tag settings. -func TargetPackages() []string { - return matchPackages("...", anyTags, false, []module.Version{Target}) +// TargetPackages returns the list of packages in the target (top-level) module +// matching pattern, which may be relative to the working directory, under all +// build tag settings. +func TargetPackages(pattern string) *search.Match { + // TargetPackages is relative to the main module, so ensure that the main + // module is a thing that can contain packages. + ModRoot() + + m := search.NewMatch(pattern) + matchPackages(m, imports.AnyTags(), omitStd, []module.Version{Target}) + return m } // BuildList returns the module build list, @@ -356,6 +473,37 @@ func SetBuildList(list []module.Version) { buildList = append([]module.Version{}, list...) } +// TidyBuildList trims the build list to the minimal requirements needed to +// retain the same versions of all packages from the preceding Load* or +// ImportPaths* call. +func TidyBuildList() { + used := map[module.Version]bool{Target: true} + for _, pkg := range loaded.pkgs { + used[pkg.mod] = true + } + + keep := []module.Version{Target} + var direct []string + for _, m := range buildList[1:] { + if used[m] { + keep = append(keep, m) + if loaded.direct[m.Path] { + direct = append(direct, m.Path) + } + } else if cfg.BuildV { + if _, ok := index.require[m]; ok { + fmt.Fprintf(os.Stderr, "unused %s\n", m.Path) + } + } + } + + min, err := mvs.Req(Target, direct, &mvsReqs{buildList: keep}) + if err != nil { + base.Fatalf("go: %v", err) + } + buildList = append([]module.Version{Target}, min...) +} + // ImportMap returns the actual package import path // for an import path found in source code. // If the given import path does not appear in the source code @@ -387,6 +535,29 @@ func PackageModule(path string) module.Version { return pkg.mod } +// PackageImports returns the imports for the package named by the import path. +// Test imports will be returned as well if tests were loaded for the package +// (i.e., if "all" was loaded or if LoadTests was set and the path was matched +// by a command line argument). PackageImports will return nil for +// unknown package paths. +func PackageImports(path string) (imports, testImports []string) { + pkg, ok := loaded.pkgCache.Get(path).(*loadPkg) + if !ok { + return nil, nil + } + imports = make([]string, len(pkg.imports)) + for i, p := range pkg.imports { + imports[i] = p.path + } + if pkg.test != nil { + testImports = make([]string, len(pkg.test.imports)) + for i, p := range pkg.test.imports { + testImports[i] = p.path + } + } + return imports, testImports +} + // ModuleUsedDirectly reports whether the main module directly imports // some package in the module with the given path. func ModuleUsedDirectly(path string) bool { @@ -394,13 +565,17 @@ func ModuleUsedDirectly(path string) bool { } // Lookup returns the source directory, import path, and any loading error for -// the package at path. +// the package at path as imported from the package in parentDir. // Lookup requires that one of the Load functions in this package has already // been called. -func Lookup(path string) (dir, realPath string, err error) { +func Lookup(parentPath string, parentIsStd bool, path string) (dir, realPath string, err error) { if path == "" { panic("Lookup called with empty package path") } + + if parentIsStd { + path = loaded.stdVendor(parentPath, path) + } pkg, ok := loaded.pkgCache.Get(path).(*loadPkg) if !ok { // The loader should have found all the relevant paths. @@ -438,6 +613,7 @@ type loader struct { testRoots bool // include tests for roots isALL bool // created with LoadALL testAll bool // include tests for all packages + forceStdVendor bool // if true, load standard-library dependencies from the vendor subtree // reset on each iteration roots []*loadPkg @@ -453,10 +629,17 @@ type loader struct { // LoadTests controls whether the loaders load tests of the root packages. var LoadTests bool -func newLoader() *loader { +func newLoader(tags map[string]bool) *loader { ld := new(loader) - ld.tags = imports.Tags() + ld.tags = tags ld.testRoots = LoadTests + + // Inside the "std" and "cmd" modules, we prefer to use the vendor directory + // unless the command explicitly changes the module graph. + if !targetInGorootSrc || (cfg.CmdName != "get" && !strings.HasPrefix(cfg.CmdName, "mod ")) { + ld.forceStdVendor = true + } + return ld } @@ -511,8 +694,13 @@ func (ld *loader) load(roots func() []string) { for _, m := range buildList { haveMod[m] = true } + modAddedBy := make(map[module.Version]*loadPkg) for _, pkg := range ld.pkgs { if err, ok := pkg.err.(*ImportMissingError); ok && err.Module.Path != "" { + if err.newMissingVersion != "" { + base.Fatalf("go: %s: package provided by %s at latest version %s but not at required version %s", pkg.stackText(), err.Module.Path, err.Module.Version, err.newMissingVersion) + } + fmt.Fprintf(os.Stderr, "go: found %s in %s %s\n", pkg.path, err.Module.Path, err.Module.Version) if added[pkg.path] { base.Fatalf("go: %s: looping trying to add package", pkg.stackText()) } @@ -520,6 +708,7 @@ func (ld *loader) load(roots func() []string) { numAdded++ if !haveMod[err.Module] { haveMod[err.Module] = true + modAddedBy[err.Module] = pkg buildList = append(buildList, err.Module) } continue @@ -535,6 +724,14 @@ func (ld *loader) load(roots func() []string) { reqs = Reqs() buildList, err = mvs.BuildList(Target, reqs) if err != nil { + // If an error was found in a newly added module, report the package + // import stack instead of the module requirement stack. Packages + // are more descriptive. + if err, ok := err.(*mvs.BuildListError); ok { + if pkg := modAddedBy[err.Module()]; pkg != nil { + base.Fatalf("go: %s: %v", pkg.stackText(), err.Err) + } + } base.Fatalf("go: %v", err) } } @@ -609,7 +806,7 @@ func (ld *loader) doPkg(item interface{}) { // Leave for error during load. return } - if build.IsLocalImport(pkg.path) { + if build.IsLocalImport(pkg.path) || filepath.IsAbs(pkg.path) { // Leave for error during load. // (Module mode does not allow local imports.) return @@ -631,7 +828,11 @@ func (ld *loader) doPkg(item interface{}) { } } + inStd := (search.IsStandardImportPath(pkg.path) && search.InDir(pkg.dir, cfg.GOROOTsrc) != "") for _, path := range imports { + if inStd { + path = ld.stdVendor(pkg.path, path) + } pkg.imports = append(pkg.imports, ld.pkg(path, false)) } @@ -642,6 +843,31 @@ func (ld *loader) doPkg(item interface{}) { } } +// stdVendor returns the canonical import path for the package with the given +// path when imported from the standard-library package at parentPath. +func (ld *loader) stdVendor(parentPath, path string) string { + if search.IsStandardImportPath(path) { + return path + } + + if str.HasPathPrefix(parentPath, "cmd") { + if ld.forceStdVendor || Target.Path != "cmd" { + vendorPath := pathpkg.Join("cmd", "vendor", path) + if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil { + return vendorPath + } + } + } else if ld.forceStdVendor || Target.Path != "std" { + vendorPath := pathpkg.Join("vendor", path) + if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil { + return vendorPath + } + } + + // Not vendored: resolve from modules. + return path +} + // computePatternAll returns the list of packages matching pattern "all", // starting with a list of the import paths for the packages in the main module. func (ld *loader) computePatternAll(paths []string) []string { @@ -737,27 +963,33 @@ func (ld *loader) buildStacks() { // stackText builds the import stack text to use when // reporting an error in pkg. It has the general form // -// import root -> -// import other -> -// import other2 -> -// import pkg +// root imports +// other imports +// other2 tested by +// other2.test imports +// pkg // func (pkg *loadPkg) stackText() string { var stack []*loadPkg - for p := pkg.stack; p != nil; p = p.stack { + for p := pkg; p != nil; p = p.stack { stack = append(stack, p) } var buf bytes.Buffer for i := len(stack) - 1; i >= 0; i-- { p := stack[i] + fmt.Fprint(&buf, p.path) if p.testOf != nil { - fmt.Fprintf(&buf, "test ->\n\t") - } else { - fmt.Fprintf(&buf, "import %q ->\n\t", p.path) + fmt.Fprint(&buf, ".test") + } + if i > 0 { + if stack[i-1].testOf == p { + fmt.Fprint(&buf, " tested by\n\t") + } else { + fmt.Fprint(&buf, " imports\n\t") + } } } - fmt.Fprintf(&buf, "import %q", pkg.path) return buf.String() } @@ -805,267 +1037,3 @@ func WhyDepth(path string) int { } return n } - -// Replacement returns the replacement for mod, if any, from go.mod. -// If there is no replacement for mod, Replacement returns -// a module.Version with Path == "". -func Replacement(mod module.Version) module.Version { - if modFile == nil { - // Happens during testing and if invoking 'go get' or 'go list' outside a module. - return module.Version{} - } - - var found *modfile.Replace - for _, r := range modFile.Replace { - if r.Old.Path == mod.Path && (r.Old.Version == "" || r.Old.Version == mod.Version) { - found = r // keep going - } - } - if found == nil { - return module.Version{} - } - return found.New -} - -// mvsReqs implements mvs.Reqs for module semantic versions, -// with any exclusions or replacements applied internally. -type mvsReqs struct { - buildList []module.Version - cache par.Cache - versions sync.Map -} - -// Reqs returns the current module requirement graph. -// Future calls to SetBuildList do not affect the operation -// of the returned Reqs. -func Reqs() mvs.Reqs { - r := &mvsReqs{ - buildList: buildList, - } - return r -} - -func (r *mvsReqs) Required(mod module.Version) ([]module.Version, error) { - type cached struct { - list []module.Version - err error - } - - c := r.cache.Do(mod, func() interface{} { - list, err := r.required(mod) - if err != nil { - return cached{nil, err} - } - for i, mv := range list { - for excluded[mv] { - mv1, err := r.next(mv) - if err != nil { - return cached{nil, err} - } - if mv1.Version == "none" { - return cached{nil, fmt.Errorf("%s(%s) depends on excluded %s(%s) with no newer version available", mod.Path, mod.Version, mv.Path, mv.Version)} - } - mv = mv1 - } - list[i] = mv - } - - return cached{list, nil} - }).(cached) - - return c.list, c.err -} - -var vendorOnce sync.Once - -var ( - vendorList []module.Version - vendorMap map[string]module.Version -) - -// readVendorList reads the list of vendored modules from vendor/modules.txt. -func readVendorList() { - vendorOnce.Do(func() { - vendorList = nil - vendorMap = make(map[string]module.Version) - data, _ := ioutil.ReadFile(filepath.Join(ModRoot(), "vendor/modules.txt")) - var m module.Version - for _, line := range strings.Split(string(data), "\n") { - if strings.HasPrefix(line, "# ") { - f := strings.Fields(line) - m = module.Version{} - if len(f) == 3 && semver.IsValid(f[2]) { - m = module.Version{Path: f[1], Version: f[2]} - vendorList = append(vendorList, m) - } - } else if m.Path != "" { - f := strings.Fields(line) - if len(f) == 1 { - vendorMap[f[0]] = m - } - } - } - }) -} - -func (r *mvsReqs) modFileToList(f *modfile.File) []module.Version { - var list []module.Version - for _, r := range f.Require { - list = append(list, r.Mod) - } - return list -} - -func (r *mvsReqs) required(mod module.Version) ([]module.Version, error) { - if mod == Target { - if modFile != nil && modFile.Go != nil { - r.versions.LoadOrStore(mod, modFile.Go.Version) - } - var list []module.Version - return append(list, r.buildList[1:]...), nil - } - - if cfg.BuildMod == "vendor" { - // For every module other than the target, - // return the full list of modules from modules.txt. - readVendorList() - return vendorList, nil - } - - origPath := mod.Path - if repl := Replacement(mod); repl.Path != "" { - if repl.Version == "" { - // TODO: need to slip the new version into the tags list etc. - dir := repl.Path - if !filepath.IsAbs(dir) { - dir = filepath.Join(ModRoot(), dir) - } - gomod := filepath.Join(dir, "go.mod") - data, err := ioutil.ReadFile(gomod) - if err != nil { - base.Errorf("go: parsing %s: %v", base.ShortPath(gomod), err) - return nil, ErrRequire - } - f, err := modfile.ParseLax(gomod, data, nil) - if err != nil { - base.Errorf("go: parsing %s: %v", base.ShortPath(gomod), err) - return nil, ErrRequire - } - if f.Go != nil { - r.versions.LoadOrStore(mod, f.Go.Version) - } - return r.modFileToList(f), nil - } - mod = repl - } - - if mod.Version == "none" { - return nil, nil - } - - if !semver.IsValid(mod.Version) { - // Disallow the broader queries supported by fetch.Lookup. - base.Fatalf("go: internal error: %s@%s: unexpected invalid semantic version", mod.Path, mod.Version) - } - - data, err := modfetch.GoMod(mod.Path, mod.Version) - if err != nil { - base.Errorf("go: %s@%s: %v\n", mod.Path, mod.Version, err) - return nil, ErrRequire - } - f, err := modfile.ParseLax("go.mod", data, nil) - if err != nil { - base.Errorf("go: %s@%s: parsing go.mod: %v", mod.Path, mod.Version, err) - return nil, ErrRequire - } - - if f.Module == nil { - base.Errorf("go: %s@%s: parsing go.mod: missing module line", mod.Path, mod.Version) - return nil, ErrRequire - } - if mpath := f.Module.Mod.Path; mpath != origPath && mpath != mod.Path { - base.Errorf("go: %s@%s: parsing go.mod: unexpected module path %q", mod.Path, mod.Version, mpath) - return nil, ErrRequire - } - if f.Go != nil { - r.versions.LoadOrStore(mod, f.Go.Version) - } - - return r.modFileToList(f), nil -} - -// ErrRequire is the sentinel error returned when Require encounters problems. -// It prints the problems directly to standard error, so that multiple errors -// can be displayed easily. -var ErrRequire = errors.New("error loading module requirements") - -func (*mvsReqs) Max(v1, v2 string) string { - if v1 != "" && semver.Compare(v1, v2) == -1 { - return v2 - } - return v1 -} - -// Upgrade is a no-op, here to implement mvs.Reqs. -// The upgrade logic for go get -u is in ../modget/get.go. -func (*mvsReqs) Upgrade(m module.Version) (module.Version, error) { - return m, nil -} - -func versions(path string) ([]string, error) { - // Note: modfetch.Lookup and repo.Versions are cached, - // so there's no need for us to add extra caching here. - repo, err := modfetch.Lookup(path) - if err != nil { - return nil, err - } - return repo.Versions("") -} - -// Previous returns the tagged version of m.Path immediately prior to -// m.Version, or version "none" if no prior version is tagged. -func (*mvsReqs) Previous(m module.Version) (module.Version, error) { - list, err := versions(m.Path) - if err != nil { - return module.Version{}, err - } - i := sort.Search(len(list), func(i int) bool { return semver.Compare(list[i], m.Version) >= 0 }) - if i > 0 { - return module.Version{Path: m.Path, Version: list[i-1]}, nil - } - return module.Version{Path: m.Path, Version: "none"}, nil -} - -// next returns the next version of m.Path after m.Version. -// It is only used by the exclusion processing in the Required method, -// not called directly by MVS. -func (*mvsReqs) next(m module.Version) (module.Version, error) { - list, err := versions(m.Path) - if err != nil { - return module.Version{}, err - } - i := sort.Search(len(list), func(i int) bool { return semver.Compare(list[i], m.Version) > 0 }) - if i < len(list) { - return module.Version{Path: m.Path, Version: list[i]}, nil - } - return module.Version{Path: m.Path, Version: "none"}, nil -} - -func fetch(mod module.Version) (dir string, isLocal bool, err error) { - if mod == Target { - return ModRoot(), true, nil - } - if r := Replacement(mod); r.Path != "" { - if r.Version == "" { - dir = r.Path - if !filepath.IsAbs(dir) { - dir = filepath.Join(ModRoot(), dir) - } - return dir, true, nil - } - mod = r - } - - dir, err = modfetch.Download(mod) - return dir, false, err -} diff --git a/cmd/go/_internal_/modload/modfile.go b/cmd/go/_internal_/modload/modfile.go new file mode 100644 index 0000000..257e2c3 --- /dev/null +++ b/cmd/go/_internal_/modload/modfile.go @@ -0,0 +1,164 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" + + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" +) + +var modFile *modfile.File + +// A modFileIndex is an index of data corresponding to a modFile +// at a specific point in time. +type modFileIndex struct { + data []byte + dataNeedsFix bool // true if fixVersion applied a change while parsing data + module module.Version + goVersion string + require map[module.Version]requireMeta + replace map[module.Version]module.Version + exclude map[module.Version]bool +} + +// index is the index of the go.mod file as of when it was last read or written. +var index *modFileIndex + +type requireMeta struct { + indirect bool +} + +// Allowed reports whether module m is allowed (not excluded) by the main module's go.mod. +func Allowed(m module.Version) bool { + return index == nil || !index.exclude[m] +} + +// Replacement returns the replacement for mod, if any, from go.mod. +// If there is no replacement for mod, Replacement returns +// a module.Version with Path == "". +func Replacement(mod module.Version) module.Version { + if index != nil { + if r, ok := index.replace[mod]; ok { + return r + } + if r, ok := index.replace[module.Version{Path: mod.Path}]; ok { + return r + } + } + return module.Version{} +} + +// indexModFile rebuilds the index of modFile. +// If modFile has been changed since it was first read, +// modFile.Cleanup must be called before indexModFile. +func indexModFile(data []byte, modFile *modfile.File, needsFix bool) *modFileIndex { + i := new(modFileIndex) + i.data = data + i.dataNeedsFix = needsFix + + i.module = module.Version{} + if modFile.Module != nil { + i.module = modFile.Module.Mod + } + + i.goVersion = "" + if modFile.Go != nil { + i.goVersion = modFile.Go.Version + } + + i.require = make(map[module.Version]requireMeta, len(modFile.Require)) + for _, r := range modFile.Require { + i.require[r.Mod] = requireMeta{indirect: r.Indirect} + } + + i.replace = make(map[module.Version]module.Version, len(modFile.Replace)) + for _, r := range modFile.Replace { + if prev, dup := i.replace[r.Old]; dup && prev != r.New { + base.Fatalf("go: conflicting replacements for %v:\n\t%v\n\t%v", r.Old, prev, r.New) + } + i.replace[r.Old] = r.New + } + + i.exclude = make(map[module.Version]bool, len(modFile.Exclude)) + for _, x := range modFile.Exclude { + i.exclude[x.Mod] = true + } + + return i +} + +// modFileIsDirty reports whether the go.mod file differs meaningfully +// from what was indexed. +// If modFile has been changed (even cosmetically) since it was first read, +// modFile.Cleanup must be called before modFileIsDirty. +func (i *modFileIndex) modFileIsDirty(modFile *modfile.File) bool { + if i == nil { + return modFile != nil + } + + if i.dataNeedsFix { + return true + } + + if modFile.Module == nil { + if i.module != (module.Version{}) { + return true + } + } else if modFile.Module.Mod != i.module { + return true + } + + if modFile.Go == nil { + if i.goVersion != "" { + return true + } + } else if modFile.Go.Version != i.goVersion { + if i.goVersion == "" && cfg.BuildMod == "readonly" { + // go.mod files did not always require a 'go' version, so do not error out + // if one is missing — we may be inside an older module in the module + // cache, and should bias toward providing useful behavior. + } else { + return true + } + } + + if len(modFile.Require) != len(i.require) || + len(modFile.Replace) != len(i.replace) || + len(modFile.Exclude) != len(i.exclude) { + return true + } + + for _, r := range modFile.Require { + if meta, ok := i.require[r.Mod]; !ok { + return true + } else if r.Indirect != meta.indirect { + if cfg.BuildMod == "readonly" { + // The module's requirements are consistent; only the "// indirect" + // comments that are wrong. But those are only guaranteed to be accurate + // after a "go mod tidy" — it's a good idea to run those before + // committing a change, but it's certainly not mandatory. + } else { + return true + } + } + } + + for _, r := range modFile.Replace { + if r.New != i.replace[r.Old] { + return true + } + } + + for _, x := range modFile.Exclude { + if !i.exclude[x.Mod] { + return true + } + } + + return false +} diff --git a/cmd/go/_internal_/modload/mvs.go b/cmd/go/_internal_/modload/mvs.go new file mode 100644 index 0000000..8899860 --- /dev/null +++ b/cmd/go/_internal_/modload/mvs.go @@ -0,0 +1,259 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "sync" + + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/lockedfile" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/mvs" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/par" + + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" + "golang.org/x/mod/semver" +) + +// mvsReqs implements mvs.Reqs for module semantic versions, +// with any exclusions or replacements applied internally. +type mvsReqs struct { + buildList []module.Version + cache par.Cache + versions sync.Map +} + +// Reqs returns the current module requirement graph. +// Future calls to SetBuildList do not affect the operation +// of the returned Reqs. +func Reqs() mvs.Reqs { + r := &mvsReqs{ + buildList: buildList, + } + return r +} + +func (r *mvsReqs) Required(mod module.Version) ([]module.Version, error) { + type cached struct { + list []module.Version + err error + } + + c := r.cache.Do(mod, func() interface{} { + list, err := r.required(mod) + if err != nil { + return cached{nil, err} + } + for i, mv := range list { + if index != nil { + for index.exclude[mv] { + mv1, err := r.next(mv) + if err != nil { + return cached{nil, err} + } + if mv1.Version == "none" { + return cached{nil, fmt.Errorf("%s(%s) depends on excluded %s(%s) with no newer version available", mod.Path, mod.Version, mv.Path, mv.Version)} + } + mv = mv1 + } + } + list[i] = mv + } + + return cached{list, nil} + }).(cached) + + return c.list, c.err +} + +func (r *mvsReqs) modFileToList(f *modfile.File) []module.Version { + list := make([]module.Version, 0, len(f.Require)) + for _, r := range f.Require { + list = append(list, r.Mod) + } + return list +} + +// required returns a unique copy of the requirements of mod. +func (r *mvsReqs) required(mod module.Version) ([]module.Version, error) { + if mod == Target { + if modFile != nil && modFile.Go != nil { + r.versions.LoadOrStore(mod, modFile.Go.Version) + } + return append([]module.Version(nil), r.buildList[1:]...), nil + } + + if cfg.BuildMod == "vendor" { + // For every module other than the target, + // return the full list of modules from modules.txt. + readVendorList() + return append([]module.Version(nil), vendorList...), nil + } + + origPath := mod.Path + if repl := Replacement(mod); repl.Path != "" { + if repl.Version == "" { + // TODO: need to slip the new version into the tags list etc. + dir := repl.Path + if !filepath.IsAbs(dir) { + dir = filepath.Join(ModRoot(), dir) + } + gomod := filepath.Join(dir, "go.mod") + data, err := lockedfile.Read(gomod) + if err != nil { + return nil, fmt.Errorf("parsing %s: %v", base.ShortPath(gomod), err) + } + f, err := modfile.ParseLax(gomod, data, nil) + if err != nil { + return nil, fmt.Errorf("parsing %s: %v", base.ShortPath(gomod), err) + } + if f.Go != nil { + r.versions.LoadOrStore(mod, f.Go.Version) + } + return r.modFileToList(f), nil + } + mod = repl + } + + if mod.Version == "none" { + return nil, nil + } + + if !semver.IsValid(mod.Version) { + // Disallow the broader queries supported by fetch.Lookup. + base.Fatalf("go: internal error: %s@%s: unexpected invalid semantic version", mod.Path, mod.Version) + } + + data, err := modfetch.GoMod(mod.Path, mod.Version) + if err != nil { + return nil, err + } + f, err := modfile.ParseLax("go.mod", data, nil) + if err != nil { + return nil, module.VersionError(mod, fmt.Errorf("parsing go.mod: %v", err)) + } + + if f.Module == nil { + return nil, module.VersionError(mod, errors.New("parsing go.mod: missing module line")) + } + if mpath := f.Module.Mod.Path; mpath != origPath && mpath != mod.Path { + return nil, module.VersionError(mod, fmt.Errorf(`parsing go.mod: + module declares its path as: %s + but was required as: %s`, mpath, origPath)) + } + if f.Go != nil { + r.versions.LoadOrStore(mod, f.Go.Version) + } + + return r.modFileToList(f), nil +} + +// Max returns the maximum of v1 and v2 according to semver.Compare. +// +// As a special case, the version "" is considered higher than all other +// versions. The main module (also known as the target) has no version and must +// be chosen over other versions of the same module in the module dependency +// graph. +func (*mvsReqs) Max(v1, v2 string) string { + if v1 != "" && semver.Compare(v1, v2) == -1 { + return v2 + } + return v1 +} + +// Upgrade is a no-op, here to implement mvs.Reqs. +// The upgrade logic for go get -u is in ../modget/get.go. +func (*mvsReqs) Upgrade(m module.Version) (module.Version, error) { + return m, nil +} + +func versions(path string) ([]string, error) { + // Note: modfetch.Lookup and repo.Versions are cached, + // so there's no need for us to add extra caching here. + var versions []string + err := modfetch.TryProxies(func(proxy string) error { + repo, err := modfetch.Lookup(proxy, path) + if err == nil { + versions, err = repo.Versions("") + } + return err + }) + return versions, err +} + +// Previous returns the tagged version of m.Path immediately prior to +// m.Version, or version "none" if no prior version is tagged. +func (*mvsReqs) Previous(m module.Version) (module.Version, error) { + list, err := versions(m.Path) + if err != nil { + return module.Version{}, err + } + i := sort.Search(len(list), func(i int) bool { return semver.Compare(list[i], m.Version) >= 0 }) + if i > 0 { + return module.Version{Path: m.Path, Version: list[i-1]}, nil + } + return module.Version{Path: m.Path, Version: "none"}, nil +} + +// next returns the next version of m.Path after m.Version. +// It is only used by the exclusion processing in the Required method, +// not called directly by MVS. +func (*mvsReqs) next(m module.Version) (module.Version, error) { + list, err := versions(m.Path) + if err != nil { + return module.Version{}, err + } + i := sort.Search(len(list), func(i int) bool { return semver.Compare(list[i], m.Version) > 0 }) + if i < len(list) { + return module.Version{Path: m.Path, Version: list[i]}, nil + } + return module.Version{Path: m.Path, Version: "none"}, nil +} + +// fetch downloads the given module (or its replacement) +// and returns its location. +// +// The isLocal return value reports whether the replacement, +// if any, is local to the filesystem. +func fetch(mod module.Version) (dir string, isLocal bool, err error) { + if mod == Target { + return ModRoot(), true, nil + } + if r := Replacement(mod); r.Path != "" { + if r.Version == "" { + dir = r.Path + if !filepath.IsAbs(dir) { + dir = filepath.Join(ModRoot(), dir) + } + // Ensure that the replacement directory actually exists: + // dirInModule does not report errors for missing modules, + // so if we don't report the error now, later failures will be + // very mysterious. + if _, err := os.Stat(dir); err != nil { + if os.IsNotExist(err) { + // Semantically the module version itself “exists” — we just don't + // have its source code. Remove the equivalence to os.ErrNotExist, + // and make the message more concise while we're at it. + err = fmt.Errorf("replacement directory %s does not exist", r.Path) + } else { + err = fmt.Errorf("replacement directory %s: %w", r.Path, err) + } + return dir, true, module.VersionError(mod, err) + } + return dir, true, nil + } + mod = r + } + + dir, err = modfetch.Download(mod) + return dir, false, err +} diff --git a/cmd/go/_internal_/modload/query.go b/cmd/go/_internal_/modload/query.go index d869fd1..7c9cc8b 100644 --- a/cmd/go/_internal_/modload/query.go +++ b/cmd/go/_internal_/modload/query.go @@ -5,35 +5,83 @@ package modload import ( - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch/codehost" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/module" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/semver" + "errors" "fmt" + "os" pathpkg "path" + "path/filepath" "strings" + "sync" + + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/imports" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/search" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/str" + + "golang.org/x/mod/module" + "golang.org/x/mod/semver" ) // Query looks up a revision of a given module given a version query string. // The module must be a complete module path. // The version must take one of the following forms: // -// - the literal string "latest", denoting the latest available, allowed tagged version, -// with non-prereleases preferred over prereleases. -// If there are no tagged versions in the repo, latest returns the most recent commit. -// - v1, denoting the latest available tagged version v1.x.x. -// - v1.2, denoting the latest available tagged version v1.2.x. -// - v1.2.3, a semantic version string denoting that tagged version. -// - v1.2.3, >=v1.2.3, -// denoting the version closest to the target and satisfying the given operator, -// with non-prereleases preferred over prereleases. -// - a repository commit identifier, denoting that commit. +// - the literal string "latest", denoting the latest available, allowed +// tagged version, with non-prereleases preferred over prereleases. +// If there are no tagged versions in the repo, latest returns the most +// recent commit. +// - the literal string "upgrade", equivalent to "latest" except that if +// current is a newer version, current will be returned (see below). +// - the literal string "patch", denoting the latest available tagged version +// with the same major and minor number as current (see below). +// - v1, denoting the latest available tagged version v1.x.x. +// - v1.2, denoting the latest available tagged version v1.2.x. +// - v1.2.3, a semantic version string denoting that tagged version. +// - v1.2.3, >=v1.2.3, +// denoting the version closest to the target and satisfying the given operator, +// with non-prereleases preferred over prereleases. +// - a repository commit identifier or tag, denoting that commit. +// +// current denotes the current version of the module; it may be "" if the +// current version is unknown or should not be considered. If query is +// "upgrade" or "patch", current will be returned if it is a newer +// semantic version or a chronologically later pseudo-version than the +// version that would otherwise be chosen. This prevents accidental downgrades +// from newer pre-release or development versions. // -// If the allowed function is non-nil, Query excludes any versions for which allowed returns false. +// If the allowed function is non-nil, Query excludes any versions for which +// allowed returns false. // // If path is the path of the main module and the query is "latest", // Query returns Target.Version as the version. -func Query(path, query string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { +func Query(path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { + var info *modfetch.RevInfo + err := modfetch.TryProxies(func(proxy string) (err error) { + info, err = queryProxy(proxy, path, query, current, allowed) + return err + }) + return info, err +} + +var errQueryDisabled error = queryDisabledError{} + +type queryDisabledError struct{} + +func (queryDisabledError) Error() string { + if cfg.BuildModReason == "" { + return fmt.Sprintf("cannot query module due to -mod=%s", cfg.BuildMod) + } + return fmt.Sprintf("cannot query module due to -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) +} + +func queryProxy(proxy, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { + if current != "" && !semver.IsValid(current) { + return nil, fmt.Errorf("invalid previous version %q", current) + } + if cfg.BuildMod == "vendor" { + return nil, errQueryDisabled + } if allowed == nil { allowed = func(module.Version) bool { return true } } @@ -43,12 +91,39 @@ func Query(path, query string, allowed func(module.Version) bool) (*modfetch.Rev badVersion := func(v string) (*modfetch.RevInfo, error) { return nil, fmt.Errorf("invalid semantic version %q in range %q", v, query) } - var ok func(module.Version) bool - var prefix string - var preferOlder bool + matchesMajor := func(v string) bool { + _, pathMajor, ok := module.SplitPathVersion(path) + if !ok { + return false + } + return module.CheckPathMajor(v, pathMajor) == nil + } + var ( + ok func(module.Version) bool + prefix string + preferOlder bool + mayUseLatest bool + preferIncompatible bool = strings.HasSuffix(current, "+incompatible") + ) switch { case query == "latest": ok = allowed + mayUseLatest = true + + case query == "upgrade": + ok = allowed + mayUseLatest = true + + case query == "patch": + if current == "" { + ok = allowed + mayUseLatest = true + } else { + prefix = semver.MajorMinor(current) + ok = func(m module.Version) bool { + return matchSemverPrefix(prefix, m.Version) && allowed(m) + } + } case strings.HasPrefix(query, "<="): v := query[len("<="):] @@ -62,6 +137,9 @@ func Query(path, query string, allowed func(module.Version) bool) (*modfetch.Rev ok = func(m module.Version) bool { return semver.Compare(m.Version, v) <= 0 && allowed(m) } + if !matchesMajor(v) { + preferIncompatible = true + } case strings.HasPrefix(query, "<"): v := query[len("<"):] @@ -71,6 +149,9 @@ func Query(path, query string, allowed func(module.Version) bool) (*modfetch.Rev ok = func(m module.Version) bool { return semver.Compare(m.Version, v) < 0 && allowed(m) } + if !matchesMajor(v) { + preferIncompatible = true + } case strings.HasPrefix(query, ">="): v := query[len(">="):] @@ -81,6 +162,9 @@ func Query(path, query string, allowed func(module.Version) bool) (*modfetch.Rev return semver.Compare(m.Version, v) >= 0 && allowed(m) } preferOlder = true + if !matchesMajor(v) { + preferIncompatible = true + } case strings.HasPrefix(query, ">"): v := query[len(">"):] @@ -95,25 +179,40 @@ func Query(path, query string, allowed func(module.Version) bool) (*modfetch.Rev return semver.Compare(m.Version, v) > 0 && allowed(m) } preferOlder = true + if !matchesMajor(v) { + preferIncompatible = true + } case semver.IsValid(query) && isSemverPrefix(query): ok = func(m module.Version) bool { return matchSemverPrefix(query, m.Version) && allowed(m) } prefix = query + "." - - case semver.IsValid(query): - vers := module.CanonicalVersion(query) - if !allowed(module.Version{Path: path, Version: vers}) { - return nil, fmt.Errorf("%s@%s excluded", path, vers) + if !matchesMajor(query) { + preferIncompatible = true } - return modfetch.Stat(path, vers) default: // Direct lookup of semantic version or commit identifier. - info, err := modfetch.Stat(path, query) + // + // If the identifier is not a canonical semver tag — including if it's a + // semver tag with a +metadata suffix — then modfetch.Stat will populate + // info.Version with a suitable pseudo-version. + info, err := modfetch.Stat(proxy, path, query) if err != nil { - return nil, err + queryErr := err + // The full query doesn't correspond to a tag. If it is a semantic version + // with a +metadata suffix, see if there is a tag without that suffix: + // semantic versioning defines them to be equivalent. + if vers := module.CanonicalVersion(query); vers != "" && vers != query { + info, err = modfetch.Stat(proxy, path, vers) + if !errors.Is(err, os.ErrNotExist) { + return info, err + } + } + if err != nil { + return nil, queryErr + } } if !allowed(module.Version{Path: path, Version: info.Version}) { return nil, fmt.Errorf("%s@%s excluded", path, info.Version) @@ -131,8 +230,12 @@ func Query(path, query string, allowed func(module.Version) bool) (*modfetch.Rev return &modfetch.RevInfo{Version: Target.Version}, nil } + if str.HasPathPrefix(path, "std") || str.HasPathPrefix(path, "cmd") { + return nil, fmt.Errorf("explicit requirement on standard-library module %s not allowed", path) + } + // Load versions and execute query. - repo, err := modfetch.Lookup(path) + repo, err := modfetch.Lookup(proxy, path) if err != nil { return nil, err } @@ -140,45 +243,62 @@ func Query(path, query string, allowed func(module.Version) bool) (*modfetch.Rev if err != nil { return nil, err } + releases, prereleases, err := filterVersions(path, versions, ok, preferIncompatible) + if err != nil { + return nil, err + } - if preferOlder { - for _, v := range versions { - if semver.Prerelease(v) == "" && ok(module.Version{Path: path, Version: v}) { - return repo.Stat(v) - } + lookup := func(v string) (*modfetch.RevInfo, error) { + rev, err := repo.Stat(v) + if err != nil { + return nil, err } - for _, v := range versions { - if semver.Prerelease(v) != "" && ok(module.Version{Path: path, Version: v}) { - return repo.Stat(v) + + // For "upgrade" and "patch", make sure we don't accidentally downgrade + // from a newer prerelease or from a chronologically newer pseudoversion. + if current != "" && (query == "upgrade" || query == "patch") { + currentTime, err := modfetch.PseudoVersionTime(current) + if semver.Compare(rev.Version, current) < 0 || (err == nil && rev.Time.Before(currentTime)) { + return repo.Stat(current) } } + + return rev, nil + } + + if preferOlder { + if len(releases) > 0 { + return lookup(releases[0]) + } + if len(prereleases) > 0 { + return lookup(prereleases[0]) + } } else { - for i := len(versions) - 1; i >= 0; i-- { - v := versions[i] - if semver.Prerelease(v) == "" && ok(module.Version{Path: path, Version: v}) { - return repo.Stat(v) - } + if len(releases) > 0 { + return lookup(releases[len(releases)-1]) } - for i := len(versions) - 1; i >= 0; i-- { - v := versions[i] - if semver.Prerelease(v) != "" && ok(module.Version{Path: path, Version: v}) { - return repo.Stat(v) - } + if len(prereleases) > 0 { + return lookup(prereleases[len(prereleases)-1]) } } - if query == "latest" { + if mayUseLatest { // Special case for "latest": if no tags match, use latest commit in repo, // provided it is not excluded. - if info, err := repo.Latest(); err == nil && allowed(module.Version{Path: path, Version: info.Version}) { - return info, nil + latest, err := repo.Latest() + if err == nil { + if allowed(module.Version{Path: path, Version: latest.Version}) { + return lookup(latest.Version) + } + } else if !errors.Is(err, os.ErrNotExist) { + return nil, err } } - return nil, fmt.Errorf("no matching versions for query %q", query) + return nil, &NoMatchingVersionError{query: query, current: current} } -// isSemverPrefix reports whether v is a semantic version prefix: v1 or v1.2 (not v1.2.3). +// isSemverPrefix reports whether v is a semantic version prefix: v1 or v1.2 (not v1.2.3). // The caller is assumed to have checked that semver.IsValid(v) is true. func isSemverPrefix(v string) bool { dots := 0 @@ -199,53 +319,399 @@ func isSemverPrefix(v string) bool { // matchSemverPrefix reports whether the shortened semantic version p // matches the full-width (non-shortened) semantic version v. func matchSemverPrefix(p, v string) bool { - return len(v) > len(p) && v[len(p)] == '.' && v[:len(p)] == p + return len(v) > len(p) && v[len(p)] == '.' && v[:len(p)] == p && semver.Prerelease(v) == "" +} + +// filterVersions classifies versions into releases and pre-releases, filtering +// out: +// 1. versions that do not satisfy the 'ok' predicate, and +// 2. "+incompatible" versions, if a compatible one satisfies the predicate +// and the incompatible version is not preferred. +func filterVersions(path string, versions []string, ok func(module.Version) bool, preferIncompatible bool) (releases, prereleases []string, err error) { + var lastCompatible string + for _, v := range versions { + if !ok(module.Version{Path: path, Version: v}) { + continue + } + + if !preferIncompatible { + if !strings.HasSuffix(v, "+incompatible") { + lastCompatible = v + } else if lastCompatible != "" { + // If the latest compatible version is allowed and has a go.mod file, + // ignore any version with a higher (+incompatible) major version. (See + // https://golang.org/issue/34165.) Note that we even prefer a + // compatible pre-release over an incompatible release. + + ok, err := versionHasGoMod(module.Version{Path: path, Version: lastCompatible}) + if err != nil { + return nil, nil, err + } + if ok { + break + } + + // No acceptable compatible release has a go.mod file, so the versioning + // for the module might not be module-aware, and we should respect + // legacy major-version tags. + preferIncompatible = true + } + } + + if semver.Prerelease(v) != "" { + prereleases = append(prereleases, v) + } else { + releases = append(releases, v) + } + } + + return releases, prereleases, nil +} + +type QueryResult struct { + Mod module.Version + Rev *modfetch.RevInfo + Packages []string } -// QueryPackage looks up a revision of a module containing path. +// QueryPackage looks up the module(s) containing path at a revision matching +// query. The results are sorted by module path length in descending order. // -// If multiple modules with revisions matching the query provide the requested -// package, QueryPackage picks the one with the longest module path. +// If the package is in the main module, QueryPackage considers only the main +// module and only the version "latest", without checking for other possible +// modules. +func QueryPackage(path, query string, allowed func(module.Version) bool) ([]QueryResult, error) { + m := search.NewMatch(path) + if m.IsLocal() || !m.IsLiteral() { + return nil, fmt.Errorf("pattern %s is not an importable package", path) + } + return QueryPattern(path, query, allowed) +} + +// QueryPattern looks up the module(s) containing at least one package matching +// the given pattern at the given version. The results are sorted by module path +// length in descending order. +// +// QueryPattern queries modules with package paths up to the first "..." +// in the pattern. For the pattern "example.com/a/b.../c", QueryPattern would +// consider prefixes of "example.com/a". If multiple modules have versions +// that match the query and packages that match the pattern, QueryPattern +// picks the one with the longest module path. // -// If the path is in the main module and the query is "latest", -// QueryPackage returns Target as the version. -func QueryPackage(path, query string, allowed func(module.Version) bool) (module.Version, *modfetch.RevInfo, error) { +// If any matching package is in the main module, QueryPattern considers only +// the main module and only the version "latest", without checking for other +// possible modules. +func QueryPattern(pattern, query string, allowed func(module.Version) bool) ([]QueryResult, error) { + base := pattern + + firstError := func(m *search.Match) error { + if len(m.Errs) == 0 { + return nil + } + return m.Errs[0] + } + + var match func(mod module.Version, root string, isLocal bool) *search.Match + + if i := strings.Index(pattern, "..."); i >= 0 { + base = pathpkg.Dir(pattern[:i+3]) + match = func(mod module.Version, root string, isLocal bool) *search.Match { + m := search.NewMatch(pattern) + matchPackages(m, imports.AnyTags(), omitStd, []module.Version{mod}) + return m + } + } else { + match = func(mod module.Version, root string, isLocal bool) *search.Match { + m := search.NewMatch(pattern) + prefix := mod.Path + if mod == Target { + prefix = targetPrefix + } + if _, ok, err := dirInModule(pattern, prefix, root, isLocal); err != nil { + m.AddError(err) + } else if ok { + m.Pkgs = []string{pattern} + } + return m + } + } + if HasModRoot() { - if _, ok := dirInModule(path, Target.Path, modRoot, true); ok { + m := match(Target, modRoot, true) + if len(m.Pkgs) > 0 { if query != "latest" { - return module.Version{}, nil, fmt.Errorf("can't query specific version (%q) for package %s in the main module (%s)", query, path, Target.Path) + return nil, fmt.Errorf("can't query specific version for package %s in the main module (%s)", pattern, Target.Path) } if !allowed(Target) { - return module.Version{}, nil, fmt.Errorf("internal error: package %s is in the main module (%s), but version is not allowed", path, Target.Path) + return nil, fmt.Errorf("internal error: package %s is in the main module (%s), but version is not allowed", pattern, Target.Path) } - return Target, &modfetch.RevInfo{Version: Target.Version}, nil + return []QueryResult{{ + Mod: Target, + Rev: &modfetch.RevInfo{Version: Target.Version}, + Packages: m.Pkgs, + }}, nil + } + if err := firstError(m); err != nil { + return nil, err } } - finalErr := errMissing - for p := path; p != "." && p != "/"; p = pathpkg.Dir(p) { - info, err := Query(p, query, allowed) - if err != nil { - if _, ok := err.(*codehost.VCSError); ok { - // A VCSError means we know where to find the code, - // we just can't. Abort search. - return module.Version{}, nil, err + var ( + results []QueryResult + candidateModules = modulePrefixesExcludingTarget(base) + ) + if len(candidateModules) == 0 { + return nil, &PackageNotInModuleError{ + Mod: Target, + Query: query, + Pattern: pattern, + } + } + + err := modfetch.TryProxies(func(proxy string) error { + queryModule := func(path string) (r QueryResult, err error) { + current := findCurrentVersion(path) + r.Mod.Path = path + r.Rev, err = queryProxy(proxy, path, query, current, allowed) + if err != nil { + return r, err } - if finalErr == errMissing { - finalErr = err + r.Mod.Version = r.Rev.Version + root, isLocal, err := fetch(r.Mod) + if err != nil { + return r, err } - continue + m := match(r.Mod, root, isLocal) + r.Packages = m.Pkgs + if len(r.Packages) == 0 { + if err := firstError(m); err != nil { + return r, err + } + return r, &PackageNotInModuleError{ + Mod: r.Mod, + Replacement: Replacement(r.Mod), + Query: query, + Pattern: pattern, + } + } + return r, nil } - m := module.Version{Path: p, Version: info.Version} - root, isLocal, err := fetch(m) - if err != nil { - return module.Version{}, nil, err + + var err error + results, err = queryPrefixModules(candidateModules, queryModule) + return err + }) + + return results, err +} + +// modulePrefixesExcludingTarget returns all prefixes of path that may plausibly +// exist as a module, excluding targetPrefix but otherwise including path +// itself, sorted by descending length. +func modulePrefixesExcludingTarget(path string) []string { + prefixes := make([]string, 0, strings.Count(path, "/")+1) + + for { + if path != targetPrefix { + if _, _, ok := module.SplitPathVersion(path); ok { + prefixes = append(prefixes, path) + } + } + + j := strings.LastIndexByte(path, '/') + if j < 0 { + break } - _, ok := dirInModule(path, m.Path, root, isLocal) - if ok { - return m, info, nil + path = path[:j] + } + + return prefixes +} + +func findCurrentVersion(path string) string { + for _, m := range buildList { + if m.Path == path { + return m.Version } } + return "" +} + +type prefixResult struct { + QueryResult + err error +} - return module.Version{}, nil, finalErr +func queryPrefixModules(candidateModules []string, queryModule func(path string) (QueryResult, error)) (found []QueryResult, err error) { + // If the path we're attempting is not in the module cache and we don't have a + // fetch result cached either, we'll end up making a (potentially slow) + // request to the proxy or (often even slower) the origin server. + // To minimize latency, execute all of those requests in parallel. + type result struct { + QueryResult + err error + } + results := make([]result, len(candidateModules)) + var wg sync.WaitGroup + wg.Add(len(candidateModules)) + for i, p := range candidateModules { + go func(p string, r *result) { + r.QueryResult, r.err = queryModule(p) + wg.Done() + }(p, &results[i]) + } + wg.Wait() + + // Classify the results. In case of failure, identify the error that the user + // is most likely to find helpful: the most useful class of error at the + // longest matching path. + var ( + noPackage *PackageNotInModuleError + noVersion *NoMatchingVersionError + notExistErr error + ) + for _, r := range results { + switch rErr := r.err.(type) { + case nil: + found = append(found, r.QueryResult) + case *PackageNotInModuleError: + // Given the option, prefer to attribute “package not in module” + // to modules other than the main one. + if noPackage == nil || noPackage.Mod == Target { + noPackage = rErr + } + case *NoMatchingVersionError: + if noVersion == nil { + noVersion = rErr + } + default: + if errors.Is(rErr, os.ErrNotExist) { + if notExistErr == nil { + notExistErr = rErr + } + } else if err == nil { + if len(found) > 0 || noPackage != nil { + // golang.org/issue/34094: If we have already found a module that + // could potentially contain the target package, ignore unclassified + // errors for modules with shorter paths. + + // golang.org/issue/34383 is a special case of this: if we have + // already found example.com/foo/v2@v2.0.0 with a matching go.mod + // file, ignore the error from example.com/foo@v2.0.0. + } else { + err = r.err + } + } + } + } + + // TODO(#26232): If len(found) == 0 and some of the errors are 4xx HTTP + // codes, have the auth package recheck the failed paths. + // If we obtain new credentials for any of them, re-run the above loop. + + if len(found) == 0 && err == nil { + switch { + case noPackage != nil: + err = noPackage + case noVersion != nil: + err = noVersion + case notExistErr != nil: + err = notExistErr + default: + panic("queryPrefixModules: no modules found, but no error detected") + } + } + + return found, err +} + +// A NoMatchingVersionError indicates that Query found a module at the requested +// path, but not at any versions satisfying the query string and allow-function. +// +// NOTE: NoMatchingVersionError MUST NOT implement Is(os.ErrNotExist). +// +// If the module came from a proxy, that proxy had to return a successful status +// code for the versions it knows about, and thus did not have the opportunity +// to return a non-400 status code to suppress fallback. +type NoMatchingVersionError struct { + query, current string +} + +func (e *NoMatchingVersionError) Error() string { + currentSuffix := "" + if (e.query == "upgrade" || e.query == "patch") && e.current != "" { + currentSuffix = fmt.Sprintf(" (current version is %s)", e.current) + } + return fmt.Sprintf("no matching versions for query %q", e.query) + currentSuffix +} + +// A PackageNotInModuleError indicates that QueryPattern found a candidate +// module at the requested version, but that module did not contain any packages +// matching the requested pattern. +// +// NOTE: PackageNotInModuleError MUST NOT implement Is(os.ErrNotExist). +// +// If the module came from a proxy, that proxy had to return a successful status +// code for the versions it knows about, and thus did not have the opportunity +// to return a non-400 status code to suppress fallback. +type PackageNotInModuleError struct { + Mod module.Version + Replacement module.Version + Query string + Pattern string +} + +func (e *PackageNotInModuleError) Error() string { + if e.Mod == Target { + if strings.Contains(e.Pattern, "...") { + return fmt.Sprintf("main module (%s) does not contain packages matching %s", Target.Path, e.Pattern) + } + return fmt.Sprintf("main module (%s) does not contain package %s", Target.Path, e.Pattern) + } + + found := "" + if r := e.Replacement; r.Path != "" { + replacement := r.Path + if r.Version != "" { + replacement = fmt.Sprintf("%s@%s", r.Path, r.Version) + } + if e.Query == e.Mod.Version { + found = fmt.Sprintf(" (replaced by %s)", replacement) + } else { + found = fmt.Sprintf(" (%s, replaced by %s)", e.Mod.Version, replacement) + } + } else if e.Query != e.Mod.Version { + found = fmt.Sprintf(" (%s)", e.Mod.Version) + } + + if strings.Contains(e.Pattern, "...") { + return fmt.Sprintf("module %s@%s found%s, but does not contain packages matching %s", e.Mod.Path, e.Query, found, e.Pattern) + } + return fmt.Sprintf("module %s@%s found%s, but does not contain package %s", e.Mod.Path, e.Query, found, e.Pattern) +} + +func (e *PackageNotInModuleError) ImportPath() string { + if !strings.Contains(e.Pattern, "...") { + return e.Pattern + } + return "" +} + +// ModuleHasRootPackage returns whether module m contains a package m.Path. +func ModuleHasRootPackage(m module.Version) (bool, error) { + root, isLocal, err := fetch(m) + if err != nil { + return false, err + } + _, ok, err := dirInModule(m.Path, m.Path, root, isLocal) + return ok, err +} + +func versionHasGoMod(m module.Version) (bool, error) { + root, _, err := fetch(m) + if err != nil { + return false, err + } + fi, err := os.Stat(filepath.Join(root, "go.mod")) + return err == nil && !fi.IsDir(), nil } diff --git a/cmd/go/_internal_/modload/search.go b/cmd/go/_internal_/modload/search.go index 1a95eac..d4eee74 100644 --- a/cmd/go/_internal_/modload/search.go +++ b/cmd/go/_internal_/modload/search.go @@ -10,21 +10,31 @@ import ( "path/filepath" "strings" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/imports" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/module" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/search" + + "golang.org/x/mod/module" +) + +type stdFilter int8 + +const ( + omitStd = stdFilter(iota) + includeStd ) -// matchPackages returns a list of packages in the list of modules -// matching the pattern. Package loading assumes the given set of tags. -func matchPackages(pattern string, tags map[string]bool, useStd bool, modules []module.Version) []string { - match := func(string) bool { return true } +// matchPackages is like m.MatchPackages, but uses a local variable (rather than +// a global) for tags, can include or exclude packages in the standard library, +// and is restricted to the given list of modules. +func matchPackages(m *search.Match, tags map[string]bool, filter stdFilter, modules []module.Version) { + m.Pkgs = []string{} + + isMatch := func(string) bool { return true } treeCanMatch := func(string) bool { return true } - if !search.IsMetaPackage(pattern) { - match = search.MatchPattern(pattern) - treeCanMatch = search.TreeCanMatchPattern(pattern) + if !m.IsMeta() { + isMatch = search.MatchPattern(m.Pattern()) + treeCanMatch = search.TreeCanMatchPattern(m.Pattern()) } have := map[string]bool{ @@ -33,37 +43,35 @@ func matchPackages(pattern string, tags map[string]bool, useStd bool, modules [] if !cfg.BuildContext.CgoEnabled { have["runtime/cgo"] = true // ignore during walk } - var pkgs []string - walkPkgs := func(root, importPathRoot string) { + type pruning int8 + const ( + pruneVendor = pruning(1 << iota) + pruneGoMod + ) + + walkPkgs := func(root, importPathRoot string, prune pruning) { root = filepath.Clean(root) - var cmd string - if root == cfg.GOROOTsrc { - cmd = filepath.Join(root, "cmd") - } - filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { + err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { if err != nil { + m.AddError(err) return nil } - // Don't use GOROOT/src but do walk down into it. - if path == root && importPathRoot == "" { - return nil - } - - // GOROOT/src/cmd makes use of GOROOT/src/cmd/vendor, - // which module mode can't deal with. Eventually we'll stop using - // that vendor directory, and then we can remove this exclusion. - // golang.org/issue/26924. - if path == cmd { - return filepath.SkipDir - } - want := true - // Avoid .foo, _foo, and testdata directory trees. - _, elem := filepath.Split(path) - if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" { - want = false + elem := "" + + // Don't use GOROOT/src but do walk down into it. + if path == root { + if importPathRoot == "" { + return nil + } + } else { + // Avoid .foo, _foo, and testdata subdirectory trees. + _, elem = filepath.Split(path) + if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" { + want = false + } } name := importPathRoot + filepath.ToSlash(path[len(root):]) @@ -86,52 +94,79 @@ func matchPackages(pattern string, tags map[string]bool, useStd bool, modules [] if !want { return filepath.SkipDir } - if path != root { - if _, err := os.Stat(filepath.Join(path, "go.mod")); err == nil { + // Stop at module boundaries. + if (prune&pruneGoMod != 0) && path != root { + if fi, err := os.Stat(filepath.Join(path, "go.mod")); err == nil && !fi.IsDir() { return filepath.SkipDir } } if !have[name] { have[name] = true - if match(name) { + if isMatch(name) { if _, _, err := scanDir(path, tags); err != imports.ErrNoGo { - pkgs = append(pkgs, name) + m.Pkgs = append(m.Pkgs, name) } } } - if elem == "vendor" { + if elem == "vendor" && (prune&pruneVendor != 0) { return filepath.SkipDir } return nil }) + if err != nil { + m.AddError(err) + } } - if useStd { - walkPkgs(cfg.GOROOTsrc, "") + if filter == includeStd { + walkPkgs(cfg.GOROOTsrc, "", pruneGoMod) + if treeCanMatch("cmd") { + walkPkgs(filepath.Join(cfg.GOROOTsrc, "cmd"), "cmd", pruneGoMod) + } + } + + if cfg.BuildMod == "vendor" { + if HasModRoot() { + walkPkgs(ModRoot(), targetPrefix, pruneGoMod|pruneVendor) + walkPkgs(filepath.Join(ModRoot(), "vendor"), "", pruneVendor) + } + return } for _, mod := range modules { if !treeCanMatch(mod.Path) { continue } - var root string - if mod.Version == "" { + + var ( + root, modPrefix string + isLocal bool + ) + if mod == Target { if !HasModRoot() { continue // If there is no main module, we can't search in it. } root = ModRoot() + modPrefix = targetPrefix + isLocal = true } else { var err error - root, _, err = fetch(mod) + root, isLocal, err = fetch(mod) if err != nil { - base.Errorf("go: %v", err) + m.AddError(err) continue } + modPrefix = mod.Path + } + + prune := pruneVendor + if isLocal { + prune |= pruneGoMod } - walkPkgs(root, mod.Path) + walkPkgs(root, modPrefix, prune) } - return pkgs + return } diff --git a/cmd/go/_internal_/modload/stat_unix.go b/cmd/go/_internal_/modload/stat_unix.go new file mode 100644 index 0000000..ea3b801 --- /dev/null +++ b/cmd/go/_internal_/modload/stat_unix.go @@ -0,0 +1,31 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux netbsd openbsd solaris + +package modload + +import ( + "os" + "syscall" +) + +// hasWritePerm reports whether the current user has permission to write to the +// file with the given info. +// +// Although the root user on most Unix systems can write to files even without +// permission, hasWritePerm reports false if no appropriate permission bit is +// set even if the current user is root. +func hasWritePerm(path string, fi os.FileInfo) bool { + if os.Getuid() == 0 { + // The root user can access any file, but we still want to default to + // read-only mode if the go.mod file is marked as globally non-writable. + // (If the user really intends not to be in readonly mode, they can + // pass -mod=mod explicitly.) + return fi.Mode()&0222 != 0 + } + + const W_OK = 0x2 + return syscall.Access(path, W_OK) == nil +} diff --git a/cmd/go/_internal_/modload/vendor.go b/cmd/go/_internal_/modload/vendor.go new file mode 100644 index 0000000..db3735c --- /dev/null +++ b/cmd/go/_internal_/modload/vendor.go @@ -0,0 +1,217 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" + + "golang.org/x/mod/module" + "golang.org/x/mod/semver" +) + +var ( + vendorOnce sync.Once + vendorList []module.Version // modules that contribute packages to the build, in order of appearance + vendorReplaced []module.Version // all replaced modules; may or may not also contribute packages + vendorVersion map[string]string // module path → selected version (if known) + vendorPkgModule map[string]module.Version // package → containing module + vendorMeta map[module.Version]vendorMetadata +) + +type vendorMetadata struct { + Explicit bool + Replacement module.Version +} + +// readVendorList reads the list of vendored modules from vendor/modules.txt. +func readVendorList() { + vendorOnce.Do(func() { + vendorList = nil + vendorPkgModule = make(map[string]module.Version) + vendorVersion = make(map[string]string) + vendorMeta = make(map[module.Version]vendorMetadata) + data, err := ioutil.ReadFile(filepath.Join(ModRoot(), "vendor/modules.txt")) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + base.Fatalf("go: %s", err) + } + return + } + + var mod module.Version + for _, line := range strings.Split(string(data), "\n") { + if strings.HasPrefix(line, "# ") { + f := strings.Fields(line) + + if len(f) < 3 { + continue + } + if semver.IsValid(f[2]) { + // A module, but we don't yet know whether it is in the build list or + // only included to indicate a replacement. + mod = module.Version{Path: f[1], Version: f[2]} + f = f[3:] + } else if f[2] == "=>" { + // A wildcard replacement found in the main module's go.mod file. + mod = module.Version{Path: f[1]} + f = f[2:] + } else { + // Not a version or a wildcard replacement. + // We don't know how to interpret this module line, so ignore it. + mod = module.Version{} + continue + } + + if len(f) >= 2 && f[0] == "=>" { + meta := vendorMeta[mod] + if len(f) == 2 { + // File replacement. + meta.Replacement = module.Version{Path: f[1]} + vendorReplaced = append(vendorReplaced, mod) + } else if len(f) == 3 && semver.IsValid(f[2]) { + // Path and version replacement. + meta.Replacement = module.Version{Path: f[1], Version: f[2]} + vendorReplaced = append(vendorReplaced, mod) + } else { + // We don't understand this replacement. Ignore it. + } + vendorMeta[mod] = meta + } + continue + } + + // Not a module line. Must be a package within a module or a metadata + // directive, either of which requires a preceding module line. + if mod.Path == "" { + continue + } + + if strings.HasPrefix(line, "## ") { + // Metadata. Take the union of annotations across multiple lines, if present. + meta := vendorMeta[mod] + for _, entry := range strings.Split(strings.TrimPrefix(line, "## "), ";") { + entry = strings.TrimSpace(entry) + if entry == "explicit" { + meta.Explicit = true + } + // All other tokens are reserved for future use. + } + vendorMeta[mod] = meta + continue + } + + if f := strings.Fields(line); len(f) == 1 && module.CheckImportPath(f[0]) == nil { + // A package within the current module. + vendorPkgModule[f[0]] = mod + + // Since this module provides a package for the build, we know that it + // is in the build list and is the selected version of its path. + // If this information is new, record it. + if v, ok := vendorVersion[mod.Path]; !ok || semver.Compare(v, mod.Version) < 0 { + vendorList = append(vendorList, mod) + vendorVersion[mod.Path] = mod.Version + } + } + } + }) +} + +// checkVendorConsistency verifies that the vendor/modules.txt file matches (if +// go 1.14) or at least does not contradict (go 1.13 or earlier) the +// requirements and replacements listed in the main module's go.mod file. +func checkVendorConsistency() { + readVendorList() + + pre114 := false + if modFile.Go == nil || semver.Compare("v"+modFile.Go.Version, "v1.14") < 0 { + // Go versions before 1.14 did not include enough information in + // vendor/modules.txt to check for consistency. + // If we know that we're on an earlier version, relax the consistency check. + pre114 = true + } + + vendErrors := new(strings.Builder) + vendErrorf := func(mod module.Version, format string, args ...interface{}) { + detail := fmt.Sprintf(format, args...) + if mod.Version == "" { + fmt.Fprintf(vendErrors, "\n\t%s: %s", mod.Path, detail) + } else { + fmt.Fprintf(vendErrors, "\n\t%s@%s: %s", mod.Path, mod.Version, detail) + } + } + + for _, r := range modFile.Require { + if !vendorMeta[r.Mod].Explicit { + if pre114 { + // Before 1.14, modules.txt did not indicate whether modules were listed + // explicitly in the main module's go.mod file. + // However, we can at least detect a version mismatch if packages were + // vendored from a non-matching version. + if vv, ok := vendorVersion[r.Mod.Path]; ok && vv != r.Mod.Version { + vendErrorf(r.Mod, fmt.Sprintf("is explicitly required in go.mod, but vendor/modules.txt indicates %s@%s", r.Mod.Path, vv)) + } + } else { + vendErrorf(r.Mod, "is explicitly required in go.mod, but not marked as explicit in vendor/modules.txt") + } + } + } + + describe := func(m module.Version) string { + if m.Version == "" { + return m.Path + } + return m.Path + "@" + m.Version + } + + // We need to verify *all* replacements that occur in modfile: even if they + // don't directly apply to any module in the vendor list, the replacement + // go.mod file can affect the selected versions of other (transitive) + // dependencies + for _, r := range modFile.Replace { + vr := vendorMeta[r.Old].Replacement + if vr == (module.Version{}) { + if pre114 && (r.Old.Version == "" || vendorVersion[r.Old.Path] != r.Old.Version) { + // Before 1.14, modules.txt omitted wildcard replacements and + // replacements for modules that did not have any packages to vendor. + } else { + vendErrorf(r.Old, "is replaced in go.mod, but not marked as replaced in vendor/modules.txt") + } + } else if vr != r.New { + vendErrorf(r.Old, "is replaced by %s in go.mod, but marked as replaced by %s in vendor/modules.txt", describe(r.New), describe(vr)) + } + } + + for _, mod := range vendorList { + meta := vendorMeta[mod] + if meta.Explicit { + if _, inGoMod := index.require[mod]; !inGoMod { + vendErrorf(mod, "is marked as explicit in vendor/modules.txt, but not explicitly required in go.mod") + } + } + } + + for _, mod := range vendorReplaced { + r := Replacement(mod) + if r == (module.Version{}) { + vendErrorf(mod, "is marked as replaced in vendor/modules.txt, but not replaced in go.mod") + continue + } + if meta := vendorMeta[mod]; r != meta.Replacement { + vendErrorf(mod, "is marked as replaced by %s in vendor/modules.txt, but replaced by %s in go.mod", describe(meta.Replacement), describe(r)) + } + } + + if vendErrors.Len() > 0 { + base.Fatalf("go: inconsistent vendoring in %s:%s\n\nrun 'go mod vendor' to sync, or use -mod=mod or -mod=readonly to ignore the vendor directory", modRoot, vendErrors) + } +} diff --git a/cmd/go/_internal_/module/module.go b/cmd/go/_internal_/module/module.go deleted file mode 100644 index 2ad7f94..0000000 --- a/cmd/go/_internal_/module/module.go +++ /dev/null @@ -1,540 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package module defines the module.Version type -// along with support code. -package module - -// IMPORTANT NOTE -// -// This file essentially defines the set of valid import paths for the go command. -// There are many subtle considerations, including Unicode ambiguity, -// security, network, and file system representations. -// -// This file also defines the set of valid module path and version combinations, -// another topic with many subtle considerations. -// -// Changes to the semantics in this file require approval from rsc. - -import ( - "fmt" - "sort" - "strings" - "unicode" - "unicode/utf8" - - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/semver" -) - -// A Version is defined by a module path and version pair. -type Version struct { - Path string - - // Version is usually a semantic version in canonical form. - // There are two exceptions to this general rule. - // First, the top-level target of a build has no specific version - // and uses Version = "". - // Second, during MVS calculations the version "none" is used - // to represent the decision to take no version of a given module. - Version string `json:",omitempty"` -} - -// Check checks that a given module path, version pair is valid. -// In addition to the path being a valid module path -// and the version being a valid semantic version, -// the two must correspond. -// For example, the path "yaml/v2" only corresponds to -// semantic versions beginning with "v2.". -func Check(path, version string) error { - if err := CheckPath(path); err != nil { - return err - } - if !semver.IsValid(version) { - return fmt.Errorf("malformed semantic version %v", version) - } - _, pathMajor, _ := SplitPathVersion(path) - if !MatchPathMajor(version, pathMajor) { - if pathMajor == "" { - pathMajor = "v0 or v1" - } - if pathMajor[0] == '.' { // .v1 - pathMajor = pathMajor[1:] - } - return fmt.Errorf("mismatched module path %v and version %v (want %v)", path, version, pathMajor) - } - return nil -} - -// firstPathOK reports whether r can appear in the first element of a module path. -// The first element of the path must be an LDH domain name, at least for now. -// To avoid case ambiguity, the domain name must be entirely lower case. -func firstPathOK(r rune) bool { - return r == '-' || r == '.' || - '0' <= r && r <= '9' || - 'a' <= r && r <= 'z' -} - -// pathOK reports whether r can appear in an import path element. -// Paths can be ASCII letters, ASCII digits, and limited ASCII punctuation: + - . _ and ~. -// This matches what "go get" has historically recognized in import paths. -// TODO(rsc): We would like to allow Unicode letters, but that requires additional -// care in the safe encoding (see note below). -func pathOK(r rune) bool { - if r < utf8.RuneSelf { - return r == '+' || r == '-' || r == '.' || r == '_' || r == '~' || - '0' <= r && r <= '9' || - 'A' <= r && r <= 'Z' || - 'a' <= r && r <= 'z' - } - return false -} - -// fileNameOK reports whether r can appear in a file name. -// For now we allow all Unicode letters but otherwise limit to pathOK plus a few more punctuation characters. -// If we expand the set of allowed characters here, we have to -// work harder at detecting potential case-folding and normalization collisions. -// See note about "safe encoding" below. -func fileNameOK(r rune) bool { - if r < utf8.RuneSelf { - // Entire set of ASCII punctuation, from which we remove characters: - // ! " # $ % & ' ( ) * + , - . / : ; < = > ? @ [ \ ] ^ _ ` { | } ~ - // We disallow some shell special characters: " ' * < > ? ` | - // (Note that some of those are disallowed by the Windows file system as well.) - // We also disallow path separators / : and \ (fileNameOK is only called on path element characters). - // We allow spaces (U+0020) in file names. - const allowed = "!#$%&()+,-.=@[]^_{}~ " - if '0' <= r && r <= '9' || 'A' <= r && r <= 'Z' || 'a' <= r && r <= 'z' { - return true - } - for i := 0; i < len(allowed); i++ { - if rune(allowed[i]) == r { - return true - } - } - return false - } - // It may be OK to add more ASCII punctuation here, but only carefully. - // For example Windows disallows < > \, and macOS disallows :, so we must not allow those. - return unicode.IsLetter(r) -} - -// CheckPath checks that a module path is valid. -func CheckPath(path string) error { - if err := checkPath(path, false); err != nil { - return fmt.Errorf("malformed module path %q: %v", path, err) - } - i := strings.Index(path, "/") - if i < 0 { - i = len(path) - } - if i == 0 { - return fmt.Errorf("malformed module path %q: leading slash", path) - } - if !strings.Contains(path[:i], ".") { - return fmt.Errorf("malformed module path %q: missing dot in first path element", path) - } - if path[0] == '-' { - return fmt.Errorf("malformed module path %q: leading dash in first path element", path) - } - for _, r := range path[:i] { - if !firstPathOK(r) { - return fmt.Errorf("malformed module path %q: invalid char %q in first path element", path, r) - } - } - if _, _, ok := SplitPathVersion(path); !ok { - return fmt.Errorf("malformed module path %q: invalid version", path) - } - return nil -} - -// CheckImportPath checks that an import path is valid. -func CheckImportPath(path string) error { - if err := checkPath(path, false); err != nil { - return fmt.Errorf("malformed import path %q: %v", path, err) - } - return nil -} - -// checkPath checks that a general path is valid. -// It returns an error describing why but not mentioning path. -// Because these checks apply to both module paths and import paths, -// the caller is expected to add the "malformed ___ path %q: " prefix. -// fileName indicates whether the final element of the path is a file name -// (as opposed to a directory name). -func checkPath(path string, fileName bool) error { - if !utf8.ValidString(path) { - return fmt.Errorf("invalid UTF-8") - } - if path == "" { - return fmt.Errorf("empty string") - } - if strings.Contains(path, "..") { - return fmt.Errorf("double dot") - } - if strings.Contains(path, "//") { - return fmt.Errorf("double slash") - } - if path[len(path)-1] == '/' { - return fmt.Errorf("trailing slash") - } - elemStart := 0 - for i, r := range path { - if r == '/' { - if err := checkElem(path[elemStart:i], fileName); err != nil { - return err - } - elemStart = i + 1 - } - } - if err := checkElem(path[elemStart:], fileName); err != nil { - return err - } - return nil -} - -// checkElem checks whether an individual path element is valid. -// fileName indicates whether the element is a file name (not a directory name). -func checkElem(elem string, fileName bool) error { - if elem == "" { - return fmt.Errorf("empty path element") - } - if strings.Count(elem, ".") == len(elem) { - return fmt.Errorf("invalid path element %q", elem) - } - if elem[0] == '.' && !fileName { - return fmt.Errorf("leading dot in path element") - } - if elem[len(elem)-1] == '.' { - return fmt.Errorf("trailing dot in path element") - } - charOK := pathOK - if fileName { - charOK = fileNameOK - } - for _, r := range elem { - if !charOK(r) { - return fmt.Errorf("invalid char %q", r) - } - } - - // Windows disallows a bunch of path elements, sadly. - // See https://docs.microsoft.com/en-us/windows/desktop/fileio/naming-a-file - short := elem - if i := strings.Index(short, "."); i >= 0 { - short = short[:i] - } - for _, bad := range badWindowsNames { - if strings.EqualFold(bad, short) { - return fmt.Errorf("%q disallowed as path element component on Windows", short) - } - } - return nil -} - -// CheckFilePath checks whether a slash-separated file path is valid. -func CheckFilePath(path string) error { - if err := checkPath(path, true); err != nil { - return fmt.Errorf("malformed file path %q: %v", path, err) - } - return nil -} - -// badWindowsNames are the reserved file path elements on Windows. -// See https://docs.microsoft.com/en-us/windows/desktop/fileio/naming-a-file -var badWindowsNames = []string{ - "CON", - "PRN", - "AUX", - "NUL", - "COM1", - "COM2", - "COM3", - "COM4", - "COM5", - "COM6", - "COM7", - "COM8", - "COM9", - "LPT1", - "LPT2", - "LPT3", - "LPT4", - "LPT5", - "LPT6", - "LPT7", - "LPT8", - "LPT9", -} - -// SplitPathVersion returns prefix and major version such that prefix+pathMajor == path -// and version is either empty or "/vN" for N >= 2. -// As a special case, gopkg.in paths are recognized directly; -// they require ".vN" instead of "/vN", and for all N, not just N >= 2. -func SplitPathVersion(path string) (prefix, pathMajor string, ok bool) { - if strings.HasPrefix(path, "gopkg.in/") { - return splitGopkgIn(path) - } - - i := len(path) - dot := false - for i > 0 && ('0' <= path[i-1] && path[i-1] <= '9' || path[i-1] == '.') { - if path[i-1] == '.' { - dot = true - } - i-- - } - if i <= 1 || i == len(path) || path[i-1] != 'v' || path[i-2] != '/' { - return path, "", true - } - prefix, pathMajor = path[:i-2], path[i-2:] - if dot || len(pathMajor) <= 2 || pathMajor[2] == '0' || pathMajor == "/v1" { - return path, "", false - } - return prefix, pathMajor, true -} - -// splitGopkgIn is like SplitPathVersion but only for gopkg.in paths. -func splitGopkgIn(path string) (prefix, pathMajor string, ok bool) { - if !strings.HasPrefix(path, "gopkg.in/") { - return path, "", false - } - i := len(path) - if strings.HasSuffix(path, "-unstable") { - i -= len("-unstable") - } - for i > 0 && ('0' <= path[i-1] && path[i-1] <= '9') { - i-- - } - if i <= 1 || path[i-1] != 'v' || path[i-2] != '.' { - // All gopkg.in paths must end in vN for some N. - return path, "", false - } - prefix, pathMajor = path[:i-2], path[i-2:] - if len(pathMajor) <= 2 || pathMajor[2] == '0' && pathMajor != ".v0" { - return path, "", false - } - return prefix, pathMajor, true -} - -// MatchPathMajor reports whether the semantic version v -// matches the path major version pathMajor. -func MatchPathMajor(v, pathMajor string) bool { - if strings.HasPrefix(pathMajor, ".v") && strings.HasSuffix(pathMajor, "-unstable") { - pathMajor = strings.TrimSuffix(pathMajor, "-unstable") - } - if strings.HasPrefix(v, "v0.0.0-") && pathMajor == ".v1" { - // Allow old bug in pseudo-versions that generated v0.0.0- pseudoversion for gopkg .v1. - // For example, gopkg.in/yaml.v2@v2.2.1's go.mod requires gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405. - return true - } - m := semver.Major(v) - if pathMajor == "" { - return m == "v0" || m == "v1" || semver.Build(v) == "+incompatible" - } - return (pathMajor[0] == '/' || pathMajor[0] == '.') && m == pathMajor[1:] -} - -// CanonicalVersion returns the canonical form of the version string v. -// It is the same as semver.Canonical(v) except that it preserves the special build suffix "+incompatible". -func CanonicalVersion(v string) string { - cv := semver.Canonical(v) - if semver.Build(v) == "+incompatible" { - cv += "+incompatible" - } - return cv -} - -// Sort sorts the list by Path, breaking ties by comparing Versions. -func Sort(list []Version) { - sort.Slice(list, func(i, j int) bool { - mi := list[i] - mj := list[j] - if mi.Path != mj.Path { - return mi.Path < mj.Path - } - // To help go.sum formatting, allow version/file. - // Compare semver prefix by semver rules, - // file by string order. - vi := mi.Version - vj := mj.Version - var fi, fj string - if k := strings.Index(vi, "/"); k >= 0 { - vi, fi = vi[:k], vi[k:] - } - if k := strings.Index(vj, "/"); k >= 0 { - vj, fj = vj[:k], vj[k:] - } - if vi != vj { - return semver.Compare(vi, vj) < 0 - } - return fi < fj - }) -} - -// Safe encodings -// -// Module paths appear as substrings of file system paths -// (in the download cache) and of web server URLs in the proxy protocol. -// In general we cannot rely on file systems to be case-sensitive, -// nor can we rely on web servers, since they read from file systems. -// That is, we cannot rely on the file system to keep rsc.io/QUOTE -// and rsc.io/quote separate. Windows and macOS don't. -// Instead, we must never require two different casings of a file path. -// Because we want the download cache to match the proxy protocol, -// and because we want the proxy protocol to be possible to serve -// from a tree of static files (which might be stored on a case-insensitive -// file system), the proxy protocol must never require two different casings -// of a URL path either. -// -// One possibility would be to make the safe encoding be the lowercase -// hexadecimal encoding of the actual path bytes. This would avoid ever -// needing different casings of a file path, but it would be fairly illegible -// to most programmers when those paths appeared in the file system -// (including in file paths in compiler errors and stack traces) -// in web server logs, and so on. Instead, we want a safe encoding that -// leaves most paths unaltered. -// -// The safe encoding is this: -// replace every uppercase letter with an exclamation mark -// followed by the letter's lowercase equivalent. -// -// For example, -// github.com/Azure/azure-sdk-for-go -> github.com/!azure/azure-sdk-for-go. -// github.com/GoogleCloudPlatform/cloudsql-proxy -> github.com/!google!cloud!platform/cloudsql-proxy -// github.com/Sirupsen/logrus -> github.com/!sirupsen/logrus. -// -// Import paths that avoid upper-case letters are left unchanged. -// Note that because import paths are ASCII-only and avoid various -// problematic punctuation (like : < and >), the safe encoding is also ASCII-only -// and avoids the same problematic punctuation. -// -// Import paths have never allowed exclamation marks, so there is no -// need to define how to encode a literal !. -// -// Although paths are disallowed from using Unicode (see pathOK above), -// the eventual plan is to allow Unicode letters as well, to assume that -// file systems and URLs are Unicode-safe (storing UTF-8), and apply -// the !-for-uppercase convention. Note however that not all runes that -// are different but case-fold equivalent are an upper/lower pair. -// For example, U+004B ('K'), U+006B ('k'), and U+212A ('K' for Kelvin) -// are considered to case-fold to each other. When we do add Unicode -// letters, we must not assume that upper/lower are the only case-equivalent pairs. -// Perhaps the Kelvin symbol would be disallowed entirely, for example. -// Or perhaps it would encode as "!!k", or perhaps as "(212A)". -// -// Also, it would be nice to allow Unicode marks as well as letters, -// but marks include combining marks, and then we must deal not -// only with case folding but also normalization: both U+00E9 ('é') -// and U+0065 U+0301 ('e' followed by combining acute accent) -// look the same on the page and are treated by some file systems -// as the same path. If we do allow Unicode marks in paths, there -// must be some kind of normalization to allow only one canonical -// encoding of any character used in an import path. - -// EncodePath returns the safe encoding of the given module path. -// It fails if the module path is invalid. -func EncodePath(path string) (encoding string, err error) { - if err := CheckPath(path); err != nil { - return "", err - } - - return encodeString(path) -} - -// EncodeVersion returns the safe encoding of the given module version. -// Versions are allowed to be in non-semver form but must be valid file names -// and not contain exclamation marks. -func EncodeVersion(v string) (encoding string, err error) { - if err := checkElem(v, true); err != nil || strings.Contains(v, "!") { - return "", fmt.Errorf("disallowed version string %q", v) - } - return encodeString(v) -} - -func encodeString(s string) (encoding string, err error) { - haveUpper := false - for _, r := range s { - if r == '!' || r >= utf8.RuneSelf { - // This should be disallowed by CheckPath, but diagnose anyway. - // The correctness of the encoding loop below depends on it. - return "", fmt.Errorf("internal error: inconsistency in EncodePath") - } - if 'A' <= r && r <= 'Z' { - haveUpper = true - } - } - - if !haveUpper { - return s, nil - } - - var buf []byte - for _, r := range s { - if 'A' <= r && r <= 'Z' { - buf = append(buf, '!', byte(r+'a'-'A')) - } else { - buf = append(buf, byte(r)) - } - } - return string(buf), nil -} - -// DecodePath returns the module path of the given safe encoding. -// It fails if the encoding is invalid or encodes an invalid path. -func DecodePath(encoding string) (path string, err error) { - path, ok := decodeString(encoding) - if !ok { - return "", fmt.Errorf("invalid module path encoding %q", encoding) - } - if err := CheckPath(path); err != nil { - return "", fmt.Errorf("invalid module path encoding %q: %v", encoding, err) - } - return path, nil -} - -// DecodeVersion returns the version string for the given safe encoding. -// It fails if the encoding is invalid or encodes an invalid version. -// Versions are allowed to be in non-semver form but must be valid file names -// and not contain exclamation marks. -func DecodeVersion(encoding string) (v string, err error) { - v, ok := decodeString(encoding) - if !ok { - return "", fmt.Errorf("invalid version encoding %q", encoding) - } - if err := checkElem(v, true); err != nil { - return "", fmt.Errorf("disallowed version string %q", v) - } - return v, nil -} - -func decodeString(encoding string) (string, bool) { - var buf []byte - - bang := false - for _, r := range encoding { - if r >= utf8.RuneSelf { - return "", false - } - if bang { - bang = false - if r < 'a' || 'z' < r { - return "", false - } - buf = append(buf, byte(r+'A'-'a')) - continue - } - if r == '!' { - bang = true - continue - } - if 'A' <= r && r <= 'Z' { - return "", false - } - buf = append(buf, byte(r)) - } - if bang { - return "", false - } - return string(buf), true -} diff --git a/cmd/go/_internal_/mvs/mvs.go b/cmd/go/_internal_/mvs/mvs.go index 24742c1..59905aa 100644 --- a/cmd/go/_internal_/mvs/mvs.go +++ b/cmd/go/_internal_/mvs/mvs.go @@ -9,11 +9,13 @@ package mvs import ( "fmt" "sort" + "strings" "sync" + "sync/atomic" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/module" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/par" + + "golang.org/x/mod/module" ) // A Reqs is the requirement graph on which Minimal Version Selection (MVS) operates. @@ -34,7 +36,7 @@ type Reqs interface { // Max returns the maximum of v1 and v2 (it returns either v1 or v2). // // For all versions v, Max(v, "none") must be v, - // and for the tanget passed as the first argument to MVS functions, + // and for the target passed as the first argument to MVS functions, // Max(target, v) must be target. // // Note that v1 < v2 can be written Max(v1, v2) != v1 @@ -59,89 +61,199 @@ type Reqs interface { Previous(m module.Version) (module.Version, error) } -type MissingModuleError struct { - Module module.Version +// BuildListError decorates an error that occurred gathering requirements +// while constructing a build list. BuildListError prints the chain +// of requirements to the module where the error occurred. +type BuildListError struct { + Err error + stack []buildListErrorElem +} + +type buildListErrorElem struct { + m module.Version + + // nextReason is the reason this module depends on the next module in the + // stack. Typically either "requires", or "upgraded to". + nextReason string +} + +// Module returns the module where the error occurred. If the module stack +// is empty, this returns a zero value. +func (e *BuildListError) Module() module.Version { + if len(e.stack) == 0 { + return module.Version{} + } + return e.stack[0].m } -func (e *MissingModuleError) Error() string { - return fmt.Sprintf("missing module: %v", e.Module) +func (e *BuildListError) Error() string { + b := &strings.Builder{} + stack := e.stack + + // Don't print modules at the beginning of the chain without a + // version. These always seem to be the main module or a + // synthetic module ("target@"). + for len(stack) > 0 && stack[len(stack)-1].m.Version == "" { + stack = stack[:len(stack)-1] + } + + for i := len(stack) - 1; i >= 1; i-- { + fmt.Fprintf(b, "%s@%s %s\n\t", stack[i].m.Path, stack[i].m.Version, stack[i].nextReason) + } + if len(stack) == 0 { + b.WriteString(e.Err.Error()) + } else { + // Ensure that the final module path and version are included as part of the + // error message. + if _, ok := e.Err.(*module.ModuleError); ok { + fmt.Fprintf(b, "%v", e.Err) + } else { + fmt.Fprintf(b, "%v", module.VersionError(stack[0].m, e.Err)) + } + } + return b.String() } // BuildList returns the build list for the target module. -// The first element is the target itself, with the remainder of the list sorted by path. +// +// target is the root vertex of a module requirement graph. For cmd/go, this is +// typically the main module, but note that this algorithm is not intended to +// be Go-specific: module paths and versions are treated as opaque values. +// +// reqs describes the module requirement graph and provides an opaque method +// for comparing versions. +// +// BuildList traverses the graph and returns a list containing the highest +// version for each visited module. The first element of the returned list is +// target itself; reqs.Max requires target.Version to compare higher than all +// other versions, so no other version can be selected. The remaining elements +// of the list are sorted by path. +// +// See https://research.swtch.com/vgo-mvs for details. func BuildList(target module.Version, reqs Reqs) ([]module.Version, error) { return buildList(target, reqs, nil) } -func buildList(target module.Version, reqs Reqs, upgrade func(module.Version) module.Version) ([]module.Version, error) { +func buildList(target module.Version, reqs Reqs, upgrade func(module.Version) (module.Version, error)) ([]module.Version, error) { // Explore work graph in parallel in case reqs.Required // does high-latency network operations. - var work par.Work - work.Add(target) + type modGraphNode struct { + m module.Version + required []module.Version + upgrade module.Version + err error + } var ( mu sync.Mutex - min = map[string]string{target.Path: target.Version} - firstErr error + modGraph = map[module.Version]*modGraphNode{} + min = map[string]string{} // maps module path to minimum required version + haveErr int32 ) + setErr := func(n *modGraphNode, err error) { + n.err = err + atomic.StoreInt32(&haveErr, 1) + } + + var work par.Work + work.Add(target) work.Do(10, func(item interface{}) { m := item.(module.Version) - required, err := reqs.Required(m) + node := &modGraphNode{m: m} mu.Lock() - if err != nil && firstErr == nil { - firstErr = err - } - if firstErr != nil { - mu.Unlock() - return - } + modGraph[m] = node if v, ok := min[m.Path]; !ok || reqs.Max(v, m.Version) != v { min[m.Path] = m.Version } mu.Unlock() - for _, r := range required { - if r.Path == "" { - base.Errorf("Required(%v) returned zero module in list", m) - continue - } + required, err := reqs.Required(m) + if err != nil { + setErr(node, err) + return + } + node.required = required + for _, r := range node.required { work.Add(r) } if upgrade != nil { - u := upgrade(m) - if u.Path == "" { - base.Errorf("Upgrade(%v) returned zero module", m) + u, err := upgrade(m) + if err != nil { + setErr(node, err) return } - work.Add(u) + if u != m { + node.upgrade = u + work.Add(u) + } } }) - if firstErr != nil { - return nil, firstErr + // If there was an error, find the shortest path from the target to the + // node where the error occurred so we can report a useful error message. + if haveErr != 0 { + // neededBy[a] = b means a was added to the module graph by b. + neededBy := make(map[*modGraphNode]*modGraphNode) + q := make([]*modGraphNode, 0, len(modGraph)) + q = append(q, modGraph[target]) + for len(q) > 0 { + node := q[0] + q = q[1:] + + if node.err != nil { + err := &BuildListError{ + Err: node.err, + stack: []buildListErrorElem{{m: node.m}}, + } + for n, prev := neededBy[node], node; n != nil; n, prev = neededBy[n], n { + reason := "requires" + if n.upgrade == prev.m { + reason = "updating to" + } + err.stack = append(err.stack, buildListErrorElem{m: n.m, nextReason: reason}) + } + return nil, err + } + + neighbors := node.required + if node.upgrade.Path != "" { + neighbors = append(neighbors, node.upgrade) + } + for _, neighbor := range neighbors { + nn := modGraph[neighbor] + if neededBy[nn] != nil { + continue + } + neededBy[nn] = node + q = append(q, nn) + } + } } + + // The final list is the minimum version of each module found in the graph. + if v := min[target.Path]; v != target.Version { + // target.Version will be "" for modload, the main client of MVS. + // "" denotes the main module, which has no version. However, MVS treats + // version strings as opaque, so "" is not a special value here. + // See golang.org/issue/31491, golang.org/issue/29773. panic(fmt.Sprintf("mistake: chose version %q instead of target %+v", v, target)) // TODO: Don't panic. } list := []module.Version{target} - listed := map[string]bool{target.Path: true} - for i := 0; i < len(list); i++ { - m := list[i] - required, err := reqs.Required(m) - if err != nil { - return nil, err + for path, vers := range min { + if path != target.Path { + list = append(list, module.Version{Path: path, Version: vers}) } + + n := modGraph[module.Version{Path: path, Version: vers}] + required := n.required for _, r := range required { v := min[r.Path] if r.Path != target.Path && reqs.Max(v, r.Version) != v { panic(fmt.Sprintf("mistake: version %q does not satisfy requirement %+v", v, r)) // TODO: Don't panic. } - if !listed[r.Path] { - list = append(list, module.Version{Path: r.Path, Version: v}) - listed[r.Path] = true - } } } @@ -152,10 +264,15 @@ func buildList(target module.Version, reqs Reqs, upgrade func(module.Version) mo return list, nil } -// Req returns the minimal requirement list for the target module -// that results in the given build list, with the constraint that all -// module paths listed in base must appear in the returned list. -func Req(target module.Version, list []module.Version, base []string, reqs Reqs) ([]module.Version, error) { +// Req returns the minimal requirement list for the target module, +// with the constraint that all module paths listed in base must +// appear in the returned list. +func Req(target module.Version, base []string, reqs Reqs) ([]module.Version, error) { + list, err := BuildList(target, reqs) + if err != nil { + return nil, err + } + // Note: Not running in parallel because we assume // that list came from a previous operation that paged // in all the requirements, so there's no I/O to overlap now. @@ -190,12 +307,12 @@ func Req(target module.Version, list []module.Version, base []string, reqs Reqs) } // Walk modules in reverse post-order, only adding those not implied already. - have := map[string]string{} + have := map[module.Version]bool{} walk = func(m module.Version) error { - if v, ok := have[m.Path]; ok && reqs.Max(m.Version, v) == v { + if have[m] { return nil } - have[m.Path] = m.Version + have[m] = true for _, m1 := range reqCache[m] { walk(m1) } @@ -223,7 +340,7 @@ func Req(target module.Version, list []module.Version, base []string, reqs Reqs) // Older version. continue } - if have[m.Path] != m.Version { + if !have[m] { min = append(min, m) walk(m) } @@ -237,17 +354,12 @@ func Req(target module.Version, list []module.Version, base []string, reqs Reqs) // UpgradeAll returns a build list for the target module // in which every module is upgraded to its latest version. func UpgradeAll(target module.Version, reqs Reqs) ([]module.Version, error) { - return buildList(target, reqs, func(m module.Version) module.Version { + return buildList(target, reqs, func(m module.Version) (module.Version, error) { if m.Path == target.Path { - return target + return target, nil } - latest, err := reqs.Upgrade(m) - if err != nil { - panic(err) // TODO - } - m.Version = latest.Version - return m + return reqs.Upgrade(m) }) } @@ -256,7 +368,7 @@ func UpgradeAll(target module.Version, reqs Reqs) ([]module.Version, error) { func Upgrade(target module.Version, reqs Reqs, upgrade ...module.Version) ([]module.Version, error) { list, err := reqs.Required(target) if err != nil { - panic(err) // TODO + return nil, err } // TODO: Maybe if an error is given, // rerun with BuildList(upgrade[0], reqs) etc @@ -275,7 +387,7 @@ func Upgrade(target module.Version, reqs Reqs, upgrade ...module.Version) ([]mod func Downgrade(target module.Version, reqs Reqs, downgrade ...module.Version) ([]module.Version, error) { list, err := reqs.Required(target) if err != nil { - panic(err) // TODO + return nil, err } max := make(map[string]string) for _, r := range list { @@ -314,7 +426,17 @@ func Downgrade(target module.Version, reqs Reqs, downgrade ...module.Version) ([ } list, err := reqs.Required(m) if err != nil { - panic(err) // TODO + // If we can't load the requirements, we couldn't load the go.mod file. + // There are a number of reasons this can happen, but this usually + // means an older version of the module had a missing or invalid + // go.mod file. For example, if example.com/mod released v2.0.0 before + // migrating to modules (v2.0.0+incompatible), then added a valid go.mod + // in v2.0.1, downgrading from v2.0.1 would cause this error. + // + // TODO(golang.org/issue/31730, golang.org/issue/30134): if the error + // is transient (we couldn't download go.mod), return the error from + // Downgrade. Currently, we can't tell what kind of error it is. + exclude(m) } for _, r := range list { add(r) @@ -334,7 +456,12 @@ List: for excluded[r] { p, err := reqs.Previous(r) if err != nil { - return nil, err // TODO + // This is likely a transient error reaching the repository, + // rather than a permanent error with the retrieved version. + // + // TODO(golang.org/issue/31730, golang.org/issue/30134): + // decode what to do based on the actual error. + return nil, err } // If the target version is a pseudo-version, it may not be // included when iterating over prior versions using reqs.Previous. diff --git a/cmd/go/_internal_/par/work.go b/cmd/go/_internal_/par/work.go index cf5278b..ecd88e9 100644 --- a/cmd/go/_internal_/par/work.go +++ b/cmd/go/_internal_/par/work.go @@ -147,3 +147,44 @@ func (c *Cache) Get(key interface{}) interface{} { } return e.result } + +// Clear removes all entries in the cache. +// +// Concurrent calls to Get may return old values. Concurrent calls to Do +// may return old values or store results in entries that have been deleted. +// +// TODO(jayconrod): Delete this after the package cache clearing functions +// in internal/load have been removed. +func (c *Cache) Clear() { + c.m.Range(func(key, value interface{}) bool { + c.m.Delete(key) + return true + }) +} + +// Delete removes an entry from the map. It is safe to call Delete for an +// entry that does not exist. Delete will return quickly, even if the result +// for a key is still being computed; the computation will finish, but the +// result won't be accessible through the cache. +// +// TODO(jayconrod): Delete this after the package cache clearing functions +// in internal/load have been removed. +func (c *Cache) Delete(key interface{}) { + c.m.Delete(key) +} + +// DeleteIf calls pred for each key in the map. If pred returns true for a key, +// DeleteIf removes the corresponding entry. If the result for a key is +// still being computed, DeleteIf will remove the entry without waiting for +// the computation to finish. The result won't be accessible through the cache. +// +// TODO(jayconrod): Delete this after the package cache clearing functions +// in internal/load have been removed. +func (c *Cache) DeleteIf(pred func(key interface{}) bool) { + c.m.Range(func(key, _ interface{}) bool { + if pred(key) { + c.Delete(key) + } + return true + }) +} diff --git a/cmd/go/_internal_/renameio/renameio.go b/cmd/go/_internal_/renameio/renameio.go index 8f59e1a..0c0a66b 100644 --- a/cmd/go/_internal_/renameio/renameio.go +++ b/cmd/go/_internal_/renameio/renameio.go @@ -8,12 +8,15 @@ package renameio import ( "bytes" "io" - "io/ioutil" + "math/rand" "os" "path/filepath" + "strconv" + + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/robustio" ) -const patternSuffix = "*.tmp" +const patternSuffix = ".tmp" // Pattern returns a glob pattern that matches the unrenamed temporary files // created when writing to filename. @@ -26,14 +29,14 @@ func Pattern(filename string) string { // final name. // // That ensures that the final location, if it exists, is always a complete file. -func WriteFile(filename string, data []byte) (err error) { - return WriteToFile(filename, bytes.NewReader(data)) +func WriteFile(filename string, data []byte, perm os.FileMode) (err error) { + return WriteToFile(filename, bytes.NewReader(data), perm) } // WriteToFile is a variant of WriteFile that accepts the data as an io.Reader // instead of a slice. -func WriteToFile(filename string, data io.Reader) (err error) { - f, err := ioutil.TempFile(filepath.Dir(filename), filepath.Base(filename)+patternSuffix) +func WriteToFile(filename string, data io.Reader, perm os.FileMode) (err error) { + f, err := tempFile(filepath.Dir(filename), filepath.Base(filename), perm) if err != nil { return err } @@ -59,5 +62,32 @@ func WriteToFile(filename string, data io.Reader) (err error) { if err := f.Close(); err != nil { return err } - return os.Rename(f.Name(), filename) + + return robustio.Rename(f.Name(), filename) +} + +// ReadFile is like ioutil.ReadFile, but on Windows retries spurious errors that +// may occur if the file is concurrently replaced. +// +// Errors are classified heuristically and retries are bounded, so even this +// function may occasionally return a spurious error on Windows. +// If so, the error will likely wrap one of: +// - syscall.ERROR_ACCESS_DENIED +// - syscall.ERROR_FILE_NOT_FOUND +// - internal/syscall/windows.ERROR_SHARING_VIOLATION +func ReadFile(filename string) ([]byte, error) { + return robustio.ReadFile(filename) +} + +// tempFile creates a new temporary file with given permission bits. +func tempFile(dir, prefix string, perm os.FileMode) (f *os.File, err error) { + for i := 0; i < 10000; i++ { + name := filepath.Join(dir, prefix+strconv.Itoa(rand.Intn(1000000000))+patternSuffix) + f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, perm) + if os.IsExist(err) { + continue + } + break + } + return } diff --git a/cmd/go/_internal_/robustio/robustio.go b/cmd/go/_internal_/robustio/robustio.go new file mode 100644 index 0000000..76e47ad --- /dev/null +++ b/cmd/go/_internal_/robustio/robustio.go @@ -0,0 +1,53 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package robustio wraps I/O functions that are prone to failure on Windows, +// transparently retrying errors up to an arbitrary timeout. +// +// Errors are classified heuristically and retries are bounded, so the functions +// in this package do not completely eliminate spurious errors. However, they do +// significantly reduce the rate of failure in practice. +// +// If so, the error will likely wrap one of: +// The functions in this package do not completely eliminate spurious errors, +// but substantially reduce their rate of occurrence in practice. +package robustio + +// Rename is like os.Rename, but on Windows retries errors that may occur if the +// file is concurrently read or overwritten. +// +// (See golang.org/issue/31247 and golang.org/issue/32188.) +func Rename(oldpath, newpath string) error { + return rename(oldpath, newpath) +} + +// ReadFile is like ioutil.ReadFile, but on Windows retries errors that may +// occur if the file is concurrently replaced. +// +// (See golang.org/issue/31247 and golang.org/issue/32188.) +func ReadFile(filename string) ([]byte, error) { + return readFile(filename) +} + +// RemoveAll is like os.RemoveAll, but on Windows retries errors that may occur +// if an executable file in the directory has recently been executed. +// +// (See golang.org/issue/19491.) +func RemoveAll(path string) error { + return removeAll(path) +} + +// IsEphemeralError reports whether err is one of the errors that the functions +// in this package attempt to mitigate. +// +// Errors considered ephemeral include: +// - syscall.ERROR_ACCESS_DENIED +// - syscall.ERROR_FILE_NOT_FOUND +// - internal/syscall/windows.ERROR_SHARING_VIOLATION +// +// This set may be expanded in the future; programs must not rely on the +// non-ephemerality of any given error. +func IsEphemeralError(err error) bool { + return isEphemeralError(err) +} diff --git a/cmd/go/_internal_/robustio/robustio_other.go b/cmd/go/_internal_/robustio/robustio_other.go new file mode 100644 index 0000000..907b556 --- /dev/null +++ b/cmd/go/_internal_/robustio/robustio_other.go @@ -0,0 +1,28 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !windows,!darwin + +package robustio + +import ( + "io/ioutil" + "os" +) + +func rename(oldpath, newpath string) error { + return os.Rename(oldpath, newpath) +} + +func readFile(filename string) ([]byte, error) { + return ioutil.ReadFile(filename) +} + +func removeAll(path string) error { + return os.RemoveAll(path) +} + +func isEphemeralError(err error) bool { + return false +} diff --git a/cmd/go/_internal_/search/search.go b/cmd/go/_internal_/search/search.go index d61f7c8..e1faa90 100644 --- a/cmd/go/_internal_/search/search.go +++ b/cmd/go/_internal_/search/search.go @@ -9,7 +9,6 @@ import ( "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" "fmt" "go/build" - "log" "os" "path" "path/filepath" @@ -19,25 +18,97 @@ import ( // A Match represents the result of matching a single package pattern. type Match struct { - Pattern string // the pattern itself - Literal bool // whether it is a literal (no wildcards) - Pkgs []string // matching packages (dirs or import paths) + pattern string // the pattern itself + Dirs []string // if the pattern is local, directories that potentially contain matching packages + Pkgs []string // matching packages (import paths) + Errs []error // errors matching the patterns to packages, NOT errors loading those packages + + // Errs may be non-empty even if len(Pkgs) > 0, indicating that some matching + // packages could be located but results may be incomplete. + // If len(Pkgs) == 0 && len(Errs) == 0, the pattern is well-formed but did not + // match any packages. } -// MatchPackages returns all the packages that can be found -// under the $GOPATH directories and $GOROOT matching pattern. -// The pattern is either "all" (all packages), "std" (standard packages), -// "cmd" (standard commands), or a path including "...". -func MatchPackages(pattern string) *Match { - m := &Match{ - Pattern: pattern, - Literal: false, +// NewMatch returns a Match describing the given pattern, +// without resolving its packages or errors. +func NewMatch(pattern string) *Match { + return &Match{pattern: pattern} +} + +// Pattern returns the pattern to be matched. +func (m *Match) Pattern() string { return m.pattern } + +// AddError appends a MatchError wrapping err to m.Errs. +func (m *Match) AddError(err error) { + m.Errs = append(m.Errs, &MatchError{Match: m, Err: err}) +} + +// Literal reports whether the pattern is free of wildcards and meta-patterns. +// +// A literal pattern must match at most one package. +func (m *Match) IsLiteral() bool { + return !strings.Contains(m.pattern, "...") && !m.IsMeta() +} + +// Local reports whether the pattern must be resolved from a specific root or +// directory, such as a filesystem path or a single module. +func (m *Match) IsLocal() bool { + return build.IsLocalImport(m.pattern) || filepath.IsAbs(m.pattern) +} + +// Meta reports whether the pattern is a “meta-package” keyword that represents +// multiple packages, such as "std", "cmd", or "all". +func (m *Match) IsMeta() bool { + return IsMetaPackage(m.pattern) +} + +// IsMetaPackage checks if name is a reserved package name that expands to multiple packages. +func IsMetaPackage(name string) bool { + return name == "std" || name == "cmd" || name == "all" +} + +// A MatchError indicates an error that occurred while attempting to match a +// pattern. +type MatchError struct { + Match *Match + Err error +} + +func (e *MatchError) Error() string { + if e.Match.IsLiteral() { + return fmt.Sprintf("%s: %v", e.Match.Pattern(), e.Err) + } + return fmt.Sprintf("pattern %s: %v", e.Match.Pattern(), e.Err) +} + +func (e *MatchError) Unwrap() error { + return e.Err +} + +// MatchPackages sets m.Pkgs to a non-nil slice containing all the packages that +// can be found under the $GOPATH directories and $GOROOT that match the +// pattern. The pattern must be either "all" (all packages), "std" (standard +// packages), "cmd" (standard commands), or a path including "...". +// +// If any errors may have caused the set of packages to be incomplete, +// MatchPackages appends those errors to m.Errs. +func (m *Match) MatchPackages() { + m.Pkgs = []string{} + if m.IsLocal() { + m.AddError(fmt.Errorf("internal error: MatchPackages: %s is not a valid package pattern", m.pattern)) + return + } + + if m.IsLiteral() { + m.Pkgs = []string{m.pattern} + return } + match := func(string) bool { return true } treeCanMatch := func(string) bool { return true } - if !IsMetaPackage(pattern) { - match = MatchPattern(pattern) - treeCanMatch = TreeCanMatchPattern(pattern) + if !m.IsMeta() { + match = MatchPattern(m.pattern) + treeCanMatch = TreeCanMatchPattern(m.pattern) } have := map[string]bool{ @@ -48,17 +119,20 @@ func MatchPackages(pattern string) *Match { } for _, src := range cfg.BuildContext.SrcDirs() { - if (pattern == "std" || pattern == "cmd") && src != cfg.GOROOTsrc { + if (m.pattern == "std" || m.pattern == "cmd") && src != cfg.GOROOTsrc { continue } src = filepath.Clean(src) + string(filepath.Separator) root := src - if pattern == "cmd" { + if m.pattern == "cmd" { root += "cmd" + string(filepath.Separator) } - filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { - if err != nil || path == src { - return nil + err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { + if err != nil { + return err // Likely a permission error, which could interfere with matching. + } + if path == src { + return nil // GOROOT/src and GOPATH/src cannot contain packages. } want := true @@ -69,7 +143,7 @@ func MatchPackages(pattern string) *Match { } name := filepath.ToSlash(path[len(src):]) - if pattern == "std" && (!IsStandardImportPath(name) || name == "cmd") { + if m.pattern == "std" && (!IsStandardImportPath(name) || name == "cmd") { // The name "std" is only the standard library. // If the name is cmd, it's the root of the command tree. want = false @@ -100,23 +174,30 @@ func MatchPackages(pattern string) *Match { pkg, err := cfg.BuildContext.ImportDir(path, 0) if err != nil { if _, noGo := err.(*build.NoGoError); noGo { + // The package does not actually exist, so record neither the package + // nor the error. return nil } + // There was an error importing path, but not matching it, + // which is all that Match promises to do. + // Ignore the import error. } // If we are expanding "cmd", skip main // packages under cmd/vendor. At least as of // March, 2017, there is one there for the // vendored pprof tool. - if pattern == "cmd" && strings.HasPrefix(pkg.ImportPath, "cmd/vendor") && pkg.Name == "main" { + if m.pattern == "cmd" && pkg != nil && strings.HasPrefix(pkg.ImportPath, "cmd/vendor") && pkg.Name == "main" { return nil } m.Pkgs = append(m.Pkgs, name) return nil }) + if err != nil { + m.AddError(err) + } } - return m } var modRoot string @@ -125,46 +206,68 @@ func SetModRoot(dir string) { modRoot = dir } -// MatchPackagesInFS is like allPackages but is passed a pattern -// beginning ./ or ../, meaning it should scan the tree rooted -// at the given directory. There are ... in the pattern too. -// (See go help packages for pattern syntax.) -func MatchPackagesInFS(pattern string) *Match { - m := &Match{ - Pattern: pattern, - Literal: false, +// MatchDirs sets m.Dirs to a non-nil slice containing all directories that +// potentially match a local pattern. The pattern must begin with an absolute +// path, or "./", or "../". On Windows, the pattern may use slash or backslash +// separators or a mix of both. +// +// If any errors may have caused the set of directories to be incomplete, +// MatchDirs appends those errors to m.Errs. +func (m *Match) MatchDirs() { + m.Dirs = []string{} + if !m.IsLocal() { + m.AddError(fmt.Errorf("internal error: MatchDirs: %s is not a valid filesystem pattern", m.pattern)) + return + } + + if m.IsLiteral() { + m.Dirs = []string{m.pattern} + return + } + + // Clean the path and create a matching predicate. + // filepath.Clean removes "./" prefixes (and ".\" on Windows). We need to + // preserve these, since they are meaningful in MatchPattern and in + // returned import paths. + cleanPattern := filepath.Clean(m.pattern) + isLocal := strings.HasPrefix(m.pattern, "./") || (os.PathSeparator == '\\' && strings.HasPrefix(m.pattern, `.\`)) + prefix := "" + if cleanPattern != "." && isLocal { + prefix = "./" + cleanPattern = "." + string(os.PathSeparator) + cleanPattern } + slashPattern := filepath.ToSlash(cleanPattern) + match := MatchPattern(slashPattern) // Find directory to begin the scan. // Could be smarter but this one optimization // is enough for now, since ... is usually at the // end of a path. - i := strings.Index(pattern, "...") - dir, _ := path.Split(pattern[:i]) + i := strings.Index(cleanPattern, "...") + dir, _ := filepath.Split(cleanPattern[:i]) // pattern begins with ./ or ../. // path.Clean will discard the ./ but not the ../. // We need to preserve the ./ for pattern matching // and in the returned import paths. - prefix := "" - if strings.HasPrefix(pattern, "./") { - prefix = "./" - } - match := MatchPattern(pattern) if modRoot != "" { abs, err := filepath.Abs(dir) if err != nil { - base.Fatalf("go: %v", err) + m.AddError(err) + return } if !hasFilepathPrefix(abs, modRoot) { - base.Fatalf("go: pattern %s refers to dir %s, outside module root %s", pattern, abs, modRoot) - return nil + m.AddError(fmt.Errorf("directory %s is outside module root (%s)", abs, modRoot)) + return } } - filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error { - if err != nil || !fi.IsDir() { + err := filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error { + if err != nil { + return err // Likely a permission error, which could interfere with matching. + } + if !fi.IsDir() { return nil } top := false @@ -190,7 +293,7 @@ func MatchPackagesInFS(pattern string) *Match { if !top && cfg.ModulesEnabled { // Ignore other modules found in subdirectories. - if _, err := os.Stat(filepath.Join(path, "go.mod")); err == nil { + if fi, err := os.Stat(filepath.Join(path, "go.mod")); err == nil && !fi.IsDir() { return filepath.SkipDir } } @@ -207,15 +310,21 @@ func MatchPackagesInFS(pattern string) *Match { // behavior means people miss serious mistakes. // See golang.org/issue/11407. if p, err := cfg.BuildContext.ImportDir(path, 0); err != nil && (p == nil || len(p.InvalidGoFiles) == 0) { - if _, noGo := err.(*build.NoGoError); !noGo { - log.Print(err) + if _, noGo := err.(*build.NoGoError); noGo { + // The package does not actually exist, so record neither the package + // nor the error. + return nil } - return nil + // There was an error importing path, but not matching it, + // which is all that Match promises to do. + // Ignore the import error. } - m.Pkgs = append(m.Pkgs, name) + m.Dirs = append(m.Dirs, name) return nil }) - return m + if err != nil { + m.AddError(err) + } } // TreeCanMatchPattern(pattern)(name) reports whether @@ -241,7 +350,7 @@ func TreeCanMatchPattern(pattern string) func(name string) bool { // // First, /... at the end of the pattern can match an empty string, // so that net/... matches both net and packages in its subdirectories, like net/http. -// Second, any slash-separted pattern element containing a wildcard never +// Second, any slash-separated pattern element containing a wildcard never // participates in a match of the "vendor" element in the path of a vendored // package, so that ./... does not match packages in subdirectories of // ./vendor or ./mycode/vendor, but ./vendor/... and ./mycode/vendor/... do. @@ -305,8 +414,8 @@ func replaceVendor(x, repl string) string { // WarnUnmatched warns about patterns that didn't match any packages. func WarnUnmatched(matches []*Match) { for _, m := range matches { - if len(m.Pkgs) == 0 { - fmt.Fprintf(os.Stderr, "go: warning: %q matched no packages\n", m.Pattern) + if len(m.Pkgs) == 0 && len(m.Errs) == 0 { + fmt.Fprintf(os.Stderr, "go: warning: %q matched no packages\n", m.pattern) } } } @@ -323,58 +432,81 @@ func ImportPaths(patterns []string) []*Match { func ImportPathsQuiet(patterns []string) []*Match { var out []*Match for _, a := range CleanPatterns(patterns) { - if IsMetaPackage(a) { - out = append(out, MatchPackages(a)) - continue - } - if strings.Contains(a, "...") { - if build.IsLocalImport(a) { - out = append(out, MatchPackagesInFS(a)) - } else { - out = append(out, MatchPackages(a)) + m := NewMatch(a) + if m.IsLocal() { + m.MatchDirs() + + // Change the file import path to a regular import path if the package + // is in GOPATH or GOROOT. We don't report errors here; LoadImport + // (or something similar) will report them later. + m.Pkgs = make([]string, len(m.Dirs)) + for i, dir := range m.Dirs { + absDir := dir + if !filepath.IsAbs(dir) { + absDir = filepath.Join(base.Cwd, dir) + } + if bp, _ := cfg.BuildContext.ImportDir(absDir, build.FindOnly); bp.ImportPath != "" && bp.ImportPath != "." { + m.Pkgs[i] = bp.ImportPath + } else { + m.Pkgs[i] = dir + } } - continue + } else { + m.MatchPackages() } - out = append(out, &Match{Pattern: a, Literal: true, Pkgs: []string{a}}) + + out = append(out, m) } return out } -// CleanPatterns returns the patterns to use for the given -// command line. It canonicalizes the patterns but does not -// evaluate any matches. +// CleanPatterns returns the patterns to use for the given command line. It +// canonicalizes the patterns but does not evaluate any matches. For patterns +// that are not local or absolute paths, it preserves text after '@' to avoid +// modifying version queries. func CleanPatterns(patterns []string) []string { if len(patterns) == 0 { return []string{"."} } var out []string for _, a := range patterns { - // Arguments are supposed to be import paths, but - // as a courtesy to Windows developers, rewrite \ to / - // in command-line arguments. Handles .\... and so on. - if filepath.Separator == '\\' { - a = strings.ReplaceAll(a, `\`, `/`) + var p, v string + if build.IsLocalImport(a) || filepath.IsAbs(a) { + p = a + } else if i := strings.IndexByte(a, '@'); i < 0 { + p = a + } else { + p = a[:i] + v = a[i:] } - // Put argument in canonical form, but preserve leading ./. - if strings.HasPrefix(a, "./") { - a = "./" + path.Clean(a) - if a == "./." { - a = "." - } + // Arguments may be either file paths or import paths. + // As a courtesy to Windows developers, rewrite \ to / + // in arguments that look like import paths. + // Don't replace slashes in absolute paths. + if filepath.IsAbs(p) { + p = filepath.Clean(p) } else { - a = path.Clean(a) + if filepath.Separator == '\\' { + p = strings.ReplaceAll(p, `\`, `/`) + } + + // Put argument in canonical form, but preserve leading ./. + if strings.HasPrefix(p, "./") { + p = "./" + path.Clean(p) + if p == "./." { + p = "." + } + } else { + p = path.Clean(p) + } } - out = append(out, a) + + out = append(out, p+v) } return out } -// IsMetaPackage checks if name is a reserved package name that expands to multiple packages. -func IsMetaPackage(name string) bool { - return name == "std" || name == "cmd" || name == "all" -} - // hasPathPrefix reports whether the path s begins with the // elements in prefix. func hasPathPrefix(s, prefix string) bool { diff --git a/cmd/go/_internal_/semver/semver.go b/cmd/go/_internal_/semver/semver.go deleted file mode 100644 index 9785e04..0000000 --- a/cmd/go/_internal_/semver/semver.go +++ /dev/null @@ -1,388 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package semver implements comparison of semantic version strings. -// In this package, semantic version strings must begin with a leading "v", -// as in "v1.0.0". -// -// The general form of a semantic version string accepted by this package is -// -// vMAJOR[.MINOR[.PATCH[-PRERELEASE][+BUILD]]] -// -// where square brackets indicate optional parts of the syntax; -// MAJOR, MINOR, and PATCH are decimal integers without extra leading zeros; -// PRERELEASE and BUILD are each a series of non-empty dot-separated identifiers -// using only alphanumeric characters and hyphens; and -// all-numeric PRERELEASE identifiers must not have leading zeros. -// -// This package follows Semantic Versioning 2.0.0 (see semver.org) -// with two exceptions. First, it requires the "v" prefix. Second, it recognizes -// vMAJOR and vMAJOR.MINOR (with no prerelease or build suffixes) -// as shorthands for vMAJOR.0.0 and vMAJOR.MINOR.0. -package semver - -// parsed returns the parsed form of a semantic version string. -type parsed struct { - major string - minor string - patch string - short string - prerelease string - build string - err string -} - -// IsValid reports whether v is a valid semantic version string. -func IsValid(v string) bool { - _, ok := parse(v) - return ok -} - -// Canonical returns the canonical formatting of the semantic version v. -// It fills in any missing .MINOR or .PATCH and discards build metadata. -// Two semantic versions compare equal only if their canonical formattings -// are identical strings. -// The canonical invalid semantic version is the empty string. -func Canonical(v string) string { - p, ok := parse(v) - if !ok { - return "" - } - if p.build != "" { - return v[:len(v)-len(p.build)] - } - if p.short != "" { - return v + p.short - } - return v -} - -// Major returns the major version prefix of the semantic version v. -// For example, Major("v2.1.0") == "v2". -// If v is an invalid semantic version string, Major returns the empty string. -func Major(v string) string { - pv, ok := parse(v) - if !ok { - return "" - } - return v[:1+len(pv.major)] -} - -// MajorMinor returns the major.minor version prefix of the semantic version v. -// For example, MajorMinor("v2.1.0") == "v2.1". -// If v is an invalid semantic version string, MajorMinor returns the empty string. -func MajorMinor(v string) string { - pv, ok := parse(v) - if !ok { - return "" - } - i := 1 + len(pv.major) - if j := i + 1 + len(pv.minor); j <= len(v) && v[i] == '.' && v[i+1:j] == pv.minor { - return v[:j] - } - return v[:i] + "." + pv.minor -} - -// Prerelease returns the prerelease suffix of the semantic version v. -// For example, Prerelease("v2.1.0-pre+meta") == "-pre". -// If v is an invalid semantic version string, Prerelease returns the empty string. -func Prerelease(v string) string { - pv, ok := parse(v) - if !ok { - return "" - } - return pv.prerelease -} - -// Build returns the build suffix of the semantic version v. -// For example, Build("v2.1.0+meta") == "+meta". -// If v is an invalid semantic version string, Build returns the empty string. -func Build(v string) string { - pv, ok := parse(v) - if !ok { - return "" - } - return pv.build -} - -// Compare returns an integer comparing two versions according to -// according to semantic version precedence. -// The result will be 0 if v == w, -1 if v < w, or +1 if v > w. -// -// An invalid semantic version string is considered less than a valid one. -// All invalid semantic version strings compare equal to each other. -func Compare(v, w string) int { - pv, ok1 := parse(v) - pw, ok2 := parse(w) - if !ok1 && !ok2 { - return 0 - } - if !ok1 { - return -1 - } - if !ok2 { - return +1 - } - if c := compareInt(pv.major, pw.major); c != 0 { - return c - } - if c := compareInt(pv.minor, pw.minor); c != 0 { - return c - } - if c := compareInt(pv.patch, pw.patch); c != 0 { - return c - } - return comparePrerelease(pv.prerelease, pw.prerelease) -} - -// Max canonicalizes its arguments and then returns the version string -// that compares greater. -func Max(v, w string) string { - v = Canonical(v) - w = Canonical(w) - if Compare(v, w) > 0 { - return v - } - return w -} - -func parse(v string) (p parsed, ok bool) { - if v == "" || v[0] != 'v' { - p.err = "missing v prefix" - return - } - p.major, v, ok = parseInt(v[1:]) - if !ok { - p.err = "bad major version" - return - } - if v == "" { - p.minor = "0" - p.patch = "0" - p.short = ".0.0" - return - } - if v[0] != '.' { - p.err = "bad minor prefix" - ok = false - return - } - p.minor, v, ok = parseInt(v[1:]) - if !ok { - p.err = "bad minor version" - return - } - if v == "" { - p.patch = "0" - p.short = ".0" - return - } - if v[0] != '.' { - p.err = "bad patch prefix" - ok = false - return - } - p.patch, v, ok = parseInt(v[1:]) - if !ok { - p.err = "bad patch version" - return - } - if len(v) > 0 && v[0] == '-' { - p.prerelease, v, ok = parsePrerelease(v) - if !ok { - p.err = "bad prerelease" - return - } - } - if len(v) > 0 && v[0] == '+' { - p.build, v, ok = parseBuild(v) - if !ok { - p.err = "bad build" - return - } - } - if v != "" { - p.err = "junk on end" - ok = false - return - } - ok = true - return -} - -func parseInt(v string) (t, rest string, ok bool) { - if v == "" { - return - } - if v[0] < '0' || '9' < v[0] { - return - } - i := 1 - for i < len(v) && '0' <= v[i] && v[i] <= '9' { - i++ - } - if v[0] == '0' && i != 1 { - return - } - return v[:i], v[i:], true -} - -func parsePrerelease(v string) (t, rest string, ok bool) { - // "A pre-release version MAY be denoted by appending a hyphen and - // a series of dot separated identifiers immediately following the patch version. - // Identifiers MUST comprise only ASCII alphanumerics and hyphen [0-9A-Za-z-]. - // Identifiers MUST NOT be empty. Numeric identifiers MUST NOT include leading zeroes." - if v == "" || v[0] != '-' { - return - } - i := 1 - start := 1 - for i < len(v) && v[i] != '+' { - if !isIdentChar(v[i]) && v[i] != '.' { - return - } - if v[i] == '.' { - if start == i || isBadNum(v[start:i]) { - return - } - start = i + 1 - } - i++ - } - if start == i || isBadNum(v[start:i]) { - return - } - return v[:i], v[i:], true -} - -func parseBuild(v string) (t, rest string, ok bool) { - if v == "" || v[0] != '+' { - return - } - i := 1 - start := 1 - for i < len(v) { - if !isIdentChar(v[i]) && v[i] != '.' { - return - } - if v[i] == '.' { - if start == i { - return - } - start = i + 1 - } - i++ - } - if start == i { - return - } - return v[:i], v[i:], true -} - -func isIdentChar(c byte) bool { - return 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' || c == '-' -} - -func isBadNum(v string) bool { - i := 0 - for i < len(v) && '0' <= v[i] && v[i] <= '9' { - i++ - } - return i == len(v) && i > 1 && v[0] == '0' -} - -func isNum(v string) bool { - i := 0 - for i < len(v) && '0' <= v[i] && v[i] <= '9' { - i++ - } - return i == len(v) -} - -func compareInt(x, y string) int { - if x == y { - return 0 - } - if len(x) < len(y) { - return -1 - } - if len(x) > len(y) { - return +1 - } - if x < y { - return -1 - } else { - return +1 - } -} - -func comparePrerelease(x, y string) int { - // "When major, minor, and patch are equal, a pre-release version has - // lower precedence than a normal version. - // Example: 1.0.0-alpha < 1.0.0. - // Precedence for two pre-release versions with the same major, minor, - // and patch version MUST be determined by comparing each dot separated - // identifier from left to right until a difference is found as follows: - // identifiers consisting of only digits are compared numerically and - // identifiers with letters or hyphens are compared lexically in ASCII - // sort order. Numeric identifiers always have lower precedence than - // non-numeric identifiers. A larger set of pre-release fields has a - // higher precedence than a smaller set, if all of the preceding - // identifiers are equal. - // Example: 1.0.0-alpha < 1.0.0-alpha.1 < 1.0.0-alpha.beta < - // 1.0.0-beta < 1.0.0-beta.2 < 1.0.0-beta.11 < 1.0.0-rc.1 < 1.0.0." - if x == y { - return 0 - } - if x == "" { - return +1 - } - if y == "" { - return -1 - } - for x != "" && y != "" { - x = x[1:] // skip - or . - y = y[1:] // skip - or . - var dx, dy string - dx, x = nextIdent(x) - dy, y = nextIdent(y) - if dx != dy { - ix := isNum(dx) - iy := isNum(dy) - if ix != iy { - if ix { - return -1 - } else { - return +1 - } - } - if ix { - if len(dx) < len(dy) { - return -1 - } - if len(dx) > len(dy) { - return +1 - } - } - if dx < dy { - return -1 - } else { - return +1 - } - } - } - if x == "" { - return -1 - } else { - return +1 - } -} - -func nextIdent(x string) (dx, rest string) { - i := 0 - for i < len(x) && x[i] != '.' { - i++ - } - return x[:i], x[i:] -} diff --git a/cmd/go/_internal_/str/path.go b/cmd/go/_internal_/str/path.go index a9b4d75..95d91a3 100644 --- a/cmd/go/_internal_/str/path.go +++ b/cmd/go/_internal_/str/path.go @@ -5,11 +5,12 @@ package str import ( + "path" "path/filepath" "strings" ) -// HasPath reports whether the slash-separated path s +// HasPathPrefix reports whether the slash-separated path s // begins with the elements in prefix. func HasPathPrefix(s, prefix string) bool { if len(s) == len(prefix) { @@ -49,3 +50,47 @@ func HasFilePathPrefix(s, prefix string) bool { return s[len(prefix)] == filepath.Separator && s[:len(prefix)] == prefix } } + +// GlobsMatchPath reports whether any path prefix of target +// matches one of the glob patterns (as defined by path.Match) +// in the comma-separated globs list. +// It ignores any empty or malformed patterns in the list. +func GlobsMatchPath(globs, target string) bool { + for globs != "" { + // Extract next non-empty glob in comma-separated list. + var glob string + if i := strings.Index(globs, ","); i >= 0 { + glob, globs = globs[:i], globs[i+1:] + } else { + glob, globs = globs, "" + } + if glob == "" { + continue + } + + // A glob with N+1 path elements (N slashes) needs to be matched + // against the first N+1 path elements of target, + // which end just before the N+1'th slash. + n := strings.Count(glob, "/") + prefix := target + // Walk target, counting slashes, truncating at the N+1'th slash. + for i := 0; i < len(target); i++ { + if target[i] == '/' { + if n == 0 { + prefix = target[:i] + break + } + n-- + } + } + if n > 0 { + // Not enough prefix elements. + continue + } + matched, _ := path.Match(glob, prefix) + if matched { + return true + } + } + return false +} diff --git a/cmd/go/_internal_/web/api.go b/cmd/go/_internal_/web/api.go new file mode 100644 index 0000000..f6c9aae --- /dev/null +++ b/cmd/go/_internal_/web/api.go @@ -0,0 +1,237 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package web defines minimal helper routines for accessing HTTP/HTTPS +// resources without requiring external dependencies on the net package. +// +// If the cmd_go_bootstrap build tag is present, web avoids the use of the net +// package and returns errors for all network operations. +package web + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "net/url" + "os" + "strings" + "unicode" + "unicode/utf8" +) + +// SecurityMode specifies whether a function should make network +// calls using insecure transports (eg, plain text HTTP). +// The zero value is "secure". +type SecurityMode int + +const ( + SecureOnly SecurityMode = iota // Reject plain HTTP; validate HTTPS. + DefaultSecurity // Allow plain HTTP if explicit; validate HTTPS. + Insecure // Allow plain HTTP if not explicitly HTTPS; skip HTTPS validation. +) + +// An HTTPError describes an HTTP error response (non-200 result). +type HTTPError struct { + URL string // redacted + Status string + StatusCode int + Err error // underlying error, if known + Detail string // limited to maxErrorDetailLines and maxErrorDetailBytes +} + +const ( + maxErrorDetailLines = 8 + maxErrorDetailBytes = maxErrorDetailLines * 81 +) + +func (e *HTTPError) Error() string { + if e.Detail != "" { + detailSep := " " + if strings.ContainsRune(e.Detail, '\n') { + detailSep = "\n\t" + } + return fmt.Sprintf("reading %s: %v\n\tserver response:%s%s", e.URL, e.Status, detailSep, e.Detail) + } + + if err := e.Err; err != nil { + if pErr, ok := e.Err.(*os.PathError); ok && strings.HasSuffix(e.URL, pErr.Path) { + // Remove the redundant copy of the path. + err = pErr.Err + } + return fmt.Sprintf("reading %s: %v", e.URL, err) + } + + return fmt.Sprintf("reading %s: %v", e.URL, e.Status) +} + +func (e *HTTPError) Is(target error) bool { + return target == os.ErrNotExist && (e.StatusCode == 404 || e.StatusCode == 410) +} + +func (e *HTTPError) Unwrap() error { + return e.Err +} + +// GetBytes returns the body of the requested resource, or an error if the +// response status was not http.StatusOK. +// +// GetBytes is a convenience wrapper around Get and Response.Err. +func GetBytes(u *url.URL) ([]byte, error) { + resp, err := Get(DefaultSecurity, u) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if err := resp.Err(); err != nil { + return nil, err + } + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading %s: %v", u.Redacted(), err) + } + return b, nil +} + +type Response struct { + URL string // redacted + Status string + StatusCode int + Header map[string][]string + Body io.ReadCloser // Either the original body or &errorDetail. + + fileErr error + errorDetail errorDetailBuffer +} + +// Err returns an *HTTPError corresponding to the response r. +// If the response r has StatusCode 200 or 0 (unset), Err returns nil. +// Otherwise, Err may read from r.Body in order to extract relevant error detail. +func (r *Response) Err() error { + if r.StatusCode == 200 || r.StatusCode == 0 { + return nil + } + + return &HTTPError{ + URL: r.URL, + Status: r.Status, + StatusCode: r.StatusCode, + Err: r.fileErr, + Detail: r.formatErrorDetail(), + } +} + +// formatErrorDetail converts r.errorDetail (a prefix of the output of r.Body) +// into a short, tab-indented summary. +func (r *Response) formatErrorDetail() string { + if r.Body != &r.errorDetail { + return "" // Error detail collection not enabled. + } + + // Ensure that r.errorDetail has been populated. + _, _ = io.Copy(ioutil.Discard, r.Body) + + s := r.errorDetail.buf.String() + if !utf8.ValidString(s) { + return "" // Don't try to recover non-UTF-8 error messages. + } + for _, r := range s { + if !unicode.IsGraphic(r) && !unicode.IsSpace(r) { + return "" // Don't let the server do any funny business with the user's terminal. + } + } + + var detail strings.Builder + for i, line := range strings.Split(s, "\n") { + if strings.TrimSpace(line) == "" { + break // Stop at the first blank line. + } + if i > 0 { + detail.WriteString("\n\t") + } + if i >= maxErrorDetailLines { + detail.WriteString("[Truncated: too many lines.]") + break + } + if detail.Len()+len(line) > maxErrorDetailBytes { + detail.WriteString("[Truncated: too long.]") + break + } + detail.WriteString(line) + } + + return detail.String() +} + +// Get returns the body of the HTTP or HTTPS resource specified at the given URL. +// +// If the URL does not include an explicit scheme, Get first tries "https". +// If the server does not respond under that scheme and the security mode is +// Insecure, Get then tries "http". +// The URL included in the response indicates which scheme was actually used, +// and it is a redacted URL suitable for use in error messages. +// +// For the "https" scheme only, credentials are attached using the +// cmd/go/internal/auth package. If the URL itself includes a username and +// password, it will not be attempted under the "http" scheme unless the +// security mode is Insecure. +// +// Get returns a non-nil error only if the request did not receive a response +// under any applicable scheme. (A non-2xx response does not cause an error.) +func Get(security SecurityMode, u *url.URL) (*Response, error) { + return get(security, u) +} + +// OpenBrowser attempts to open the requested URL in a web browser. +func OpenBrowser(url string) (opened bool) { + return openBrowser(url) +} + +// Join returns the result of adding the slash-separated +// path elements to the end of u's path. +func Join(u *url.URL, path string) *url.URL { + j := *u + if path == "" { + return &j + } + j.Path = strings.TrimSuffix(u.Path, "/") + "/" + strings.TrimPrefix(path, "/") + j.RawPath = strings.TrimSuffix(u.RawPath, "/") + "/" + strings.TrimPrefix(path, "/") + return &j +} + +// An errorDetailBuffer is an io.ReadCloser that copies up to +// maxErrorDetailLines into a buffer for later inspection. +type errorDetailBuffer struct { + r io.ReadCloser + buf strings.Builder + bufLines int +} + +func (b *errorDetailBuffer) Close() error { + return b.r.Close() +} + +func (b *errorDetailBuffer) Read(p []byte) (n int, err error) { + n, err = b.r.Read(p) + + // Copy the first maxErrorDetailLines+1 lines into b.buf, + // discarding any further lines. + // + // Note that the read may begin or end in the middle of a UTF-8 character, + // so don't try to do anything fancy with characters that encode to larger + // than one byte. + if b.bufLines <= maxErrorDetailLines { + for _, line := range bytes.SplitAfterN(p[:n], []byte("\n"), maxErrorDetailLines-b.bufLines) { + b.buf.Write(line) + if len(line) > 0 && line[len(line)-1] == '\n' { + b.bufLines++ + if b.bufLines > maxErrorDetailLines { + break + } + } + } + } + + return n, err +} diff --git a/cmd/go/_internal_/web/http.go b/cmd/go/_internal_/web/http.go index 35aac99..0f5525b 100644 --- a/cmd/go/_internal_/web/http.go +++ b/cmd/go/_internal_/web/http.go @@ -13,22 +13,20 @@ package web import ( "crypto/tls" + "errors" "fmt" - "io" - "io/ioutil" - "log" + "mime" "net/http" - "net/url" + urlpkg "net/url" + "os" + "strings" "time" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/auth" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" "github.com/dependabot/gomodules-extracted/cmd/_internal_/browser" ) -// httpClient is the default HTTP client, but a variable so it can be -// changed by tests, without modifying http.DefaultClient. -var httpClient = http.DefaultClient - // impatientInsecureHTTPClient is used in -insecure mode, // when we're connecting to https servers that might not be there // or might be using self-signed certificates. @@ -42,81 +40,203 @@ var impatientInsecureHTTPClient = &http.Client{ }, } -type HTTPError struct { - status string - StatusCode int - url string -} +// securityPreservingHTTPClient is like the default HTTP client, but rejects +// redirects to plain-HTTP URLs if the original URL was secure. +var securityPreservingHTTPClient = &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) > 0 && via[0].URL.Scheme == "https" && req.URL.Scheme != "https" { + lastHop := via[len(via)-1].URL + return fmt.Errorf("redirected from secure URL %s to insecure URL %s", lastHop, req.URL) + } -func (e *HTTPError) Error() string { - return fmt.Sprintf("%s: %s", e.url, e.status) + // Go's http.DefaultClient allows 10 redirects before returning an error. + // The securityPreservingHTTPClient also uses this default policy to avoid + // Go command hangs. + if len(via) >= 10 { + return errors.New("stopped after 10 redirects") + } + return nil + }, } -// Get returns the data from an HTTP GET request for the given URL. -func Get(url string) ([]byte, error) { - resp, err := httpClient.Get(url) - if err != nil { - return nil, err - } - defer resp.Body.Close() - if resp.StatusCode != 200 { - err := &HTTPError{status: resp.Status, StatusCode: resp.StatusCode, url: url} +func get(security SecurityMode, url *urlpkg.URL) (*Response, error) { + start := time.Now() - return nil, err + if url.Scheme == "file" { + return getFile(url) } - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("%s: %v", url, err) + + if os.Getenv("TESTGOPROXY404") == "1" && url.Host == "proxy.golang.org" { + res := &Response{ + URL: url.Redacted(), + Status: "404 testing", + StatusCode: 404, + Header: make(map[string][]string), + Body: http.NoBody, + } + if cfg.BuildX { + fmt.Fprintf(os.Stderr, "# get %s: %v (%.3fs)\n", url.Redacted(), res.Status, time.Since(start).Seconds()) + } + return res, nil } - return b, nil -} -// GetMaybeInsecure returns the body of either the importPath's -// https resource or, if unavailable and permitted by the security mode, the http resource. -func GetMaybeInsecure(importPath string, security SecurityMode) (urlStr string, body io.ReadCloser, err error) { - fetch := func(scheme string) (urlStr string, res *http.Response, err error) { - u, err := url.Parse(scheme + "://" + importPath) + fetch := func(url *urlpkg.URL) (*urlpkg.URL, *http.Response, error) { + // Note: The -v build flag does not mean "print logging information", + // despite its historical misuse for this in GOPATH-based go get. + // We print extra logging in -x mode instead, which traces what + // commands are executed. + if cfg.BuildX { + fmt.Fprintf(os.Stderr, "# get %s\n", url.Redacted()) + } + + req, err := http.NewRequest("GET", url.String(), nil) if err != nil { - return "", nil, err + return nil, nil, err } - u.RawQuery = "go-get=1" - urlStr = u.String() - if cfg.BuildV { - log.Printf("Fetching %s", urlStr) + if url.Scheme == "https" { + auth.AddCredentials(req) } - if security == Insecure && scheme == "https" { // fail earlier - res, err = impatientInsecureHTTPClient.Get(urlStr) + + var res *http.Response + if security == Insecure && url.Scheme == "https" { // fail earlier + res, err = impatientInsecureHTTPClient.Do(req) } else { - res, err = httpClient.Get(urlStr) + res, err = securityPreservingHTTPClient.Do(req) } - return + return url, res, err } - closeBody := func(res *http.Response) { - if res != nil { - res.Body.Close() + + var ( + fetched *urlpkg.URL + res *http.Response + err error + ) + if url.Scheme == "" || url.Scheme == "https" { + secure := new(urlpkg.URL) + *secure = *url + secure.Scheme = "https" + + fetched, res, err = fetch(secure) + if err != nil { + if cfg.BuildX { + fmt.Fprintf(os.Stderr, "# get %s: %v\n", secure.Redacted(), err) + } + if security != Insecure || url.Scheme == "https" { + // HTTPS failed, and we can't fall back to plain HTTP. + // Report the error from the HTTPS attempt. + return nil, err + } } } - urlStr, res, err := fetch("https") - if err != nil { - if cfg.BuildV { - log.Printf("https fetch failed: %v", err) + + if res == nil { + switch url.Scheme { + case "http": + if security == SecureOnly { + if cfg.BuildX { + fmt.Fprintf(os.Stderr, "# get %s: insecure\n", url.Redacted()) + } + return nil, fmt.Errorf("insecure URL: %s", url.Redacted()) + } + case "": + if security != Insecure { + panic("should have returned after HTTPS failure") + } + default: + if cfg.BuildX { + fmt.Fprintf(os.Stderr, "# get %s: unsupported\n", url.Redacted()) + } + return nil, fmt.Errorf("unsupported scheme: %s", url.Redacted()) } - if security == Insecure { - closeBody(res) - urlStr, res, err = fetch("http") + + insecure := new(urlpkg.URL) + *insecure = *url + insecure.Scheme = "http" + if insecure.User != nil && security != Insecure { + if cfg.BuildX { + fmt.Fprintf(os.Stderr, "# get %s: insecure credentials\n", insecure.Redacted()) + } + return nil, fmt.Errorf("refusing to pass credentials to insecure URL: %s", insecure.Redacted()) + } + + fetched, res, err = fetch(insecure) + if err != nil { + if cfg.BuildX { + fmt.Fprintf(os.Stderr, "# get %s: %v\n", insecure.Redacted(), err) + } + // HTTP failed, and we already tried HTTPS if applicable. + // Report the error from the HTTP attempt. + return nil, err } } - if err != nil { - closeBody(res) - return "", nil, err - } + // Note: accepting a non-200 OK here, so people can serve a // meta import in their http 404 page. - if cfg.BuildV { - log.Printf("Parsing meta tags from %s (status code %d)", urlStr, res.StatusCode) + if cfg.BuildX { + fmt.Fprintf(os.Stderr, "# get %s: %v (%.3fs)\n", fetched.Redacted(), res.Status, time.Since(start).Seconds()) + } + + r := &Response{ + URL: fetched.Redacted(), + Status: res.Status, + StatusCode: res.StatusCode, + Header: map[string][]string(res.Header), + Body: res.Body, + } + + if res.StatusCode != http.StatusOK { + contentType := res.Header.Get("Content-Type") + if mediaType, params, _ := mime.ParseMediaType(contentType); mediaType == "text/plain" { + switch charset := strings.ToLower(params["charset"]); charset { + case "us-ascii", "utf-8", "": + // Body claims to be plain text in UTF-8 or a subset thereof. + // Try to extract a useful error message from it. + r.errorDetail.r = res.Body + r.Body = &r.errorDetail + } + } + } + + return r, nil +} + +func getFile(u *urlpkg.URL) (*Response, error) { + path, err := urlToFilePath(u) + if err != nil { + return nil, err + } + f, err := os.Open(path) + + if os.IsNotExist(err) { + return &Response{ + URL: u.Redacted(), + Status: http.StatusText(http.StatusNotFound), + StatusCode: http.StatusNotFound, + Body: http.NoBody, + fileErr: err, + }, nil + } + + if os.IsPermission(err) { + return &Response{ + URL: u.Redacted(), + Status: http.StatusText(http.StatusForbidden), + StatusCode: http.StatusForbidden, + Body: http.NoBody, + fileErr: err, + }, nil + } + + if err != nil { + return nil, err } - return urlStr, res.Body, nil + + return &Response{ + URL: u.Redacted(), + Status: http.StatusText(http.StatusOK), + StatusCode: http.StatusOK, + Body: f, + }, nil } -func QueryEscape(s string) string { return url.QueryEscape(s) } -func OpenBrowser(url string) bool { return browser.Open(url) } +func openBrowser(url string) bool { return browser.Open(url) } diff --git a/cmd/go/_internal_/web/security.go b/cmd/go/_internal_/web/security.go deleted file mode 100644 index a4a02de..0000000 --- a/cmd/go/_internal_/web/security.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package web defines helper routines for accessing HTTP/HTTPS resources. -package web - -// SecurityMode specifies whether a function should make network -// calls using insecure transports (eg, plain text HTTP). -// The zero value is "secure". -type SecurityMode int - -const ( - Secure SecurityMode = iota - Insecure -) diff --git a/cmd/go/_internal_/web/url.go b/cmd/go/_internal_/web/url.go new file mode 100644 index 0000000..0ab82e9 --- /dev/null +++ b/cmd/go/_internal_/web/url.go @@ -0,0 +1,95 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package web + +import ( + "errors" + "net/url" + "path/filepath" + "strings" +) + +// TODO(golang.org/issue/32456): If accepted, move these functions into the +// net/url package. + +var errNotAbsolute = errors.New("path is not absolute") + +func urlToFilePath(u *url.URL) (string, error) { + if u.Scheme != "file" { + return "", errors.New("non-file URL") + } + + checkAbs := func(path string) (string, error) { + if !filepath.IsAbs(path) { + return "", errNotAbsolute + } + return path, nil + } + + if u.Path == "" { + if u.Host != "" || u.Opaque == "" { + return "", errors.New("file URL missing path") + } + return checkAbs(filepath.FromSlash(u.Opaque)) + } + + path, err := convertFileURLPath(u.Host, u.Path) + if err != nil { + return path, err + } + return checkAbs(path) +} + +func urlFromFilePath(path string) (*url.URL, error) { + if !filepath.IsAbs(path) { + return nil, errNotAbsolute + } + + // If path has a Windows volume name, convert the volume to a host and prefix + // per https://blogs.msdn.microsoft.com/ie/2006/12/06/file-uris-in-windows/. + if vol := filepath.VolumeName(path); vol != "" { + if strings.HasPrefix(vol, `\\`) { + path = filepath.ToSlash(path[2:]) + i := strings.IndexByte(path, '/') + + if i < 0 { + // A degenerate case. + // \\host.example.com (without a share name) + // becomes + // file://host.example.com/ + return &url.URL{ + Scheme: "file", + Host: path, + Path: "/", + }, nil + } + + // \\host.example.com\Share\path\to\file + // becomes + // file://host.example.com/Share/path/to/file + return &url.URL{ + Scheme: "file", + Host: path[:i], + Path: filepath.ToSlash(path[i:]), + }, nil + } + + // C:\path\to\file + // becomes + // file:///C:/path/to/file + return &url.URL{ + Scheme: "file", + Path: "/" + filepath.ToSlash(path), + }, nil + } + + // /path/to/file + // becomes + // file:///path/to/file + return &url.URL{ + Scheme: "file", + Path: filepath.ToSlash(path), + }, nil +} diff --git a/cmd/go/_internal_/web/url_other.go b/cmd/go/_internal_/web/url_other.go new file mode 100644 index 0000000..2641ee6 --- /dev/null +++ b/cmd/go/_internal_/web/url_other.go @@ -0,0 +1,21 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !windows + +package web + +import ( + "errors" + "path/filepath" +) + +func convertFileURLPath(host, path string) (string, error) { + switch host { + case "", "localhost": + default: + return "", errors.New("file URL specifies non-local host") + } + return filepath.FromSlash(path), nil +} diff --git a/cmd/go/_internal_/web2/web.go b/cmd/go/_internal_/web2/web.go deleted file mode 100644 index b3eff29..0000000 --- a/cmd/go/_internal_/web2/web.go +++ /dev/null @@ -1,299 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package web2 - -import ( - "bytes" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" - "encoding/json" - "flag" - "fmt" - "io" - "io/ioutil" - "log" - "net/http" - "os" - "path/filepath" - "runtime" - "runtime/debug" - "strings" - "sync" -) - -var TraceGET = false -var webstack = false - -func init() { - flag.BoolVar(&TraceGET, "webtrace", TraceGET, "trace GET requests") - flag.BoolVar(&webstack, "webstack", webstack, "print stack for GET requests") -} - -type netrcLine struct { - machine string - login string - password string -} - -var netrcOnce sync.Once -var netrc []netrcLine - -func parseNetrc(data string) []netrcLine { - var nrc []netrcLine - var l netrcLine - for _, line := range strings.Split(data, "\n") { - f := strings.Fields(line) - for i := 0; i < len(f)-1; i += 2 { - switch f[i] { - case "machine": - l.machine = f[i+1] - case "login": - l.login = f[i+1] - case "password": - l.password = f[i+1] - } - } - if l.machine != "" && l.login != "" && l.password != "" { - nrc = append(nrc, l) - l = netrcLine{} - } - } - return nrc -} - -func havePassword(machine string) bool { - netrcOnce.Do(readNetrc) - for _, line := range netrc { - if line.machine == machine { - return true - } - } - return false -} - -func netrcPath() string { - switch runtime.GOOS { - case "windows": - return filepath.Join(os.Getenv("USERPROFILE"), "_netrc") - case "plan9": - return filepath.Join(os.Getenv("home"), ".netrc") - default: - return filepath.Join(os.Getenv("HOME"), ".netrc") - } -} - -func readNetrc() { - data, err := ioutil.ReadFile(netrcPath()) - if err != nil { - return - } - netrc = parseNetrc(string(data)) -} - -type getState struct { - req *http.Request - resp *http.Response - body io.ReadCloser - non200ok bool -} - -type Option interface { - option(*getState) error -} - -func Non200OK() Option { - return optionFunc(func(g *getState) error { - g.non200ok = true - return nil - }) -} - -type optionFunc func(*getState) error - -func (f optionFunc) option(g *getState) error { - return f(g) -} - -func DecodeJSON(dst interface{}) Option { - return optionFunc(func(g *getState) error { - if g.resp != nil { - return json.NewDecoder(g.body).Decode(dst) - } - return nil - }) -} - -func ReadAllBody(body *[]byte) Option { - return optionFunc(func(g *getState) error { - if g.resp != nil { - var err error - *body, err = ioutil.ReadAll(g.body) - return err - } - return nil - }) -} - -func Body(body *io.ReadCloser) Option { - return optionFunc(func(g *getState) error { - if g.resp != nil { - *body = g.body - g.body = nil - } - return nil - }) -} - -func Header(hdr *http.Header) Option { - return optionFunc(func(g *getState) error { - if g.resp != nil { - *hdr = CopyHeader(g.resp.Header) - } - return nil - }) -} - -func CopyHeader(hdr http.Header) http.Header { - if hdr == nil { - return nil - } - h2 := make(http.Header) - for k, v := range hdr { - v2 := make([]string, len(v)) - copy(v2, v) - h2[k] = v2 - } - return h2 -} - -var cache struct { - mu sync.Mutex - byURL map[string]*cacheEntry -} - -type cacheEntry struct { - mu sync.Mutex - resp *http.Response - body []byte -} - -var httpDo = http.DefaultClient.Do - -func SetHTTPDoForTesting(do func(*http.Request) (*http.Response, error)) { - if do == nil { - do = http.DefaultClient.Do - } - httpDo = do -} - -func Get(url string, options ...Option) error { - if TraceGET || webstack || cfg.BuildV { - log.Printf("Fetching %s", url) - if webstack { - log.Println(string(debug.Stack())) - } - } - - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return err - } - - netrcOnce.Do(readNetrc) - for _, l := range netrc { - if l.machine == req.URL.Host { - req.SetBasicAuth(l.login, l.password) - break - } - } - - g := &getState{req: req} - for _, o := range options { - if err := o.option(g); err != nil { - return err - } - } - - cache.mu.Lock() - e := cache.byURL[url] - if e == nil { - e = new(cacheEntry) - if !strings.HasPrefix(url, "file:") { - if cache.byURL == nil { - cache.byURL = make(map[string]*cacheEntry) - } - cache.byURL[url] = e - } - } - cache.mu.Unlock() - - e.mu.Lock() - if strings.HasPrefix(url, "file:") { - body, err := ioutil.ReadFile(req.URL.Path) - if err != nil { - e.mu.Unlock() - return err - } - e.body = body - e.resp = &http.Response{ - StatusCode: 200, - } - } else if e.resp == nil { - resp, err := httpDo(req) - if err != nil { - e.mu.Unlock() - return err - } - e.resp = resp - // TODO: Spool to temp file. - body, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - resp.Body = nil - if err != nil { - e.mu.Unlock() - return err - } - e.body = body - } - g.resp = e.resp - g.body = ioutil.NopCloser(bytes.NewReader(e.body)) - e.mu.Unlock() - - defer func() { - if g.body != nil { - g.body.Close() - } - }() - - if g.resp.StatusCode == 403 && req.URL.Host == "api.github.com" && !havePassword("api.github.com") { - base.Errorf("%s", githubMessage) - } - if !g.non200ok && g.resp.StatusCode != 200 { - return fmt.Errorf("unexpected status (%s): %v", url, g.resp.Status) - } - - for _, o := range options { - if err := o.option(g); err != nil { - return err - } - } - return err -} - -var githubMessage = `go: 403 response from api.github.com - -GitHub applies fairly small rate limits to unauthenticated users, and -you appear to be hitting them. To authenticate, please visit -https://github.com/settings/tokens and click "Generate New Token" to -create a Personal Access Token. The token only needs "public_repo" -scope, but you can add "repo" if you want to access private -repositories too. - -Add the token to your $HOME/.netrc (%USERPROFILE%\_netrc on Windows): - - machine api.github.com login YOU password TOKEN - -Sorry for the interruption. -` diff --git a/cmd/go/_internal_/work/action.go b/cmd/go/_internal_/work/action.go index 59d61a7..27916c8 100644 --- a/cmd/go/_internal_/work/action.go +++ b/cmd/go/_internal_/work/action.go @@ -16,8 +16,10 @@ import ( "io/ioutil" "os" "path/filepath" + "runtime" "strings" "sync" + "time" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cache" @@ -39,7 +41,7 @@ type Builder struct { IsCmdList bool // running as part of go list; set p.Stale and additional fields below NeedError bool // list needs p.Error NeedExport bool // list needs p.Export - NeedCompiledGoFiles bool // list needs p.CompiledGoFIles + NeedCompiledGoFiles bool // list needs p.CompiledGoFiles objdirSeq int // counter for NewObjdir pkgSeq int @@ -84,13 +86,15 @@ type Action struct { VetxOnly bool // Mode=="vet": only being called to supply info about dependencies needVet bool // Mode=="build": need to fill in vet config + needBuild bool // Mode=="build": need to do actual build (can be false if needVet is true) vetCfg *vetConfig // vet config output []byte // output redirect buffer (nil means use b.Print) // Execution state. - pending int // number of deps yet to complete - priority int // relative execution priority - Failed bool // whether the action failed + pending int // number of deps yet to complete + priority int // relative execution priority + Failed bool // whether the action failed + json *actionJSON // action graph information } // BuildActionID returns the action ID section of a's build ID. @@ -122,6 +126,9 @@ func (q *actionQueue) Pop() interface{} { } func (q *actionQueue) push(a *Action) { + if a.json != nil { + a.json.TimeReady = time.Now() + } heap.Push(q, a) } @@ -143,6 +150,18 @@ type actionJSON struct { Failed bool `json:",omitempty"` Built string `json:",omitempty"` VetxOnly bool `json:",omitempty"` + NeedVet bool `json:",omitempty"` + NeedBuild bool `json:",omitempty"` + ActionID string `json:",omitempty"` + BuildID string `json:",omitempty"` + TimeReady time.Time `json:",omitempty"` + TimeStart time.Time `json:",omitempty"` + TimeDone time.Time `json:",omitempty"` + + Cmd []string // `json:",omitempty"` + CmdReal time.Duration `json:",omitempty"` + CmdUser time.Duration `json:",omitempty"` + CmdSys time.Duration `json:",omitempty"` } // cacheKey is the key for the action cache. @@ -172,26 +191,30 @@ func actionGraphJSON(a *Action) string { var list []*actionJSON for id, a := range workq { - aj := &actionJSON{ - Mode: a.Mode, - ID: id, - IgnoreFail: a.IgnoreFail, - Args: a.Args, - Objdir: a.Objdir, - Target: a.Target, - Failed: a.Failed, - Priority: a.priority, - Built: a.built, - VetxOnly: a.VetxOnly, - } - if a.Package != nil { - // TODO(rsc): Make this a unique key for a.Package somehow. - aj.Package = a.Package.ImportPath - } - for _, a1 := range a.Deps { - aj.Deps = append(aj.Deps, inWorkq[a1]) + if a.json == nil { + a.json = &actionJSON{ + Mode: a.Mode, + ID: id, + IgnoreFail: a.IgnoreFail, + Args: a.Args, + Objdir: a.Objdir, + Target: a.Target, + Failed: a.Failed, + Priority: a.priority, + Built: a.built, + VetxOnly: a.VetxOnly, + NeedBuild: a.needBuild, + NeedVet: a.needVet, + } + if a.Package != nil { + // TODO(rsc): Make this a unique key for a.Package somehow. + a.json.Package = a.Package.ImportPath + } + for _, a1 := range a.Deps { + a.json.Deps = append(a.json.Deps, inWorkq[a1]) + } } - list = append(list, aj) + list = append(list, a.json) } js, err := json.MarshalIndent(list, "", "\t") @@ -210,6 +233,8 @@ const ( ModeBuild BuildMode = iota ModeInstall ModeBuggyInstall + + ModeVetOnly = 1 << 8 ) func (b *Builder) Init() { @@ -224,7 +249,7 @@ func (b *Builder) Init() { if cfg.BuildN { b.WorkDir = "$WORK" } else { - tmp, err := ioutil.TempDir(os.Getenv("GOTMPDIR"), "go-build") + tmp, err := ioutil.TempDir(cfg.Getenv("GOTMPDIR"), "go-build") if err != nil { base.Fatalf("go: creating work dir: %v", err) } @@ -242,22 +267,51 @@ func (b *Builder) Init() { } if !cfg.BuildWork { workdir := b.WorkDir - base.AtExit(func() { os.RemoveAll(workdir) }) + base.AtExit(func() { + start := time.Now() + for { + err := os.RemoveAll(workdir) + if err == nil { + return + } + + // On some configurations of Windows, directories containing executable + // files may be locked for a while after the executable exits (perhaps + // due to antivirus scans?). It's probably worth a little extra latency + // on exit to avoid filling up the user's temporary directory with leaked + // files. (See golang.org/issue/30789.) + if runtime.GOOS != "windows" || time.Since(start) >= 500*time.Millisecond { + fmt.Fprintf(os.Stderr, "go: failed to remove work dir: %s\n", err) + return + } + time.Sleep(5 * time.Millisecond) + } + }) } } - if _, ok := cfg.OSArchSupportsCgo[cfg.Goos+"/"+cfg.Goarch]; !ok && cfg.BuildContext.Compiler == "gc" { - fmt.Fprintf(os.Stderr, "cmd/go: unsupported GOOS/GOARCH pair %s/%s\n", cfg.Goos, cfg.Goarch) - os.Exit(2) + if err := CheckGOOSARCHPair(cfg.Goos, cfg.Goarch); err != nil { + fmt.Fprintf(os.Stderr, "cmd/go: %v\n", err) + base.SetExitStatus(2) + base.Exit() } + for _, tag := range cfg.BuildContext.BuildTags { if strings.Contains(tag, ",") { fmt.Fprintf(os.Stderr, "cmd/go: -tags space-separated list contains comma\n") - os.Exit(2) + base.SetExitStatus(2) + base.Exit() } } } +func CheckGOOSARCHPair(goos, goarch string) error { + if _, ok := cfg.OSArchSupportsCgo[goos+"/"+goarch]; !ok && cfg.BuildContext.Compiler == "gc" { + return fmt.Errorf("unsupported GOOS/GOARCH pair %s/%s", goos, goarch) + } + return nil +} + // NewObjdir returns the name of a fresh object directory under b.WorkDir. // It is up to the caller to call b.Mkdir on the result at an appropriate time. // The result ends in a slash, so that file names in that directory @@ -287,7 +341,7 @@ func readpkglist(shlibpath string) (pkgs []*load.Package) { if strings.HasPrefix(t, "pkgpath ") { t = strings.TrimPrefix(t, "pkgpath ") t = strings.TrimSuffix(t, ";") - pkgs = append(pkgs, load.LoadPackage(t, &stk)) + pkgs = append(pkgs, load.LoadImportWithFlags(t, base.Cwd, nil, &stk, nil, 0)) } } } else { @@ -298,7 +352,7 @@ func readpkglist(shlibpath string) (pkgs []*load.Package) { scanner := bufio.NewScanner(bytes.NewBuffer(pkglistbytes)) for scanner.Scan() { t := scanner.Text() - pkgs = append(pkgs, load.LoadPackage(t, &stk)) + pkgs = append(pkgs, load.LoadImportWithFlags(t, base.Cwd, nil, &stk, nil, 0)) } } return @@ -331,6 +385,9 @@ func (b *Builder) AutoAction(mode, depMode BuildMode, p *load.Package) *Action { // depMode is the action (build or install) to use when building dependencies. // To turn package main into an executable, call b.Link instead. func (b *Builder) CompileAction(mode, depMode BuildMode, p *load.Package) *Action { + vetOnly := mode&ModeVetOnly != 0 + mode &^= ModeVetOnly + if mode != ModeBuild && (p.Internal.Local || p.Module != nil) && p.Target == "" { // Imported via local path or using modules. No permanent target. mode = ModeBuild @@ -377,6 +434,19 @@ func (b *Builder) CompileAction(mode, depMode BuildMode, p *load.Package) *Actio return a }) + // Find the build action; the cache entry may have been replaced + // by the install action during (*Builder).installAction. + buildAction := a + switch buildAction.Mode { + case "build", "built-in package", "gccgo stdlib": + // ok + case "build-install": + buildAction = a.Deps[0] + default: + panic("lost build action: " + buildAction.Mode) + } + buildAction.needBuild = buildAction.needBuild || !vetOnly + // Construct install action. if mode == ModeInstall || mode == ModeBuggyInstall { a = b.installAction(a, mode) @@ -398,12 +468,12 @@ func (b *Builder) VetAction(mode, depMode BuildMode, p *load.Package) *Action { func (b *Builder) vetAction(mode, depMode BuildMode, p *load.Package) *Action { // Construct vet action. a := b.cacheAction("vet", p, func() *Action { - a1 := b.CompileAction(mode, depMode, p) + a1 := b.CompileAction(mode|ModeVetOnly, depMode, p) // vet expects to be able to import "fmt". var stk load.ImportStack stk.Push("vet") - p1 := load.LoadPackage("fmt", &stk) + p1 := load.LoadImportWithFlags("fmt", p.Dir, p, &stk, nil, 0) stk.Pop() aFmt := b.CompileAction(ModeBuild, depMode, p1) @@ -417,7 +487,7 @@ func (b *Builder) vetAction(mode, depMode BuildMode, p *load.Package) *Action { } else { deps = []*Action{a1, aFmt} } - for _, p1 := range load.PackageList(p.Internal.Imports) { + for _, p1 := range p.Internal.Imports { deps = append(deps, b.vetAction(mode, depMode, p1)) } @@ -640,7 +710,7 @@ func (b *Builder) addInstallHeaderAction(a *Action) { } // buildmodeShared takes the "go build" action a1 into the building of a shared library of a1.Deps. -// That is, the input a1 represents "go build pkgs" and the result represents "go build -buidmode=shared pkgs". +// That is, the input a1 represents "go build pkgs" and the result represents "go build -buildmode=shared pkgs". func (b *Builder) buildmodeShared(mode, depMode BuildMode, args []string, pkgs []*load.Package, a1 *Action) *Action { name, err := libname(args, pkgs) if err != nil { @@ -703,7 +773,7 @@ func (b *Builder) linkSharedAction(mode, depMode BuildMode, shlib string, a1 *Ac } } var stk load.ImportStack - p := load.LoadPackage(pkg, &stk) + p := load.LoadImportWithFlags(pkg, base.Cwd, nil, &stk, nil, 0) if p.Error != nil { base.Fatalf("load %s: %v", pkg, p.Error) } diff --git a/cmd/go/_internal_/work/build.go b/cmd/go/_internal_/work/build.go index d288316..c5a950a 100644 --- a/cmd/go/_internal_/work/build.go +++ b/cmd/go/_internal_/work/build.go @@ -27,8 +27,10 @@ var CmdBuild = &base.Command{ Build compiles the packages named by the import paths, along with their dependencies, but it does not install the results. -If the arguments to build are a list of .go files, build treats -them as a list of source files specifying a single package. +If the arguments to build are a list of .go files from a single directory, +build treats them as a list of source files specifying a single package. + +When compiling packages, build ignores files that end in '_test.go'. When compiling a single main package, build writes the resulting executable to an output file named after @@ -40,12 +42,10 @@ When compiling multiple packages or a single non-main package, build compiles the packages but discards the resulting object, serving only as a check that the packages can be built. -When compiling packages, build ignores files that end in '_test.go'. - -The -o flag, only allowed when compiling a single package, -forces build to write the resulting executable or object -to the named output file, instead of the default behavior described -in the last two paragraphs. +The -o flag forces build to write the resulting executable or object +to the named output file or directory, instead of the default behavior described +in the last two paragraphs. If the named output is a directory that exists, +then any resulting executables will be written to that directory. The -i flag installs the packages that are dependencies of the target. @@ -62,11 +62,13 @@ and test commands: The default is the number of CPUs available. -race enable data race detection. - Supported only on linux/amd64, freebsd/amd64, darwin/amd64 and windows/amd64. + Supported only on linux/amd64, freebsd/amd64, darwin/amd64, windows/amd64, + linux/ppc64le and linux/arm64 (only for 48-bit VMA). -msan enable interoperation with memory sanitizer. Supported only on linux/amd64, linux/arm64 and only with Clang/LLVM as the host C compiler. + On linux/arm64, pie build mode will be used. -v print the names of packages as they are compiled. -work @@ -95,19 +97,37 @@ and test commands: -ldflags '[pattern=]arg list' arguments to pass on each go tool link invocation. -linkshared - link against shared libraries previously created with - -buildmode=shared. + build code that will be linked against shared libraries previously + created with -buildmode=shared. -mod mode - module download mode to use: readonly or vendor. + module download mode to use: readonly, vendor, or mod. See 'go help modules' for more. + -modcacherw + leave newly-created directories in the module cache read-write + instead of making them read-only. + -modfile file + in module aware mode, read (and possibly write) an alternate go.mod + file instead of the one in the module root directory. A file named + "go.mod" must still be present in order to determine the module root + directory, but it is not accessed. When -modfile is specified, an + alternate go.sum file is also used: its path is derived from the + -modfile flag by trimming the ".mod" extension and appending ".sum". -pkgdir dir install and load all packages from dir instead of the usual locations. For example, when building with a non-standard configuration, use -pkgdir to keep generated packages in a separate location. - -tags 'tag list' - a space-separated list of build tags to consider satisfied during the + -tags tag,list + a comma-separated list of build tags to consider satisfied during the build. For more information about build tags, see the description of build constraints in the documentation for the go/build package. + (Earlier versions of Go used a space-separated list, and that form + is deprecated but still recognized.) + -trimpath + remove all file system paths from the resulting executable. + Instead of absolute file system paths, the recorded file names + will begin with either "go" (for the standard library), + or a module path@version (when using modules), + or a plain import path (when using GOPATH). -toolexec 'cmd args' a program to use to invoke toolchain programs like vet and asm. For example, instead of running asm, the go command will run @@ -153,12 +173,12 @@ func init() { CmdInstall.Run = runInstall CmdBuild.Flag.BoolVar(&cfg.BuildI, "i", false, "") - CmdBuild.Flag.StringVar(&cfg.BuildO, "o", "", "output file") + CmdBuild.Flag.StringVar(&cfg.BuildO, "o", "", "output file or directory") CmdInstall.Flag.BoolVar(&cfg.BuildI, "i", false, "") - AddBuildFlags(CmdBuild) - AddBuildFlags(CmdInstall) + AddBuildFlags(CmdBuild, DefaultBuildFlags) + AddBuildFlags(CmdInstall, DefaultBuildFlags) } // Note that flags consulted by other parts of the code @@ -206,13 +226,24 @@ func init() { } } -// addBuildFlags adds the flags common to the build, clean, get, +type BuildFlagMask int + +const ( + DefaultBuildFlags BuildFlagMask = 0 + OmitModFlag BuildFlagMask = 1 << iota + OmitModCommonFlags + OmitVFlag +) + +// AddBuildFlags adds the flags common to the build, clean, get, // install, list, run, and test commands. -func AddBuildFlags(cmd *base.Command) { +func AddBuildFlags(cmd *base.Command, mask BuildFlagMask) { cmd.Flag.BoolVar(&cfg.BuildA, "a", false, "") cmd.Flag.BoolVar(&cfg.BuildN, "n", false, "") cmd.Flag.IntVar(&cfg.BuildP, "p", cfg.BuildP, "") - cmd.Flag.BoolVar(&cfg.BuildV, "v", false, "") + if mask&OmitVFlag == 0 { + cmd.Flag.BoolVar(&cfg.BuildV, "v", false, "") + } cmd.Flag.BoolVar(&cfg.BuildX, "x", false, "") cmd.Flag.Var(&load.BuildAsmflags, "asmflags", "") @@ -220,21 +251,57 @@ func AddBuildFlags(cmd *base.Command) { cmd.Flag.StringVar(&cfg.BuildBuildmode, "buildmode", "default", "") cmd.Flag.Var(&load.BuildGcflags, "gcflags", "") cmd.Flag.Var(&load.BuildGccgoflags, "gccgoflags", "") - cmd.Flag.StringVar(&cfg.BuildMod, "mod", "", "") + if mask&OmitModFlag == 0 { + cmd.Flag.StringVar(&cfg.BuildMod, "mod", "", "") + } + if mask&OmitModCommonFlags == 0 { + AddModCommonFlags(cmd) + } cmd.Flag.StringVar(&cfg.BuildContext.InstallSuffix, "installsuffix", "", "") cmd.Flag.Var(&load.BuildLdflags, "ldflags", "") cmd.Flag.BoolVar(&cfg.BuildLinkshared, "linkshared", false, "") cmd.Flag.StringVar(&cfg.BuildPkgdir, "pkgdir", "", "") cmd.Flag.BoolVar(&cfg.BuildRace, "race", false, "") cmd.Flag.BoolVar(&cfg.BuildMSan, "msan", false, "") - cmd.Flag.Var((*base.StringsFlag)(&cfg.BuildContext.BuildTags), "tags", "") + cmd.Flag.Var((*tagsFlag)(&cfg.BuildContext.BuildTags), "tags", "") cmd.Flag.Var((*base.StringsFlag)(&cfg.BuildToolexec), "toolexec", "") + cmd.Flag.BoolVar(&cfg.BuildTrimpath, "trimpath", false, "") cmd.Flag.BoolVar(&cfg.BuildWork, "work", false, "") // Undocumented, unstable debugging flags. cmd.Flag.StringVar(&cfg.DebugActiongraph, "debug-actiongraph", "", "") } +// AddModCommonFlags adds the module-related flags common to build commands +// and 'go mod' subcommands. +func AddModCommonFlags(cmd *base.Command) { + cmd.Flag.BoolVar(&cfg.ModCacheRW, "modcacherw", false, "") + cmd.Flag.StringVar(&cfg.ModFile, "modfile", "", "") +} + +// tagsFlag is the implementation of the -tags flag. +type tagsFlag []string + +func (v *tagsFlag) Set(s string) error { + // For compatibility with Go 1.12 and earlier, allow "-tags='a b c'" or even just "-tags='a'". + if strings.Contains(s, " ") || strings.Contains(s, "'") { + return (*base.StringsFlag)(v).Set(s) + } + + // Split on commas, ignore empty strings. + *v = []string{} + for _, s := range strings.Split(s, ",") { + if s != "" { + *v = append(*v, s) + } + } + return nil +} + +func (v *tagsFlag) String() string { + return "" +} + // fileExtSplit expects a filename and returns the name // and ext (without the dot). If the file has no // extension, ext will be empty. @@ -283,8 +350,10 @@ func runBuild(cmd *base.Command, args []string) { pkgs := load.PackagesForBuild(args) + explicitO := len(cfg.BuildO) > 0 + if len(pkgs) == 1 && pkgs[0].Name == "main" && cfg.BuildO == "" { - cfg.BuildO = load.DefaultExecName(pkgs[0].ImportPath) + cfg.BuildO = pkgs[0].DefaultExecName() cfg.BuildO += cfg.ExeSuffix } @@ -316,8 +385,33 @@ func runBuild(cmd *base.Command, args []string) { } if cfg.BuildO != "" { + // If the -o name exists and is a directory, then + // write all main packages to that directory. + // Otherwise require only a single package be built. + if fi, err := os.Stat(cfg.BuildO); err == nil && fi.IsDir() { + if !explicitO { + base.Fatalf("go build: build output %q already exists and is a directory", cfg.BuildO) + } + a := &Action{Mode: "go build"} + for _, p := range pkgs { + if p.Name != "main" { + continue + } + + p.Target = filepath.Join(cfg.BuildO, p.DefaultExecName()) + p.Target += cfg.ExeSuffix + p.Stale = true + p.StaleReason = "build -o flag in use" + a.Deps = append(a.Deps, b.AutoAction(ModeInstall, depMode, p)) + } + if len(a.Deps) == 0 { + base.Fatalf("go build: no main packages to build") + } + b.Do(a) + return + } if len(pkgs) > 1 { - base.Fatalf("go build: cannot use -o with multiple packages") + base.Fatalf("go build: cannot write multiple packages to non-directory %s", cfg.BuildO) } else if len(pkgs) == 0 { base.Fatalf("no packages to build") } @@ -346,6 +440,15 @@ var CmdInstall = &base.Command{ Long: ` Install compiles and installs the packages named by the import paths. +Executables are installed in the directory named by the GOBIN environment +variable, which defaults to $GOPATH/bin or $HOME/go/bin if the GOPATH +environment variable is not set. Executables in $GOROOT +are installed in $GOROOT/bin or $GOTOOLDIR instead of $GOBIN. + +When module-aware mode is disabled, other packages are installed in the +directory $GOPATH/pkg/$GOOS_$GOARCH. When module-aware mode is enabled, +other packages are built and cached but not installed. + The -i flag installs the dependencies of the named packages as well. For more about the build flags, see 'go help build'. @@ -517,7 +620,7 @@ func InstallPackages(patterns []string, pkgs []*load.Package) { if len(patterns) == 0 && len(pkgs) == 1 && pkgs[0].Name == "main" { // Compute file 'go build' would have created. // If it exists and is an executable file, remove it. - targ := load.DefaultExecName(pkgs[0].ImportPath) + targ := pkgs[0].DefaultExecName() targ += cfg.ExeSuffix if filepath.Join(pkgs[0].Dir, targ) != pkgs[0].Target { // maybe $GOBIN is the current directory fi, err := os.Stat(targ) diff --git a/cmd/go/_internal_/work/buildid.go b/cmd/go/_internal_/work/buildid.go index 4c891a5..c7e18af 100644 --- a/cmd/go/_internal_/work/buildid.go +++ b/cmd/go/_internal_/work/buildid.go @@ -15,7 +15,6 @@ import ( "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cache" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/load" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/str" "github.com/dependabot/gomodules-extracted/cmd/_internal_/buildid" ) @@ -159,7 +158,7 @@ func hashToString(h [cache.HashSize]byte) string { // which influences the action ID half of the build ID, is based on the content ID, // then the Linux compiler binary and Mac compiler binary will have different tool IDs // and therefore produce executables with different action IDs. -// To avoids this problem, for releases we use the release version string instead +// To avoid this problem, for releases we use the release version string instead // of the compiler binary's content hash. This assumes that all compilers built // on all different systems are semantically equivalent, which is of course only true // modulo bugs. (Producing the exact same executables also requires that the different @@ -186,7 +185,7 @@ func (b *Builder) toolID(name string) string { cmdline := str.StringList(cfg.BuildToolexec, path, "-V=full") cmd := exec.Command(cmdline[0], cmdline[1:]...) - cmd.Env = base.EnvForDir(cmd.Dir, os.Environ()) + cmd.Env = base.AppendPWD(os.Environ(), cmd.Dir) var stdout, stderr bytes.Buffer cmd.Stdout = &stdout cmd.Stderr = &stderr @@ -203,8 +202,9 @@ func (b *Builder) toolID(name string) string { // On the development branch, use the content ID part of the build ID. id = contentID(f[len(f)-1]) } else { - // For a release, the output is like: "compile version go1.9.1". Use the whole line. - id = f[2] + // For a release, the output is like: "compile version go1.9.1 X:framepointer". + // Use the whole line. + id = strings.TrimSpace(line) } b.id.Lock() @@ -215,7 +215,7 @@ func (b *Builder) toolID(name string) string { } // gccToolID returns the unique ID to use for a tool that is invoked -// by the GCC driver. This is in particular gccgo, but this can also +// by the GCC driver. This is used particularly for gccgo, but this can also // be used for gcc, g++, gfortran, etc.; those tools all use the GCC // driver under different names. The approach used here should also // work for sufficiently new versions of clang. Unlike toolID, the @@ -244,7 +244,7 @@ func (b *Builder) gccgoToolID(name, language string) (string, error) { // compile an empty file on standard input. cmdline := str.StringList(cfg.BuildToolexec, name, "-###", "-x", language, "-c", "-") cmd := exec.Command(cmdline[0], cmdline[1:]...) - cmd.Env = base.EnvForDir(cmd.Dir, os.Environ()) + cmd.Env = base.AppendPWD(os.Environ(), cmd.Dir) // Force untranslated output so that we see the string "version". cmd.Env = append(cmd.Env, "LC_ALL=C") out, err := cmd.CombinedOutput() @@ -291,14 +291,19 @@ func (b *Builder) gccgoToolID(name, language string) (string, error) { exe = lp } } - if _, err := os.Stat(exe); err != nil { - return "", fmt.Errorf("%s: can not find compiler %q: %v; output %q", name, exe, err, out) + id, err = buildid.ReadFile(exe) + if err != nil { + return "", err + } + + // If we can't find a build ID, use a hash. + if id == "" { + id = b.fileHash(exe) } - id = b.fileHash(exe) } b.id.Lock() - b.toolIDCache[name] = id + b.toolIDCache[key] = id b.id.Unlock() return id, nil @@ -326,7 +331,7 @@ func (b *Builder) gccgoBuildIDFile(a *Action) (string, error) { var buf bytes.Buffer if cfg.Goos == "aix" { fmt.Fprintf(&buf, "\t.csect .go.buildid[XO]\n") - } else if cfg.Goos != "solaris" || assemblerIsGas() { + } else if (cfg.Goos != "solaris" && cfg.Goos != "illumos") || assemblerIsGas() { fmt.Fprintf(&buf, "\t"+`.section .go.buildid,"e"`+"\n") } else if cfg.Goarch == "sparc" || cfg.Goarch == "sparc64" { fmt.Fprintf(&buf, "\t"+`.section ".go.buildid",#exclude`+"\n") @@ -345,7 +350,7 @@ func (b *Builder) gccgoBuildIDFile(a *Action) (string, error) { fmt.Fprintf(&buf, "%#02x", a.buildID[i]) } fmt.Fprintf(&buf, "\n") - if cfg.Goos != "solaris" && cfg.Goos != "aix" { + if cfg.Goos != "solaris" && cfg.Goos != "illumos" && cfg.Goos != "aix" { secType := "@progbits" if cfg.Goarch == "arm" { secType = "%progbits" @@ -415,7 +420,7 @@ func (b *Builder) fileHash(file string) string { // during a's work. The caller should defer b.flushOutput(a), to make sure // that flushOutput is eventually called regardless of whether the action // succeeds. The flushOutput call must happen after updateBuildID. -func (b *Builder) useCache(a *Action, p *load.Package, actionHash cache.ActionID, target string) bool { +func (b *Builder) useCache(a *Action, actionHash cache.ActionID, target string) bool { // The second half of the build ID here is a placeholder for the content hash. // It's important that the overall buildID be unlikely verging on impossible // to appear in the output by chance, but that should be taken care of by @@ -423,6 +428,9 @@ func (b *Builder) useCache(a *Action, p *load.Package, actionHash cache.ActionID // engineered 96-bit partial SHA256 collision. a.actionID = actionHash actionID := hashToString(actionHash) + if a.json != nil { + a.json.ActionID = actionID + } contentID := actionID // temporary placeholder, likely unique a.buildID = actionID + buildIDSeparator + contentID @@ -440,6 +448,9 @@ func (b *Builder) useCache(a *Action, p *load.Package, actionHash cache.ActionID buildID, _ = buildid.ReadFile(target) if strings.HasPrefix(buildID, actionID+buildIDSeparator) { a.buildID = buildID + if a.json != nil { + a.json.BuildID = a.buildID + } a.built = target // Poison a.Target to catch uses later in the build. a.Target = "DO NOT USE - " + a.Mode @@ -482,6 +493,9 @@ func (b *Builder) useCache(a *Action, p *load.Package, actionHash cache.ActionID // Poison a.Target to catch uses later in the build. a.Target = "DO NOT USE - main build pseudo-cache Target" a.built = "DO NOT USE - main build pseudo-cache built" + if a.json != nil { + a.json.BuildID = a.buildID + } return true } // Otherwise restore old build ID for main build. @@ -549,6 +563,9 @@ func (b *Builder) useCache(a *Action, p *load.Package, actionHash cache.ActionID a.built = file a.Target = "DO NOT USE - using cache" a.buildID = buildID + if a.json != nil { + a.json.BuildID = a.buildID + } if p := a.Package; p != nil { // Clearer than explaining that something else is stale. p.StaleReason = "not installed but available in build cache" @@ -644,6 +661,9 @@ func (b *Builder) updateBuildID(a *Action, target string, rewrite bool) error { // Replace with new content-based ID. a.buildID = newID + if a.json != nil { + a.json.BuildID = a.buildID + } if len(matches) == 0 { // Assume the user specified -buildid= to override what we were going to choose. return nil diff --git a/cmd/go/_internal_/work/exec.go b/cmd/go/_internal_/work/exec.go index 690329b..12304e7 100644 --- a/cmd/go/_internal_/work/exec.go +++ b/cmd/go/_internal_/work/exec.go @@ -11,6 +11,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/dependabot/gomodules-extracted/_internal_/lazyregexp" "io" "io/ioutil" "log" @@ -54,8 +55,9 @@ func actionList(root *Action) []*Action { // do runs the action graph rooted at root. func (b *Builder) Do(root *Action) { - if c := cache.Default(); c != nil && !b.IsCmdList { + if !b.IsCmdList { // If we're doing real work, take time at the end to trim the cache. + c := cache.Default() defer c.Trim() } @@ -75,13 +77,22 @@ func (b *Builder) Do(root *Action) { a.priority = i } - if cfg.DebugActiongraph != "" { - js := actionGraphJSON(root) - if err := ioutil.WriteFile(cfg.DebugActiongraph, []byte(js), 0666); err != nil { - fmt.Fprintf(os.Stderr, "go: writing action graph: %v\n", err) - base.SetExitStatus(1) + // Write action graph, without timing information, in case we fail and exit early. + writeActionGraph := func() { + if file := cfg.DebugActiongraph; file != "" { + if strings.HasSuffix(file, ".go") { + // Do not overwrite Go source code in: + // go build -debug-actiongraph x.go + base.Fatalf("go: refusing to write action graph to %v\n", file) + } + js := actionGraphJSON(root) + if err := ioutil.WriteFile(file, []byte(js), 0666); err != nil { + fmt.Fprintf(os.Stderr, "go: writing action graph: %v\n", err) + base.SetExitStatus(1) + } } } + writeActionGraph() b.readySema = make(chan bool, len(all)) @@ -100,12 +111,15 @@ func (b *Builder) Do(root *Action) { // Handle runs a single action and takes care of triggering // any actions that are runnable as a result. handle := func(a *Action) { + if a.json != nil { + a.json.TimeStart = time.Now() + } var err error - if a.Func != nil && (!a.Failed || a.IgnoreFail) { - if err == nil { - err = a.Func(b, a) - } + err = a.Func(b, a) + } + if a.json != nil { + a.json.TimeDone = time.Now() } // The actions run in parallel but all the updates to the @@ -172,6 +186,9 @@ func (b *Builder) Do(root *Action) { } wg.Wait() + + // Write action graph again, this time with timing information. + writeActionGraph() } // buildActionID computes the action ID for a build action. @@ -185,17 +202,27 @@ func (b *Builder) buildActionID(a *Action) cache.ActionID { // same compiler settings and can reuse each other's results. // If not, the reason is already recorded in buildGcflags. fmt.Fprintf(h, "compile\n") + // Only include the package directory if it may affect the output. + // We trim workspace paths for all packages when -trimpath is set. // The compiler hides the exact value of $GOROOT - // when building things in GOROOT, - // but it does not hide the exact value of $GOPATH. - // Include the full dir in that case. + // when building things in GOROOT. // Assume b.WorkDir is being trimmed properly. - if !p.Goroot && !strings.HasPrefix(p.Dir, b.WorkDir) { + // When -trimpath is used with a package built from the module cache, + // use the module path and version instead of the directory. + if !p.Goroot && !cfg.BuildTrimpath && !strings.HasPrefix(p.Dir, b.WorkDir) { fmt.Fprintf(h, "dir %s\n", p.Dir) + } else if cfg.BuildTrimpath && p.Module != nil { + fmt.Fprintf(h, "module %s@%s\n", p.Module.Path, p.Module.Version) + } + if p.Module != nil { + fmt.Fprintf(h, "go %s\n", p.Module.GoVersion) } fmt.Fprintf(h, "goos %s goarch %s\n", cfg.Goos, cfg.Goarch) fmt.Fprintf(h, "import %q\n", p.ImportPath) fmt.Fprintf(h, "omitdebug %v standard %v local %v prefix %q\n", p.Internal.OmitDebug, p.Standard, p.Internal.Local, p.Internal.LocalPrefix) + if cfg.BuildTrimpath { + fmt.Fprintln(h, "trimpath") + } if p.Internal.ForceLibrary { fmt.Fprintf(h, "forcelibrary\n") } @@ -225,12 +252,16 @@ func (b *Builder) buildActionID(a *Action) cache.ActionID { if len(p.SFiles) > 0 { fmt.Fprintf(h, "asm %q %q %q\n", b.toolID("asm"), forcedAsmflags, p.Internal.Asmflags) } + // GO386, GOARM, GOMIPS, etc. - baseArch := strings.TrimSuffix(cfg.BuildContext.GOARCH, "le") - fmt.Fprintf(h, "GO$GOARCH=%s\n", os.Getenv("GO"+strings.ToUpper(baseArch))) + key, val := cfg.GetArchEnv() + fmt.Fprintf(h, "%s=%s\n", key, val) // TODO(rsc): Convince compiler team not to add more magic environment variables, // or perhaps restrict the environment variables passed to subprocesses. + // Because these are clumsy, undocumented special-case hacks + // for debugging the compiler, they are not settable using 'go env -w', + // and so here we use os.Getenv, not cfg.Getenv. magic := []string{ "GOCLOBBERDEADHASH", "GOSSAFUNC", @@ -267,8 +298,9 @@ func (b *Builder) buildActionID(a *Action) cache.ActionID { } fmt.Fprintf(h, "compile %s %q %q\n", id, forcedGccgoflags, p.Internal.Gccgoflags) fmt.Fprintf(h, "pkgpath %s\n", gccgoPkgpath(p)) + fmt.Fprintf(h, "ar %q\n", BuildToolchain.(gccgoToolchain).ar()) if len(p.SFiles) > 0 { - id, err = b.gccgoToolID(BuildToolchain.compiler(), "assembler-with-cpp") + id, _ = b.gccgoToolID(BuildToolchain.compiler(), "assembler-with-cpp") // Ignore error; different assembler versions // are unlikely to make any difference anyhow. fmt.Fprintf(h, "asm %q\n", id) @@ -364,34 +396,38 @@ func (b *Builder) build(a *Action) (err error) { return 0 } - cached := false - need := bit(needBuild, !b.IsCmdList || b.NeedExport) | + cachedBuild := false + need := bit(needBuild, !b.IsCmdList && a.needBuild || b.NeedExport) | bit(needCgoHdr, b.needCgoHdr(a)) | bit(needVet, a.needVet) | bit(needCompiledGoFiles, b.NeedCompiledGoFiles) if !p.BinaryOnly { - if b.useCache(a, p, b.buildActionID(a), p.Target) { + if b.useCache(a, b.buildActionID(a), p.Target) { // We found the main output in the cache. // If we don't need any other outputs, we can stop. + // Otherwise, we need to write files to a.Objdir (needVet, needCgoHdr). + // Remember that we might have them in cache + // and check again after we create a.Objdir. + cachedBuild = true + a.output = []byte{} // start saving output in case we miss any cache results need &^= needBuild if b.NeedExport { p.Export = a.built } - if need&needCompiledGoFiles != 0 && b.loadCachedSrcFiles(a) { - need &^= needCompiledGoFiles + if need&needCompiledGoFiles != 0 { + if err := b.loadCachedSrcFiles(a); err == nil { + need &^= needCompiledGoFiles + } } - // Otherwise, we need to write files to a.Objdir (needVet, needCgoHdr). - // Remember that we might have them in cache - // and check again after we create a.Objdir. - cached = true - a.output = []byte{} // start saving output in case we miss any cache results } // Source files might be cached, even if the full action is not // (e.g., go list -compiled -find). - if !cached && need&needCompiledGoFiles != 0 && b.loadCachedSrcFiles(a) { - need &^= needCompiledGoFiles + if !cachedBuild && need&needCompiledGoFiles != 0 { + if err := b.loadCachedSrcFiles(a); err == nil { + need &^= needCompiledGoFiles + } } if need == 0 { @@ -405,7 +441,7 @@ func (b *Builder) build(a *Action) (err error) { err = fmt.Errorf("go build %s: %v", a.Package.ImportPath, err) } if err != nil && b.IsCmdList && b.NeedError && p.Error == nil { - p.Error = &load.PackageError{Err: err.Error()} + p.Error = &load.PackageError{Err: err} } }() if cfg.BuildN { @@ -422,24 +458,12 @@ func (b *Builder) build(a *Action) (err error) { } if a.Package.BinaryOnly { - _, err := os.Stat(a.Package.Target) - if err == nil { - a.built = a.Package.Target - a.Target = a.Package.Target - if b.NeedExport { - a.Package.Export = a.Package.Target - } - a.buildID = b.fileHash(a.Package.Target) - a.Package.Stale = false - a.Package.StaleReason = "binary-only package" - return nil - } - a.Package.Stale = true - a.Package.StaleReason = "missing or invalid binary-only package" + p.Stale = true + p.StaleReason = "binary-only packages are no longer supported" if b.IsCmdList { return nil } - return fmt.Errorf("missing or invalid binary-only package; expected file %q", a.Package.Target) + return errors.New("binary-only packages are no longer supported") } if err := b.Mkdir(a.Objdir); err != nil { @@ -447,21 +471,28 @@ func (b *Builder) build(a *Action) (err error) { } objdir := a.Objdir - if cached { - if need&needCgoHdr != 0 && b.loadCachedCgoHdr(a) { + // Load cached cgo header, but only if we're skipping the main build (cachedBuild==true). + if cachedBuild && need&needCgoHdr != 0 { + if err := b.loadCachedCgoHdr(a); err == nil { need &^= needCgoHdr } + } - // Load cached vet config, but only if that's all we have left - // (need == needVet, not testing just the one bit). - // If we are going to do a full build anyway, - // we're going to regenerate the files below anyway. - if need == needVet && b.loadCachedVet(a) { + // Load cached vet config, but only if that's all we have left + // (need == needVet, not testing just the one bit). + // If we are going to do a full build anyway, + // we're going to regenerate the files below anyway. + if need == needVet { + if err := b.loadCachedVet(a); err == nil { need &^= needVet } - if need == 0 { - return nil - } + } + if need == 0 { + return nil + } + + if err := allowInstall(a); err != nil { + return err } // make target directory @@ -605,8 +636,8 @@ func (b *Builder) build(a *Action) (err error) { need &^= needVet } if need&needCompiledGoFiles != 0 { - if !b.loadCachedSrcFiles(a) { - return fmt.Errorf("failed to cache compiled Go files") + if err := b.loadCachedSrcFiles(a); err != nil { + return fmt.Errorf("loading compiled Go files from cache: %w", err) } need &^= needCompiledGoFiles } @@ -644,7 +675,7 @@ func (b *Builder) build(a *Action) (err error) { } if p.Internal.BuildInfo != "" && cfg.ModulesEnabled { - if err := b.writeFile(objdir+"_gomod_.go", load.ModInfoProg(p.Internal.BuildInfo)); err != nil { + if err := b.writeFile(objdir+"_gomod_.go", load.ModInfoProg(p.Internal.BuildInfo, cfg.BuildToolchainName == "gccgo")); err != nil { return err } gofiles = append(gofiles, objdir+"_gomod_.go") @@ -722,7 +753,7 @@ func (b *Builder) build(a *Action) (err error) { // This is read by readGccgoArchive in cmd/internal/buildid/buildid.go. if a.buildID != "" && cfg.BuildToolchainName == "gccgo" { switch cfg.Goos { - case "aix", "android", "dragonfly", "freebsd", "linux", "netbsd", "openbsd", "solaris": + case "aix", "android", "dragonfly", "freebsd", "illumos", "linux", "netbsd", "openbsd", "solaris": asmfile, err := b.gccgoBuildIDFile(a) if err != nil { return err @@ -778,7 +809,7 @@ func (b *Builder) cacheObjdirFile(a *Action, c *cache.Cache, name string) error func (b *Builder) findCachedObjdirFile(a *Action, c *cache.Cache, name string) (string, error) { file, _, err := c.GetFile(cache.Subkey(a.actionID, name)) if err != nil { - return "", err + return "", fmt.Errorf("loading cached file %s: %w", name, err) } return file, nil } @@ -793,26 +824,16 @@ func (b *Builder) loadCachedObjdirFile(a *Action, c *cache.Cache, name string) e func (b *Builder) cacheCgoHdr(a *Action) { c := cache.Default() - if c == nil { - return - } b.cacheObjdirFile(a, c, "_cgo_install.h") } -func (b *Builder) loadCachedCgoHdr(a *Action) bool { +func (b *Builder) loadCachedCgoHdr(a *Action) error { c := cache.Default() - if c == nil { - return false - } - err := b.loadCachedObjdirFile(a, c, "_cgo_install.h") - return err == nil + return b.loadCachedObjdirFile(a, c, "_cgo_install.h") } func (b *Builder) cacheSrcFiles(a *Action, srcfiles []string) { c := cache.Default() - if c == nil { - return - } var buf bytes.Buffer for _, file := range srcfiles { if !strings.HasPrefix(file, a.Objdir) { @@ -832,14 +853,11 @@ func (b *Builder) cacheSrcFiles(a *Action, srcfiles []string) { c.PutBytes(cache.Subkey(a.actionID, "srcfiles"), buf.Bytes()) } -func (b *Builder) loadCachedVet(a *Action) bool { +func (b *Builder) loadCachedVet(a *Action) error { c := cache.Default() - if c == nil { - return false - } list, _, err := c.GetBytes(cache.Subkey(a.actionID, "srcfiles")) if err != nil { - return false + return fmt.Errorf("reading srcfiles list: %w", err) } var srcfiles []string for _, name := range strings.Split(string(list), "\n") { @@ -851,22 +869,19 @@ func (b *Builder) loadCachedVet(a *Action) bool { continue } if err := b.loadCachedObjdirFile(a, c, name); err != nil { - return false + return err } srcfiles = append(srcfiles, a.Objdir+name) } buildVetConfig(a, srcfiles) - return true + return nil } -func (b *Builder) loadCachedSrcFiles(a *Action) bool { +func (b *Builder) loadCachedSrcFiles(a *Action) error { c := cache.Default() - if c == nil { - return false - } list, _, err := c.GetBytes(cache.Subkey(a.actionID, "srcfiles")) if err != nil { - return false + return fmt.Errorf("reading srcfiles list: %w", err) } var files []string for _, name := range strings.Split(string(list), "\n") { @@ -879,12 +894,12 @@ func (b *Builder) loadCachedSrcFiles(a *Action) bool { } file, err := b.findCachedObjdirFile(a, c, name) if err != nil { - return false + return fmt.Errorf("finding %s: %w", name, err) } files = append(files, file) } a.Package.CompiledGoFiles = files - return true + return nil } // vetConfig is the configuration passed to vet describing a single package. @@ -969,10 +984,13 @@ func buildVetConfig(a *Action, srcfiles []string) { // The caller is expected to set it (if needed) before executing any vet actions. var VetTool string -// VetFlags are the flags to pass to vet. +// VetFlags are the default flags to pass to vet. // The caller is expected to set them before executing any vet actions. var VetFlags []string +// VetExplicit records whether the vet flags were set explicitly on the command line. +var VetExplicit bool + func (b *Builder) vet(a *Action) error { // a.Deps[0] is the build of the package being vetted. // a.Deps[1] is the build of the "fmt" package. @@ -999,12 +1017,42 @@ func (b *Builder) vet(a *Action) error { h := cache.NewHash("vet " + a.Package.ImportPath) fmt.Fprintf(h, "vet %q\n", b.toolID("vet")) + vetFlags := VetFlags + + // In GOROOT, we enable all the vet tests during 'go test', + // not just the high-confidence subset. This gets us extra + // checking for the standard library (at some compliance cost) + // and helps us gain experience about how well the checks + // work, to help decide which should be turned on by default. + // The command-line still wins. + // + // Note that this flag change applies even when running vet as + // a dependency of vetting a package outside std. + // (Otherwise we'd have to introduce a whole separate + // space of "vet fmt as a dependency of a std top-level vet" + // versus "vet fmt as a dependency of a non-std top-level vet".) + // This is OK as long as the packages that are farther down the + // dependency tree turn on *more* analysis, as here. + // (The unsafeptr check does not write any facts for use by + // later vet runs.) + if a.Package.Goroot && !VetExplicit && VetTool == "" { + // Note that $GOROOT/src/buildall.bash + // does the same for the misc-compile trybots + // and should be updated if these flags are + // changed here. + // + // There's too much unsafe.Pointer code + // that vet doesn't like in low-level packages + // like runtime, sync, and reflect. + vetFlags = []string{"-unsafeptr=false"} + } + // Note: We could decide that vet should compute export data for // all analyses, in which case we don't need to include the flags here. // But that would mean that if an analysis causes problems like // unexpected crashes there would be no way to turn it off. // It seems better to let the flags disable export analysis too. - fmt.Fprintf(h, "vetflags %q\n", VetFlags) + fmt.Fprintf(h, "vetflags %q\n", vetFlags) fmt.Fprintf(h, "pkg %q\n", a.Deps[0].actionID) for _, a1 := range a.Deps { @@ -1015,35 +1063,14 @@ func (b *Builder) vet(a *Action) error { } key := cache.ActionID(h.Sum()) - if vcfg.VetxOnly { - if c := cache.Default(); c != nil && !cfg.BuildA { - if file, _, err := c.GetFile(key); err == nil { - a.built = file - return nil - } - } - } - - // TODO(adonovan): delete this when we use the new vet printf checker. - // https://github.com/golang/go/issues/28756 - if vcfg.ImportMap["fmt"] == "" { - a1 := a.Deps[1] - vcfg.ImportMap["fmt"] = "fmt" - if a1.built != "" { - vcfg.PackageFile["fmt"] = a1.built + if vcfg.VetxOnly && !cfg.BuildA { + c := cache.Default() + if file, _, err := c.GetFile(key); err == nil { + a.built = file + return nil } - vcfg.Standard["fmt"] = true } - // During go test, ignore type-checking failures during vet. - // We only run vet if the compilation has succeeded, - // so at least for now assume the bug is in vet. - // We know of at least #18395. - // TODO(rsc,gri): Try to remove this for Go 1.11. - // - // Disabled 2018-04-20. Let's see if we can do without it. - // vcfg.SucceedOnTypecheckFailure = cfg.CmdName == "test" - js, err := json.MarshalIndent(vcfg, "", "\t") if err != nil { return fmt.Errorf("internal error marshaling vet config: %v", err) @@ -1063,14 +1090,12 @@ func (b *Builder) vet(a *Action) error { if tool == "" { tool = base.Tool("vet") } - runErr := b.run(a, p.Dir, p.ImportPath, env, cfg.BuildToolexec, tool, VetFlags, a.Objdir+"vet.cfg") + runErr := b.run(a, p.Dir, p.ImportPath, env, cfg.BuildToolexec, tool, vetFlags, a.Objdir+"vet.cfg") // If vet wrote export data, save it for input to future vets. if f, err := os.Open(vcfg.VetxOutput); err == nil { a.built = vcfg.VetxOutput - if c := cache.Default(); c != nil { - c.Put(key, f) - } + cache.Default().Put(key, f) f.Close() } @@ -1087,6 +1112,9 @@ func (b *Builder) linkActionID(a *Action) cache.ActionID { fmt.Fprintf(h, "buildmode %s goos %s goarch %s\n", cfg.BuildBuildmode, cfg.Goos, cfg.Goarch) fmt.Fprintf(h, "import %q\n", p.ImportPath) fmt.Fprintf(h, "omitdebug %v standard %v local %v prefix %q\n", p.Internal.OmitDebug, p.Standard, p.Internal.Local, p.Internal.LocalPrefix) + if cfg.BuildTrimpath { + fmt.Fprintln(h, "trimpath") + } // Toolchain-dependent configuration, shared with b.linkSharedActionID. b.printLinkerConfig(h, p) @@ -1128,21 +1156,16 @@ func (b *Builder) printLinkerConfig(h io.Writer, p *load.Package) { if p != nil { fmt.Fprintf(h, "linkflags %q\n", p.Internal.Ldflags) } - fmt.Fprintf(h, "GO$GOARCH=%s\n", os.Getenv("GO"+strings.ToUpper(cfg.BuildContext.GOARCH))) // GO386, GOARM, etc + + // GO386, GOARM, GOMIPS, etc. + key, val := cfg.GetArchEnv() + fmt.Fprintf(h, "%s=%s\n", key, val) // The linker writes source file paths that say GOROOT_FINAL. fmt.Fprintf(h, "GOROOT=%s\n", cfg.GOROOT_FINAL) - // TODO(rsc): Convince linker team not to add more magic environment variables, - // or perhaps restrict the environment variables passed to subprocesses. - magic := []string{ - "GO_EXTLINK_ENABLED", - } - for _, env := range magic { - if x := os.Getenv(env); x != "" { - fmt.Fprintf(h, "magic %s=%s\n", env, x) - } - } + // GO_EXTLINK_ENABLED controls whether the external linker is used. + fmt.Fprintf(h, "GO_EXTLINK_ENABLED=%s\n", cfg.Getenv("GO_EXTLINK_ENABLED")) // TODO(rsc): Do cgo settings and flags need to be included? // Or external linker settings and flags? @@ -1160,7 +1183,7 @@ func (b *Builder) printLinkerConfig(h io.Writer, p *load.Package) { // link is the action for linking a single command. // Note that any new influence on this logic must be reported in b.linkActionID above as well. func (b *Builder) link(a *Action) (err error) { - if b.useCache(a, a.Package, b.linkActionID(a), a.Package.Target) || b.IsCmdList { + if b.useCache(a, b.linkActionID(a), a.Package.Target) || b.IsCmdList { return nil } defer b.flushOutput(a) @@ -1174,6 +1197,10 @@ func (b *Builder) link(a *Action) (err error) { return err } + if err := allowInstall(a); err != nil { + return err + } + // make target directory dir, _ := filepath.Split(a.Target) if dir != "" { @@ -1315,7 +1342,7 @@ func (b *Builder) getPkgConfigFlags(p *load.Package) (cflags, ldflags []string, } } var out []byte - out, err = b.runOut(p.Dir, nil, b.PkgconfigCmd(), "--cflags", pcflags, "--", pkgs) + out, err = b.runOut(nil, p.Dir, nil, b.PkgconfigCmd(), "--cflags", pcflags, "--", pkgs) if err != nil { b.showOutput(nil, p.Dir, b.PkgconfigCmd()+" --cflags "+strings.Join(pcflags, " ")+" -- "+strings.Join(pkgs, " "), string(out)) b.Print(err.Error() + "\n") @@ -1330,7 +1357,7 @@ func (b *Builder) getPkgConfigFlags(p *load.Package) (cflags, ldflags []string, return nil, nil, err } } - out, err = b.runOut(p.Dir, nil, b.PkgconfigCmd(), "--libs", pcflags, "--", pkgs) + out, err = b.runOut(nil, p.Dir, nil, b.PkgconfigCmd(), "--libs", pcflags, "--", pkgs) if err != nil { b.showOutput(nil, p.Dir, b.PkgconfigCmd()+" --libs "+strings.Join(pcflags, " ")+" -- "+strings.Join(pkgs, " "), string(out)) b.Print(err.Error() + "\n") @@ -1348,6 +1375,10 @@ func (b *Builder) getPkgConfigFlags(p *load.Package) (cflags, ldflags []string, } func (b *Builder) installShlibname(a *Action) error { + if err := allowInstall(a); err != nil { + return err + } + // TODO: BuildN a1 := a.Deps[0] err := ioutil.WriteFile(a.Target, []byte(filepath.Base(a1.Target)+"\n"), 0666) @@ -1393,11 +1424,15 @@ func (b *Builder) linkSharedActionID(a *Action) cache.ActionID { } func (b *Builder) linkShared(a *Action) (err error) { - if b.useCache(a, nil, b.linkSharedActionID(a), a.Target) || b.IsCmdList { + if b.useCache(a, b.linkSharedActionID(a), a.Target) || b.IsCmdList { return nil } defer b.flushOutput(a) + if err := allowInstall(a); err != nil { + return err + } + if err := b.Mkdir(a.Objdir); err != nil { return err } @@ -1430,6 +1465,9 @@ func BuildInstallFunc(b *Builder, a *Action) (err error) { a1 := a.Deps[0] a.buildID = a1.buildID + if a.json != nil { + a.json.BuildID = a.buildID + } // If we are using the eventual install target as an up-to-date // cached copy of the thing we built, then there's no need to @@ -1460,8 +1498,12 @@ func BuildInstallFunc(b *Builder, a *Action) (err error) { // advertise it by touching the mtimes (usually the libraries are up // to date). if !a.buggyInstall && !b.IsCmdList { - now := time.Now() - os.Chtimes(a.Target, now, now) + if cfg.BuildN { + b.Showcmd("", "touch %s", a.Target) + } else if err := allowInstall(a); err == nil { + now := time.Now() + os.Chtimes(a.Target, now, now) + } } return nil } @@ -1472,6 +1514,9 @@ func BuildInstallFunc(b *Builder, a *Action) (err error) { a.built = a1.built return nil } + if err := allowInstall(a); err != nil { + return err + } if err := b.Mkdir(a.Objdir); err != nil { return err @@ -1501,6 +1546,13 @@ func BuildInstallFunc(b *Builder, a *Action) (err error) { return b.moveOrCopyFile(a.Target, a1.built, perm, false) } +// allowInstall returns a non-nil error if this invocation of the go command is +// allowed to install a.Target. +// +// (The build of cmd/go running under its own test is forbidden from installing +// to its original GOROOT.) +var allowInstall = func(*Action) error { return nil } + // cleanup removes a's object dir to keep the amount of // on-disk garbage down in a large build. On an operating system // with aggressive buffering, cleaning incrementally like @@ -1596,12 +1648,12 @@ func (b *Builder) copyFile(dst, src string, perm os.FileMode, force bool) error // Be careful about removing/overwriting dst. // Do not remove/overwrite if dst exists and is a directory - // or a non-object file. + // or a non-empty non-object file. if fi, err := os.Stat(dst); err == nil { if fi.IsDir() { return fmt.Errorf("build output %q already exists and is a directory", dst) } - if !force && fi.Mode().IsRegular() && !isObject(dst) { + if !force && fi.Mode().IsRegular() && fi.Size() != 0 && !isObject(dst) { return fmt.Errorf("build output %q already exists and is not an object file", dst) } } @@ -1626,7 +1678,7 @@ func (b *Builder) copyFile(dst, src string, perm os.FileMode, force bool) error df, err = os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) } if err != nil { - return err + return fmt.Errorf("copying %s: %w", src, err) // err should already refer to dst } _, err = io.Copy(df, sf) @@ -1649,25 +1701,6 @@ func (b *Builder) writeFile(file string, text []byte) error { return ioutil.WriteFile(file, text, 0666) } -// appendFile appends the text to file. -func (b *Builder) appendFile(file string, text []byte) error { - if cfg.BuildN || cfg.BuildX { - b.Showcmd("", "cat >>%s << 'EOF' # internal\n%sEOF", file, text) - } - if cfg.BuildN { - return nil - } - f, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) - if err != nil { - return err - } - defer f.Close() - if _, err = f.Write(text); err != nil { - return err - } - return f.Close() -} - // Install the cgo export header file, if there is one. func (b *Builder) installHeader(a *Action) error { src := a.Objdir + "_cgo_install.h" @@ -1683,6 +1716,10 @@ func (b *Builder) installHeader(a *Action) error { return nil } + if err := allowInstall(a); err != nil { + return err + } + dir, _ := filepath.Split(a.Target) if dir != "" { if err := b.Mkdir(dir); err != nil { @@ -1775,6 +1812,11 @@ func (b *Builder) fmtcmd(dir string, format string, args ...interface{}) string } if b.WorkDir != "" { cmd = strings.ReplaceAll(cmd, b.WorkDir, "$WORK") + escaped := strconv.Quote(b.WorkDir) + escaped = escaped[1 : len(escaped)-1] // strip quote characters + if escaped != b.WorkDir { + cmd = strings.ReplaceAll(cmd, escaped, "$WORK") + } } return cmd } @@ -1839,14 +1881,14 @@ func (b *Builder) showOutput(a *Action, dir, desc, out string) { // print this error. var errPrintedOutput = errors.New("already printed output - no need to show error") -var cgoLine = regexp.MustCompile(`\[[^\[\]]+\.(cgo1|cover)\.go:[0-9]+(:[0-9]+)?\]`) -var cgoTypeSigRe = regexp.MustCompile(`\b_C2?(type|func|var|macro)_\B`) +var cgoLine = lazyregexp.New(`\[[^\[\]]+\.(cgo1|cover)\.go:[0-9]+(:[0-9]+)?\]`) +var cgoTypeSigRe = lazyregexp.New(`\b_C2?(type|func|var|macro)_\B`) // run runs the command given by cmdline in the directory dir. // If the command fails, run prints information about the failure // and returns a non-nil error. func (b *Builder) run(a *Action, dir string, desc string, env []string, cmdargs ...interface{}) error { - out, err := b.runOut(dir, env, cmdargs...) + out, err := b.runOut(a, dir, env, cmdargs...) if len(out) > 0 { if desc == "" { desc = b.fmtcmd(dir, "%s", strings.Join(str.StringList(cmdargs...), " ")) @@ -1878,7 +1920,8 @@ func (b *Builder) processOutput(out []byte) string { // runOut runs the command given by cmdline in the directory dir. // It returns the command output and any errors that occurred. -func (b *Builder) runOut(dir string, env []string, cmdargs ...interface{}) ([]byte, error) { +// It accumulates execution time in a. +func (b *Builder) runOut(a *Action, dir string, env []string, cmdargs ...interface{}) ([]byte, error) { cmdline := str.StringList(cmdargs...) for _, arg := range cmdline { @@ -1917,8 +1960,19 @@ func (b *Builder) runOut(dir string, env []string, cmdargs ...interface{}) ([]by cleanup := passLongArgsInResponseFiles(cmd) defer cleanup() cmd.Dir = dir - cmd.Env = base.MergeEnvLists(env, base.EnvForDir(cmd.Dir, os.Environ())) + cmd.Env = base.AppendPWD(os.Environ(), cmd.Dir) + cmd.Env = append(cmd.Env, env...) + start := time.Now() err := cmd.Run() + if a != nil && a.json != nil { + aj := a.json + aj.Cmd = append(aj.Cmd, joinUnambiguously(cmdline)) + aj.CmdReal += time.Since(start) + if ps := cmd.ProcessState; ps != nil { + aj.CmdUser += ps.UserTime() + aj.CmdSys += ps.SystemTime() + } + } // err can be something like 'exit status 1'. // Add information about what program was running. @@ -1943,7 +1997,8 @@ func joinUnambiguously(a []string) string { q := strconv.Quote(s) // A gccgo command line can contain -( and -). // Make sure we quote them since they are special to the shell. - if s == "" || strings.ContainsAny(s, " ()") || len(q) > len(s)+2 { + // The trimpath argument can also contain > (part of =>) and ;. Quote those too. + if s == "" || strings.ContainsAny(s, " ()>;") || len(q) > len(s)+2 { buf.WriteString(q) } else { buf.WriteString(s) @@ -2112,10 +2167,42 @@ func (b *Builder) gfortran(a *Action, p *load.Package, workdir, out string, flag func (b *Builder) ccompile(a *Action, p *load.Package, outfile string, flags []string, file string, compiler []string) error { file = mkAbs(p.Dir, file) desc := p.ImportPath - if !filepath.IsAbs(outfile) { - outfile = filepath.Join(p.Dir, outfile) + outfile = mkAbs(p.Dir, outfile) + + // Elide source directory paths if -trimpath or GOROOT_FINAL is set. + // This is needed for source files (e.g., a .c file in a package directory). + // TODO(golang.org/issue/36072): cgo also generates files with #line + // directives pointing to the source directory. It should not generate those + // when -trimpath is enabled. + if b.gccSupportsFlag(compiler, "-fdebug-prefix-map=a=b") { + if cfg.BuildTrimpath { + // Keep in sync with Action.trimpath. + // The trimmed paths are a little different, but we need to trim in the + // same situations. + var from, toPath string + if m := p.Module; m != nil { + from = m.Dir + toPath = m.Path + "@" + m.Version + } else { + from = p.Dir + toPath = p.ImportPath + } + // -fdebug-prefix-map requires an absolute "to" path (or it joins the path + // with the working directory). Pick something that makes sense for the + // target platform. + var to string + if cfg.BuildContext.GOOS == "windows" { + to = filepath.Join(`\\_\_`, toPath) + } else { + to = filepath.Join("/_", toPath) + } + flags = append(flags[:len(flags):len(flags)], "-fdebug-prefix-map="+from+"="+to) + } else if p.Goroot && cfg.GOROOT_FINAL != cfg.GOROOT { + flags = append(flags[:len(flags):len(flags)], "-fdebug-prefix-map="+cfg.GOROOT+"="+cfg.GOROOT_FINAL) + } } - output, err := b.runOut(filepath.Dir(file), b.cCompilerEnv(), compiler, flags, "-o", outfile, "-c", filepath.Base(file)) + + output, err := b.runOut(a, filepath.Dir(file), b.cCompilerEnv(), compiler, flags, "-o", outfile, "-c", filepath.Base(file)) if len(output) > 0 { // On FreeBSD 11, when we pass -g to clang 3.8 it // invokes its internal assembler with -dwarf-version=2. @@ -2148,7 +2235,7 @@ func (b *Builder) ccompile(a *Action, p *load.Package, outfile string, flags []s } // gccld runs the gcc linker to create an executable from a set of object files. -func (b *Builder) gccld(p *load.Package, objdir, outfile string, flags []string, objs []string) error { +func (b *Builder) gccld(a *Action, p *load.Package, objdir, outfile string, flags []string, objs []string) error { var cmd []string if len(p.CXXFiles) > 0 || len(p.SwigCXXFiles) > 0 { cmd = b.GxxCmd(p.Dir, objdir) @@ -2158,16 +2245,37 @@ func (b *Builder) gccld(p *load.Package, objdir, outfile string, flags []string, cmdargs := []interface{}{cmd, "-o", outfile, objs, flags} dir := p.Dir - out, err := b.runOut(dir, b.cCompilerEnv(), cmdargs...) + out, err := b.runOut(a, dir, b.cCompilerEnv(), cmdargs...) if len(out) > 0 { // Filter out useless linker warnings caused by bugs outside Go. // See also cmd/link/internal/ld's hostlink method. var save [][]byte + var skipLines int for _, line := range bytes.SplitAfter(out, []byte("\n")) { // golang.org/issue/26073 - Apple Xcode bug if bytes.Contains(line, []byte("ld: warning: text-based stub file")) { continue } + + if skipLines > 0 { + skipLines-- + continue + } + + // Remove duplicate main symbol with runtime/cgo on AIX. + // With runtime/cgo, two main are available: + // One is generated by cgo tool with {return 0;}. + // The other one is the main calling runtime.rt0_go + // in runtime/cgo. + // The second can't be used by cgo programs because + // runtime.rt0_go is unknown to them. + // Therefore, we let ld remove this main version + // and used the cgo generated one. + if p.ImportPath == "runtime/cgo" && bytes.Contains(line, []byte("ld: 0711-224 WARNING: Duplicate symbol: .main")) { + skipLines = 1 + continue + } + save = append(save, line) } out = bytes.Join(save, nil) @@ -2183,8 +2291,8 @@ func (b *Builder) gccld(p *load.Package, objdir, outfile string, flags []string, // Grab these before main helpfully overwrites them. var ( - origCC = os.Getenv("CC") - origCXX = os.Getenv("CXX") + origCC = cfg.Getenv("CC") + origCXX = cfg.Getenv("CXX") ) // gccCmd returns a gcc command line prefix @@ -2216,7 +2324,7 @@ func (b *Builder) cxxExe() []string { // fcExe returns the FC compiler setting without all the extra flags we add implicitly. func (b *Builder) fcExe() []string { - return b.compilerExe(os.Getenv("FC"), "gfortran") + return b.compilerExe(cfg.Getenv("FC"), "gfortran") } // compilerExe returns the compiler to use given an @@ -2258,6 +2366,11 @@ func (b *Builder) compilerCmd(compiler []string, incdir, workdir string) []strin } } + if cfg.Goos == "aix" { + // mcmodel=large must always be enabled to allow large TOC. + a = append(a, "-mcmodel=large") + } + // disable ASCII art in clang errors, if possible if b.gccSupportsFlag(compiler, "-fno-caret-diagnostics") { a = append(a, "-fno-caret-diagnostics") @@ -2321,13 +2434,25 @@ func (b *Builder) gccSupportsFlag(compiler []string, flag string) bool { if b.flagCache == nil { b.flagCache = make(map[[2]string]bool) } + + tmp := os.DevNull + if runtime.GOOS == "windows" { + f, err := ioutil.TempFile(b.WorkDir, "") + if err != nil { + return false + } + f.Close() + tmp = f.Name() + defer os.Remove(tmp) + } + // We used to write an empty C file, but that gets complicated with // go build -n. We tried using a file that does not exist, but that // fails on systems with GCC version 4.2.1; that is the last GPLv2 // version of GCC, so some systems have frozen on it. // Now we pass an empty file on stdin, which should work at least for // GCC and clang. - cmdArgs := str.StringList(compiler, flag, "-c", "-x", "c", "-") + cmdArgs := str.StringList(compiler, flag, "-c", "-x", "c", "-", "-o", tmp) if cfg.BuildN || cfg.BuildX { b.Showcmd(b.WorkDir, "%s || true", joinUnambiguously(cmdArgs)) if cfg.BuildN { @@ -2336,7 +2461,8 @@ func (b *Builder) gccSupportsFlag(compiler []string, flag string) bool { } cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...) cmd.Dir = b.WorkDir - cmd.Env = base.MergeEnvLists([]string{"LC_ALL=C"}, base.EnvForDir(cmd.Dir, os.Environ())) + cmd.Env = base.AppendPWD(os.Environ(), cmd.Dir) + cmd.Env = append(cmd.Env, "LC_ALL=C") out, _ := cmd.CombinedOutput() // GCC says "unrecognized command line option". // clang says "unknown argument". @@ -2355,7 +2481,7 @@ func (b *Builder) gccArchArgs() []string { switch cfg.Goarch { case "386": return []string{"-m32"} - case "amd64", "amd64p32": + case "amd64": return []string{"-m64"} case "arm": return []string{"-marm"} // not thumb @@ -2376,7 +2502,7 @@ func (b *Builder) gccArchArgs() []string { // envList returns the value of the given environment variable broken // into fields, using the default value when the variable is empty. func envList(key, def string) []string { - v := os.Getenv(key) + v := cfg.Getenv(key) if v == "" { v = def } @@ -2413,7 +2539,7 @@ func buildFlags(name, defaults string, fromPackage []string, check func(string, return str.StringList(envList("CGO_"+name, defaults), fromPackage), nil } -var cgoRe = regexp.MustCompile(`[/\\:]`) +var cgoRe = lazyregexp.New(`[/\\:]`) func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgofiles, gccfiles, gxxfiles, mfiles, ffiles []string) (outGo, outObj []string, err error) { p := a.Package @@ -2433,7 +2559,7 @@ func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgo // Support gfortran out of the box and let others pass the correct link options // via CGO_LDFLAGS if len(ffiles) > 0 { - fc := os.Getenv("FC") + fc := cfg.Getenv("FC") if fc == "" { fc = "gfortran" } @@ -2487,8 +2613,7 @@ func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgo } if cfg.BuildToolchainName == "gccgo" { - switch cfg.Goarch { - case "386", "amd64": + if b.gccSupportsFlag([]string{BuildToolchain.compiler()}, "-fsplit-stack") { cgoCFLAGS = append(cgoCFLAGS, "-fsplit-stack") } cgoflags = append(cgoflags, "-gccgo") @@ -2615,7 +2740,7 @@ func (b *Builder) dynimport(a *Action, p *load.Package, objdir, importGo, cgoExe } ldflags = append(n, "-pie") } - if err := b.gccld(p, objdir, dynobj, ldflags, linkobj); err != nil { + if err := b.gccld(a, p, objdir, dynobj, ldflags, linkobj); err != nil { return err } @@ -2674,7 +2799,7 @@ var ( ) func (b *Builder) swigDoVersionCheck() error { - out, err := b.runOut("", nil, "swig", "-version") + out, err := b.runOut(nil, "", nil, "swig", "-version") if err != nil { return err } @@ -2829,7 +2954,7 @@ func (b *Builder) swigOne(a *Action, p *load.Package, file, objdir string, pcCFL args = append(args, "-c++") } - out, err := b.runOut(p.Dir, nil, "swig", args, file) + out, err := b.runOut(a, p.Dir, nil, "swig", args, file) if err != nil { if len(out) > 0 { if bytes.Contains(out, []byte("-intgosize")) || bytes.Contains(out, []byte("-cgo")) { @@ -2892,13 +3017,13 @@ func mkAbsFiles(dir string, files []string) []string { return abs } -// passLongArgsInResponseFiles modifies cmd on Windows such that, for +// passLongArgsInResponseFiles modifies cmd such that, for // certain programs, long arguments are passed in "response files", a // file on disk with the arguments, with one arg per line. An actual // argument starting with '@' means that the rest of the argument is // a filename of arguments to expand. // -// See Issue 18468. +// See issues 18468 (Windows) and 37768 (Darwin). func passLongArgsInResponseFiles(cmd *exec.Cmd) (cleanup func()) { cleanup = func() {} // no cleanup by default @@ -2936,11 +3061,6 @@ func passLongArgsInResponseFiles(cmd *exec.Cmd) (cleanup func()) { } func useResponseFile(path string, argLen int) bool { - // Unless we're on Windows, don't use response files. - if runtime.GOOS != "windows" { - return false - } - // Unless the program uses objabi.Flagparse, which understands // response files, don't use response files. // TODO: do we need more commands? asm? cgo? For now, no. @@ -2953,6 +3073,8 @@ func useResponseFile(path string, argLen int) bool { // Windows has a limit of 32 KB arguments. To be conservative and not // worry about whether that includes spaces or not, just use 30 KB. + // Darwin's limit is less clear. The OS claims 256KB, but we've seen + // failures with arglen as small as 50KB. if argLen > (30 << 10) { return true } diff --git a/cmd/go/_internal_/work/gc.go b/cmd/go/_internal_/work/gc.go index 791a1b7..5f8b1b0 100644 --- a/cmd/go/_internal_/work/gc.go +++ b/cmd/go/_internal_/work/gc.go @@ -21,6 +21,7 @@ import ( "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/load" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/str" "github.com/dependabot/gomodules-extracted/cmd/_internal_/objabi" + "github.com/dependabot/gomodules-extracted/cmd/_internal_/sys" "crypto/sha1" ) @@ -36,6 +37,17 @@ func (gcToolchain) linker() string { return base.Tool("link") } +func pkgPath(a *Action) string { + p := a.Package + ppath := p.ImportPath + if cfg.BuildBuildmode == "plugin" { + ppath = pluginPath(a) + } else if p.Name == "main" && !p.Internal.ForceLibrary { + ppath = "main" + } + return ppath +} + func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, symabis string, asmhdr bool, gofiles []string) (ofile string, output []byte, err error) { p := a.Package objdir := a.Objdir @@ -46,12 +58,7 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, s ofile = objdir + out } - pkgpath := p.ImportPath - if cfg.BuildBuildmode == "plugin" { - pkgpath = pluginPath(a) - } else if p.Name == "main" && !p.Internal.ForceLibrary { - pkgpath = "main" - } + pkgpath := pkgPath(a) gcargs := []string{"-p", pkgpath} if p.Module != nil && p.Module.GoVersion != "" && allowedVersion(p.Module.GoVersion) { gcargs = append(gcargs, "-lang=go"+p.Module.GoVersion) @@ -91,8 +98,7 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, s if a.buildID != "" { gcargs = append(gcargs, "-buildid", a.buildID) } - platform := cfg.Goos + "/" + cfg.Goarch - if p.Internal.OmitDebug || platform == "nacl/amd64p32" || cfg.Goos == "plan9" || cfg.Goarch == "wasm" { + if p.Internal.OmitDebug || cfg.Goos == "plan9" || cfg.Goarch == "wasm" { gcargs = append(gcargs, "-dwarf=false") } if strings.HasPrefix(runtimeVersion, "go1") && !strings.Contains(os.Args[0], "go_bootstrap") { @@ -116,7 +122,7 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, s } } - args := []interface{}{cfg.BuildToolexec, base.Tool("compile"), "-o", ofile, "-trimpath", trimDir(a.Objdir), gcflags, gcargs, "-D", p.Internal.LocalPrefix} + args := []interface{}{cfg.BuildToolexec, base.Tool("compile"), "-o", ofile, "-trimpath", a.trimpath(), gcflags, gcargs, "-D", p.Internal.LocalPrefix} if importcfg != nil { if err := b.writeFile(objdir+"importcfg", importcfg); err != nil { return "", nil, err @@ -139,7 +145,7 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, s args = append(args, mkAbs(p.Dir, f)) } - output, err = b.runOut(p.Dir, nil, args...) + output, err = b.runOut(a, p.Dir, nil, args...) return ofile, output, err } @@ -162,7 +168,7 @@ func gcBackendConcurrency(gcflags []string) int { CheckFlags: for _, flag := range gcflags { // Concurrent compilation is presumed incompatible with any gcflags, - // except for a small whitelist of commonly used flags. + // except for known commonly used flags. // If the user knows better, they can manually add their own -c to the gcflags. switch flag { case "-N", "-l", "-S", "-B", "-C", "-I": @@ -214,17 +220,38 @@ CheckFlags: return c } -func trimDir(dir string) string { - if len(dir) > 1 && dir[len(dir)-1] == filepath.Separator { - dir = dir[:len(dir)-1] +// trimpath returns the -trimpath argument to use +// when compiling the action. +func (a *Action) trimpath() string { + // Keep in sync with Builder.ccompile + // The trimmed paths are a little different, but we need to trim in the + // same situations. + + // Strip the object directory entirely. + objdir := a.Objdir + if len(objdir) > 1 && objdir[len(objdir)-1] == filepath.Separator { + objdir = objdir[:len(objdir)-1] } - return dir + rewrite := objdir + "=>" + + // For "go build -trimpath", rewrite package source directory + // to a file system-independent path (just the import path). + if cfg.BuildTrimpath { + if m := a.Package.Module; m != nil && m.Version != "" { + rewrite += ";" + a.Package.Dir + "=>" + m.Path + "@" + m.Version + strings.TrimPrefix(a.Package.ImportPath, m.Path) + } else { + rewrite += ";" + a.Package.Dir + "=>" + a.Package.ImportPath + } + } + + return rewrite } func asmArgs(a *Action, p *load.Package) []interface{} { // Add -I pkg/GOOS_GOARCH so #include "textflag.h" works in .s files. inc := filepath.Join(cfg.GOROOT, "pkg", "include") - args := []interface{}{cfg.BuildToolexec, base.Tool("asm"), "-trimpath", trimDir(a.Objdir), "-I", a.Objdir, "-I", inc, "-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch, forcedAsmflags, p.Internal.Asmflags} + pkgpath := pkgPath(a) + args := []interface{}{cfg.BuildToolexec, base.Tool("asm"), "-p", pkgpath, "-trimpath", a.trimpath(), "-I", a.Objdir, "-I", inc, "-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch, forcedAsmflags, p.Internal.Asmflags} if p.ImportPath == "runtime" && cfg.Goarch == "386" { for _, arg := range forcedAsmflags { if arg == "-dynlink" { @@ -292,55 +319,6 @@ func (gcToolchain) symabis(b *Builder, a *Action, sfiles []string) (string, erro } } - // Gather known cross-package references from assembly code. - var otherPkgs []string - if p.ImportPath == "runtime" { - // Assembly in the following packages references - // symbols in runtime. - otherPkgs = []string{"syscall", "internal/syscall/unix", "runtime/cgo"} - } else if p.ImportPath == "runtime/internal/atomic" { - // sync/atomic is an assembly wrapper around - // runtime/internal/atomic. - otherPkgs = []string{"sync/atomic"} - } - for _, p2name := range otherPkgs { - p2 := load.LoadPackage(p2name, &load.ImportStack{}) - if len(p2.SFiles) == 0 { - continue - } - - symabis2 := a.Objdir + "symabis2" - if err := mkSymabis(p2, p2.SFiles, symabis2); err != nil { - return "", err - } - - // Filter out just the symbol refs and append them to - // the symabis file. - if cfg.BuildN { - // -x will print the lines from symabis2 that are actually appended - // to symabis. With -n, we don't know what those lines will be. - b.Showcmd("", `grep '^ref' <%s | grep -v '^ref\s*""\.' >>%s`, symabis2, a.Objdir+"symabis") - continue - } - abis2, err := ioutil.ReadFile(symabis2) - if err != nil { - return "", err - } - var refs bytes.Buffer - for _, line := range strings.Split(string(abis2), "\n") { - fs := strings.Fields(line) - if len(fs) >= 2 && fs[0] == "ref" && !strings.HasPrefix(fs[1], `"".`) { - fmt.Fprintf(&refs, "%s\n", line) - } - } - if refs.Len() != 0 { - symabis = a.Objdir + "symabis" - if err := b.appendFile(symabis, refs.Bytes()); err != nil { - return "", err - } - } - } - return symabis, nil } @@ -493,7 +471,21 @@ func pluginPath(a *Action) string { return p.ImportPath } h := sha1.New() - fmt.Fprintf(h, "build ID: %s\n", a.buildID) + buildID := a.buildID + if a.Mode == "link" { + // For linking, use the main package's build ID instead of + // the binary's build ID, so it is the same hash used in + // compiling and linking. + // When compiling, we use actionID/actionID (instead of + // actionID/contentID) as a temporary build ID to compute + // the hash. Do the same here. (See buildid.go:useCache) + // The build ID matters because it affects the overall hash + // in the plugin's pseudo-import path returned below. + // We need to use the same import path when compiling and linking. + id := strings.Split(buildID, buildIDSeparator) + buildID = id[1] + buildIDSeparator + id[1] + } + fmt.Fprintf(h, "build ID: %s\n", buildID) for _, file := range str.StringList(p.GoFiles, p.CgoFiles, p.SFiles) { data, err := ioutil.ReadFile(filepath.Join(p.Dir, file)) if err != nil { @@ -525,7 +517,13 @@ func (gcToolchain) ld(b *Builder, root *Action, out, importcfg, mainpkg string) // Store BuildID inside toolchain binaries as a unique identifier of the // tool being run, for use by content-based staleness determination. if root.Package.Goroot && strings.HasPrefix(root.Package.ImportPath, "cmd/") { - ldflags = append(ldflags, "-X=cmd/internal/objabi.buildID="+root.buildID) + // External linking will include our build id in the external + // linker's build id, which will cause our build id to not + // match the next time the tool is built. + // Rely on the external build id instead. + if !sys.MustLinkExternal(cfg.Goos, cfg.Goarch) { + ldflags = append(ldflags, "-X=cmd/internal/objabi.buildID="+root.buildID) + } } // If the user has not specified the -extld option, then specify the @@ -560,7 +558,11 @@ func (gcToolchain) ld(b *Builder, root *Action, out, importcfg, mainpkg string) dir, out = filepath.Split(out) } - return b.run(root, dir, root.Package.ImportPath, nil, cfg.BuildToolexec, base.Tool("link"), "-o", out, "-importcfg", importcfg, ldflags, mainpkg) + env := []string{} + if cfg.BuildTrimpath { + env = append(env, "GOROOT_FINAL=go") + } + return b.run(root, dir, root.Package.ImportPath, env, cfg.BuildToolexec, base.Tool("link"), "-o", out, "-importcfg", importcfg, ldflags, mainpkg) } func (gcToolchain) ldShared(b *Builder, root *Action, toplevelactions []*Action, out, importcfg string, allactions []*Action) error { diff --git a/cmd/go/_internal_/work/gccgo.go b/cmd/go/_internal_/work/gccgo.go index 8bffe1f..7e7f959 100644 --- a/cmd/go/_internal_/work/gccgo.go +++ b/cmd/go/_internal_/work/gccgo.go @@ -26,7 +26,7 @@ var GccgoName, GccgoBin string var gccgoErr error func init() { - GccgoName = os.Getenv("GCCGO") + GccgoName = cfg.Getenv("GCCGO") if GccgoName == "" { GccgoName = "gccgo" } @@ -44,7 +44,7 @@ func (gccgoToolchain) linker() string { } func (gccgoToolchain) ar() string { - ar := os.Getenv("AR") + ar := cfg.Getenv("AR") if ar == "" { ar = "ar" } @@ -56,7 +56,8 @@ func checkGccgoBin() { return } fmt.Fprintf(os.Stderr, "cmd/go: gccgo: %s\n", gccgoErr) - os.Exit(2) + base.SetExitStatus(2) + base.Exit() } func (tools gccgoToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, symabis string, asmhdr bool, gofiles []string) (ofile string, output []byte, err error) { @@ -66,6 +67,8 @@ func (tools gccgoToolchain) gc(b *Builder, a *Action, archive string, importcfg ofile = objdir + out gcargs := []string{"-g"} gcargs = append(gcargs, b.gccArchArgs()...) + gcargs = append(gcargs, "-fdebug-prefix-map="+b.WorkDir+"=/tmp/go-build") + gcargs = append(gcargs, "-gno-record-gcc-switches") if pkgpath := gccgoPkgpath(p); pkgpath != "" { gcargs = append(gcargs, "-fgo-pkgpath="+pkgpath) } @@ -88,12 +91,16 @@ func (tools gccgoToolchain) gc(b *Builder, a *Action, archive string, importcfg args = append(args, "-I", root) } } + if cfg.BuildTrimpath && b.gccSupportsFlag(args[:1], "-ffile-prefix-map=a=b") { + args = append(args, "-ffile-prefix-map="+base.Cwd+"=.") + args = append(args, "-ffile-prefix-map="+b.WorkDir+"=/tmp/go-build") + } args = append(args, a.Package.Internal.Gccgoflags...) for _, f := range gofiles { args = append(args, mkAbs(p.Dir, f)) } - output, err = b.runOut(p.Dir, nil, args) + output, err = b.runOut(a, p.Dir, nil, args) return ofile, output, err } @@ -202,11 +209,21 @@ func (tools gccgoToolchain) pack(b *Builder, a *Action, afile string, ofiles []s if cfg.Goos == "aix" && cfg.Goarch == "ppc64" { // AIX puts both 32-bit and 64-bit objects in the same archive. // Tell the AIX "ar" command to only care about 64-bit objects. - // AIX "ar" command does not know D option. arArgs = []string{"-X64"} } + absAfile := mkAbs(objdir, afile) + // Try with D modifier first, then without if that fails. + output, err := b.runOut(a, p.Dir, nil, tools.ar(), arArgs, "rcD", absAfile, absOfiles) + if err != nil { + return b.run(a, p.Dir, p.ImportPath, nil, tools.ar(), arArgs, "rc", absAfile, absOfiles) + } + + if len(output) > 0 { + // Show the output if there is any even without errors. + b.showOutput(a, p.Dir, p.ImportPath, b.processOutput(output)) + } - return b.run(a, p.Dir, p.ImportPath, nil, tools.ar(), arArgs, "rc", mkAbs(objdir, afile), absOfiles) + return nil } func (tools gccgoToolchain) link(b *Builder, root *Action, out, importcfg string, allactions []*Action, buildmode, desc string) error { @@ -248,6 +265,13 @@ func (tools gccgoToolchain) link(b *Builder, root *Action, out, importcfg string return nil } + var arArgs []string + if cfg.Goos == "aix" && cfg.Goarch == "ppc64" { + // AIX puts both 32-bit and 64-bit objects in the same archive. + // Tell the AIX "ar" command to only care about 64-bit objects. + arArgs = []string{"-X64"} + } + newID := 0 readAndRemoveCgoFlags := func(archive string) (string, error) { newID++ @@ -265,11 +289,11 @@ func (tools gccgoToolchain) link(b *Builder, root *Action, out, importcfg string return "", nil } } - err := b.run(root, root.Objdir, desc, nil, tools.ar(), "x", newArchive, "_cgo_flags") + err := b.run(root, root.Objdir, desc, nil, tools.ar(), arArgs, "x", newArchive, "_cgo_flags") if err != nil { return "", err } - err = b.run(root, ".", desc, nil, tools.ar(), "d", newArchive, "_cgo_flags") + err = b.run(root, ".", desc, nil, tools.ar(), arArgs, "d", newArchive, "_cgo_flags") if err != nil { return "", err } @@ -312,7 +336,7 @@ func (tools gccgoToolchain) link(b *Builder, root *Action, out, importcfg string } if haveShlib[filepath.Base(a.Target)] { - // This is a shared library we want to link againt. + // This is a shared library we want to link against. if !addedShlib[a.Target] { shlibs = append(shlibs, a.Target) addedShlib[a.Target] = true @@ -407,6 +431,7 @@ func (tools gccgoToolchain) link(b *Builder, root *Action, out, importcfg string } var realOut string + goLibBegin := str.StringList(wholeArchive, "-lgolibbegin", noWholeArchive) switch buildmode { case "exe": if usesCgo && cfg.Goos == "linux" { @@ -428,7 +453,8 @@ func (tools gccgoToolchain) link(b *Builder, root *Action, out, importcfg string // split-stack and non-split-stack code in a single -r // link, and libgo picks up non-split-stack code from // libffi. - ldflags = append(ldflags, "-Wl,-r", "-nostdlib", "-Wl,--whole-archive", "-lgolibbegin", "-Wl,--no-whole-archive") + ldflags = append(ldflags, "-Wl,-r", "-nostdlib") + ldflags = append(ldflags, goLibBegin...) if nopie := b.gccNoPie([]string{tools.linker()}); nopie != "" { ldflags = append(ldflags, nopie) @@ -443,7 +469,10 @@ func (tools gccgoToolchain) link(b *Builder, root *Action, out, importcfg string out = out + ".o" case "c-shared": - ldflags = append(ldflags, "-shared", "-nostdlib", "-Wl,--whole-archive", "-lgolibbegin", "-Wl,--no-whole-archive", "-lgo", "-lgcc_s", "-lgcc", "-lc", "-lgcc") + ldflags = append(ldflags, "-shared", "-nostdlib") + ldflags = append(ldflags, goLibBegin...) + ldflags = append(ldflags, "-lgo", "-lgcc_s", "-lgcc", "-lc", "-lgcc") + case "shared": if cfg.Goos != "aix" { ldflags = append(ldflags, "-zdefs") @@ -463,7 +492,7 @@ func (tools gccgoToolchain) link(b *Builder, root *Action, out, importcfg string ldflags = append(ldflags, "-lobjc") } if fortran { - fc := os.Getenv("FC") + fc := cfg.Getenv("FC") if fc == "" { fc = "gfortran" } @@ -481,7 +510,7 @@ func (tools gccgoToolchain) link(b *Builder, root *Action, out, importcfg string switch buildmode { case "c-archive": - if err := b.run(root, ".", desc, nil, tools.ar(), "rc", realOut, out); err != nil { + if err := b.run(root, ".", desc, nil, tools.ar(), arArgs, "rc", realOut, out); err != nil { return err } } @@ -505,12 +534,21 @@ func (tools gccgoToolchain) cc(b *Builder, a *Action, ofile, cfile string) error if pkgpath := gccgoCleanPkgpath(p); pkgpath != "" { defs = append(defs, `-D`, `GOPKGPATH="`+pkgpath+`"`) } - switch cfg.Goarch { - case "386", "amd64": + compiler := envList("CC", cfg.DefaultCC(cfg.Goos, cfg.Goarch)) + if b.gccSupportsFlag(compiler, "-fsplit-stack") { defs = append(defs, "-fsplit-stack") } defs = tools.maybePIC(defs) - return b.run(a, p.Dir, p.ImportPath, nil, envList("CC", cfg.DefaultCC(cfg.Goos, cfg.Goarch)), "-Wall", "-g", + if b.gccSupportsFlag(compiler, "-ffile-prefix-map=a=b") { + defs = append(defs, "-ffile-prefix-map="+base.Cwd+"=.") + defs = append(defs, "-ffile-prefix-map="+b.WorkDir+"=/tmp/go-build") + } else if b.gccSupportsFlag(compiler, "-fdebug-prefix-map=a=b") { + defs = append(defs, "-fdebug-prefix-map="+b.WorkDir+"=/tmp/go-build") + } + if b.gccSupportsFlag(compiler, "-gno-record-gcc-switches") { + defs = append(defs, "-gno-record-gcc-switches") + } + return b.run(a, p.Dir, p.ImportPath, nil, compiler, "-Wall", "-g", "-I", a.Objdir, "-I", inc, "-o", ofile, defs, "-c", cfile) } diff --git a/cmd/go/_internal_/work/init.go b/cmd/go/_internal_/work/init.go index 3cbad6b..990f59d 100644 --- a/cmd/go/_internal_/work/init.go +++ b/cmd/go/_internal_/work/init.go @@ -10,11 +10,13 @@ import ( "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/load" + "github.com/dependabot/gomodules-extracted/cmd/_internal_/objabi" "github.com/dependabot/gomodules-extracted/cmd/_internal_/sys" "flag" "fmt" "os" "path/filepath" + "runtime" "strings" ) @@ -29,10 +31,25 @@ func BuildInit() { p, err := filepath.Abs(cfg.BuildPkgdir) if err != nil { fmt.Fprintf(os.Stderr, "go %s: evaluating -pkgdir: %v\n", flag.Args()[0], err) - os.Exit(2) + base.SetExitStatus(2) + base.Exit() } cfg.BuildPkgdir = p } + + // For each experiment that has been enabled in the toolchain, define a + // build tag with the same name but prefixed by "goexperiment." which can be + // used for compiling alternative files for the experiment. This allows + // changes for the experiment, like extra struct fields in the runtime, + // without affecting the base non-experiment code at all. [2:] strips the + // leading "X:" from objabi.Expstring(). + exp := objabi.Expstring()[2:] + if exp != "none" { + experiments := strings.Split(exp, ",") + for _, expt := range experiments { + cfg.BuildContext.BuildTags = append(cfg.BuildContext.BuildTags, "goexperiment."+expt) + } + } } func instrumentInit() { @@ -41,27 +58,41 @@ func instrumentInit() { } if cfg.BuildRace && cfg.BuildMSan { fmt.Fprintf(os.Stderr, "go %s: may not use -race and -msan simultaneously\n", flag.Args()[0]) - os.Exit(2) + base.SetExitStatus(2) + base.Exit() } if cfg.BuildMSan && !sys.MSanSupported(cfg.Goos, cfg.Goarch) { fmt.Fprintf(os.Stderr, "-msan is not supported on %s/%s\n", cfg.Goos, cfg.Goarch) - os.Exit(2) + base.SetExitStatus(2) + base.Exit() } if cfg.BuildRace { if !sys.RaceDetectorSupported(cfg.Goos, cfg.Goarch) { fmt.Fprintf(os.Stderr, "go %s: -race is only supported on linux/amd64, linux/ppc64le, linux/arm64, freebsd/amd64, netbsd/amd64, darwin/amd64 and windows/amd64\n", flag.Args()[0]) - os.Exit(2) + base.SetExitStatus(2) + base.Exit() } } mode := "race" if cfg.BuildMSan { mode = "msan" + // MSAN does not support non-PIE binaries on ARM64. + // See issue #33712 for details. + if cfg.Goos == "linux" && cfg.Goarch == "arm64" && cfg.BuildBuildmode == "default" { + cfg.BuildBuildmode = "pie" + } } modeFlag := "-" + mode if !cfg.BuildContext.CgoEnabled { - fmt.Fprintf(os.Stderr, "go %s: %s requires cgo; enable cgo by setting CGO_ENABLED=1\n", flag.Args()[0], modeFlag) - os.Exit(2) + if runtime.GOOS != cfg.Goos || runtime.GOARCH != cfg.Goarch { + fmt.Fprintf(os.Stderr, "go %s: %s requires cgo\n", flag.Args()[0], modeFlag) + } else { + fmt.Fprintf(os.Stderr, "go %s: %s requires cgo; enable cgo by setting CGO_ENABLED=1\n", flag.Args()[0], modeFlag) + } + + base.SetExitStatus(2) + base.Exit() } forcedGcflags = append(forcedGcflags, modeFlag) forcedLdflags = append(forcedLdflags, modeFlag) @@ -76,7 +107,11 @@ func instrumentInit() { func buildModeInit() { gccgo := cfg.BuildToolchainName == "gccgo" var codegenArg string - platform := cfg.Goos + "/" + cfg.Goarch + + // Configure the build mode first, then verify that it is supported. + // That way, if the flag is completely bogus we will prefer to error out with + // "-buildmode=%s not supported" instead of naming the specific platform. + switch cfg.BuildBuildmode { case "archive": pkgsFilter = pkgsNotMain @@ -85,20 +120,18 @@ func buildModeInit() { if gccgo { codegenArg = "-fPIC" } else { - switch platform { - case "darwin/arm", "darwin/arm64": - codegenArg = "-shared" - default: - switch cfg.Goos { - case "dragonfly", "freebsd", "linux", "netbsd", "openbsd", "solaris": - if platform == "linux/ppc64" { - base.Fatalf("-buildmode=c-archive not supported on %s\n", platform) - } - // Use -shared so that the result is - // suitable for inclusion in a PIE or - // shared library. + switch cfg.Goos { + case "darwin": + switch cfg.Goarch { + case "arm64": codegenArg = "-shared" } + + case "dragonfly", "freebsd", "illumos", "linux", "netbsd", "openbsd", "solaris": + // Use -shared so that the result is + // suitable for inclusion in a PIE or + // shared library. + codegenArg = "-shared" } } cfg.ExeSuffix = ".a" @@ -108,27 +141,27 @@ func buildModeInit() { if gccgo { codegenArg = "-fPIC" } else { - switch platform { - case "linux/amd64", "linux/arm", "linux/arm64", "linux/386", "linux/ppc64le", "linux/s390x", - "android/amd64", "android/arm", "android/arm64", "android/386", - "freebsd/amd64": + switch cfg.Goos { + case "linux", "android", "freebsd": codegenArg = "-shared" - case "darwin/amd64", "darwin/386": - case "windows/amd64", "windows/386": + case "windows": // Do not add usual .exe suffix to the .dll file. cfg.ExeSuffix = "" - default: - base.Fatalf("-buildmode=c-shared not supported on %s\n", platform) } } ldBuildmode = "c-shared" case "default": - switch platform { - case "android/arm", "android/arm64", "android/amd64", "android/386": + switch cfg.Goos { + case "android": codegenArg = "-shared" ldBuildmode = "pie" - case "darwin/arm", "darwin/arm64": - codegenArg = "-shared" + case "windows": + ldBuildmode = "pie" + case "darwin": + switch cfg.Goarch { + case "arm64": + codegenArg = "-shared" + } fallthrough default: ldBuildmode = "exe" @@ -152,15 +185,10 @@ func buildModeInit() { if gccgo { codegenArg = "-fPIE" } else { - switch platform { - case "linux/386", "linux/amd64", "linux/arm", "linux/arm64", "linux/ppc64le", "linux/s390x", - "android/amd64", "android/arm", "android/arm64", "android/386", - "freebsd/amd64": - codegenArg = "-shared" - case "darwin/amd64": - codegenArg = "-shared" + switch cfg.Goos { + case "aix", "windows": default: - base.Fatalf("-buildmode=pie not supported on %s\n", platform) + codegenArg = "-shared" } } ldBuildmode = "pie" @@ -169,11 +197,6 @@ func buildModeInit() { if gccgo { codegenArg = "-fPIC" } else { - switch platform { - case "linux/386", "linux/amd64", "linux/arm", "linux/arm64", "linux/ppc64le", "linux/s390x": - default: - base.Fatalf("-buildmode=shared not supported on %s\n", platform) - } codegenArg = "-dynlink" } if cfg.BuildO != "" { @@ -185,15 +208,6 @@ func buildModeInit() { if gccgo { codegenArg = "-fPIC" } else { - switch platform { - case "linux/amd64", "linux/arm", "linux/arm64", "linux/386", "linux/s390x", "linux/ppc64le", - "android/amd64", "android/arm", "android/arm64", "android/386": - case "darwin/amd64": - // Skip DWARF generation due to #21647 - forcedLdflags = append(forcedLdflags, "-w") - default: - base.Fatalf("-buildmode=plugin not supported on %s\n", platform) - } codegenArg = "-dynlink" } cfg.ExeSuffix = ".so" @@ -201,17 +215,21 @@ func buildModeInit() { default: base.Fatalf("buildmode=%s not supported", cfg.BuildBuildmode) } + + if !sys.BuildModeSupported(cfg.BuildToolchainName, cfg.BuildBuildmode, cfg.Goos, cfg.Goarch) { + base.Fatalf("-buildmode=%s not supported on %s/%s\n", cfg.BuildBuildmode, cfg.Goos, cfg.Goarch) + } + if cfg.BuildLinkshared { + if !sys.BuildModeSupported(cfg.BuildToolchainName, "shared", cfg.Goos, cfg.Goarch) { + base.Fatalf("-linkshared not supported on %s/%s\n", cfg.Goos, cfg.Goarch) + } if gccgo { codegenArg = "-fPIC" } else { - switch platform { - case "linux/386", "linux/amd64", "linux/arm", "linux/arm64", "linux/ppc64le", "linux/s390x": - forcedAsmflags = append(forcedAsmflags, "-D=GOBUILDMODE_shared=1") - default: - base.Fatalf("-linkshared not supported on %s\n", platform) - } + forcedAsmflags = append(forcedAsmflags, "-D=GOBUILDMODE_shared=1") codegenArg = "-dynlink" + forcedGcflags = append(forcedGcflags, "-linkshared") // TODO(mwhudson): remove -w when that gets fixed in linker. forcedLdflags = append(forcedLdflags, "-linkshared", "-w") } @@ -235,12 +253,20 @@ func buildModeInit() { switch cfg.BuildMod { case "": // ok - case "readonly", "vendor": - if load.ModLookup == nil && !inGOFLAGS("-mod") { + case "readonly", "vendor", "mod": + if !cfg.ModulesEnabled && !inGOFLAGS("-mod") { base.Fatalf("build flag -mod=%s only valid when using modules", cfg.BuildMod) } default: - base.Fatalf("-mod=%s not supported (can be '', 'readonly', or 'vendor')", cfg.BuildMod) + base.Fatalf("-mod=%s not supported (can be '', 'mod', 'readonly', or 'vendor')", cfg.BuildMod) + } + if !cfg.ModulesEnabled { + if cfg.ModCacheRW && !inGOFLAGS("-modcacherw") { + base.Fatalf("build flag -modcacherw only valid when using modules") + } + if cfg.ModFile != "" && !inGOFLAGS("-mod") { + base.Fatalf("build flag -modfile only valid when using modules") + } } } diff --git a/cmd/go/_internal_/work/security.go b/cmd/go/_internal_/work/security.go index 7577bc4..8df8acc 100644 --- a/cmd/go/_internal_/work/security.go +++ b/cmd/go/_internal_/work/security.go @@ -30,17 +30,20 @@ package work import ( - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/load" "fmt" - "os" + "github.com/dependabot/gomodules-extracted/_internal_/lazyregexp" "regexp" "strings" + + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/load" ) -var re = regexp.MustCompile +var re = lazyregexp.New -var validCompilerFlags = []*regexp.Regexp{ +var validCompilerFlags = []*lazyregexp.Regexp{ re(`-D([A-Za-z_].*)`), + re(`-U([A-Za-z_]*)`), re(`-F([^@\-].*)`), re(`-I([^@\-].*)`), re(`-O`), @@ -49,6 +52,7 @@ var validCompilerFlags = []*regexp.Regexp{ re(`-W([^@,]+)`), // -Wall but not -Wa,-foo. re(`-Wa,-mbig-obj`), re(`-Wp,-D([A-Za-z_].*)`), + re(`-Wp,-U([A-Za-z_]*)`), re(`-ansi`), re(`-f(no-)?asynchronous-unwind-tables`), re(`-f(no-)?blocks`), @@ -100,6 +104,10 @@ var validCompilerFlags = []*regexp.Regexp{ re(`-mmacosx-(.+)`), re(`-mios-simulator-version-min=(.+)`), re(`-miphoneos-version-min=(.+)`), + re(`-mtvos-simulator-version-min=(.+)`), + re(`-mtvos-version-min=(.+)`), + re(`-mwatchos-simulator-version-min=(.+)`), + re(`-mwatchos-version-min=(.+)`), re(`-mnop-fun-dllimport`), re(`-m(no-)?sse[0-9.]*`), re(`-m(no-)?ssse3`), @@ -121,6 +129,7 @@ var validCompilerFlags = []*regexp.Regexp{ var validCompilerFlagsWithNextArg = []string{ "-arch", "-D", + "-U", "-I", "-framework", "-isysroot", @@ -130,7 +139,7 @@ var validCompilerFlagsWithNextArg = []string{ "-x", } -var validLinkerFlags = []*regexp.Regexp{ +var validLinkerFlags = []*lazyregexp.Regexp{ re(`-F([^@\-].*)`), re(`-l([^@\-].*)`), re(`-L([^@\-].*)`), @@ -166,6 +175,7 @@ var validLinkerFlags = []*regexp.Regexp{ re(`-Wl,--(no-)?allow-shlib-undefined`), re(`-Wl,--(no-)?as-needed`), re(`-Wl,-Bdynamic`), + re(`-Wl,-berok`), re(`-Wl,-Bstatic`), re(`-WL,-O([^@,\-][^,]*)?`), re(`-Wl,-d[ny]`), @@ -174,9 +184,13 @@ var validLinkerFlags = []*regexp.Regexp{ re(`-Wl,--enable-new-dtags`), re(`-Wl,--end-group`), re(`-Wl,--(no-)?export-dynamic`), + re(`-Wl,-E`), re(`-Wl,-framework,[^,@\-][^,]+`), + re(`-Wl,--hash-style=(sysv|gnu|both)`), re(`-Wl,-headerpad_max_install_names`), re(`-Wl,--no-undefined`), + re(`-Wl,-R([^@\-][^,@]*$)`), + re(`-Wl,--just-symbols[=,]([^,@\-][^,@]+)`), re(`-Wl,-rpath(-link)?[=,]([^,@\-][^,]+)`), re(`-Wl,-s`), re(`-Wl,-search_paths_first`), @@ -188,6 +202,7 @@ var validLinkerFlags = []*regexp.Regexp{ re(`-Wl,-undefined[=,]([^,@\-][^,]+)`), re(`-Wl,-?-unresolved-symbols=[^,]+`), re(`-Wl,--(no-)?warn-([^,]+)`), + re(`-Wl,-?-wrap[=,][^,@\-][^,]*`), re(`-Wl,-z,(no)?execstack`), re(`-Wl,-z,relro`), @@ -206,6 +221,8 @@ var validLinkerFlagsWithNextArg = []string{ "-target", "-Wl,-framework", "-Wl,-rpath", + "-Wl,-R", + "-Wl,--just-symbols", "-Wl,-undefined", } @@ -217,20 +234,20 @@ func checkLinkerFlags(name, source string, list []string) error { return checkFlags(name, source, list, validLinkerFlags, validLinkerFlagsWithNextArg) } -func checkFlags(name, source string, list []string, valid []*regexp.Regexp, validNext []string) error { +func checkFlags(name, source string, list []string, valid []*lazyregexp.Regexp, validNext []string) error { // Let users override rules with $CGO_CFLAGS_ALLOW, $CGO_CFLAGS_DISALLOW, etc. var ( allow *regexp.Regexp disallow *regexp.Regexp ) - if env := os.Getenv("CGO_" + name + "_ALLOW"); env != "" { + if env := cfg.Getenv("CGO_" + name + "_ALLOW"); env != "" { r, err := regexp.Compile(env) if err != nil { return fmt.Errorf("parsing $CGO_%s_ALLOW: %v", name, err) } allow = r } - if env := os.Getenv("CGO_" + name + "_DISALLOW"); env != "" { + if env := cfg.Getenv("CGO_" + name + "_DISALLOW"); env != "" { r, err := regexp.Compile(env) if err != nil { return fmt.Errorf("parsing $CGO_%s_DISALLOW: %v", name, err) @@ -269,6 +286,15 @@ Args: continue Args } + // Permit -I= /path, -I $SYSROOT. + if i+1 < len(list) && arg == "-I" { + if (strings.HasPrefix(list[i+1], "=") || strings.HasPrefix(list[i+1], "$SYSROOT")) && + load.SafeArg(list[i+1][1:]) { + i++ + continue Args + } + } + if i+1 < len(list) { return fmt.Errorf("invalid flag in %s: %s %s (see https://golang.org/s/invalidflag)", source, arg, list[i+1]) } diff --git a/extract/go.mod b/extract/go.mod index 9928de8..dca9cb0 100644 --- a/extract/go.mod +++ b/extract/go.mod @@ -1,3 +1,5 @@ module github.com/dependabot/gomodules-extracted/extract +go 1.15 + require github.com/hmarr/pkgextract v0.1.0 diff --git a/extract/main.go b/extract/main.go index d33ec66..3d230ac 100644 --- a/extract/main.go +++ b/extract/main.go @@ -26,5 +26,5 @@ func main() { log.Fatal(err) } - fmt.Println("!!! Remember to copy the LICENSE across - https://github.com/golang/go/blob/master/LICENSE") + fmt.Println("!!! Remember to copy the LICENSE across - https://raw.githubusercontent.com/golang/go/master/LICENSE") } From f77138ee6357a9c759165d558730578e2a5e28a4 Mon Sep 17 00:00:00 2001 From: Henning Surmeier Date: Thu, 8 Oct 2020 16:07:34 +0200 Subject: [PATCH 2/4] add go.mod --- go.mod | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 go.mod diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..e819ba2 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/dependabot/gomodules-extracted + +go 1.15 From 8ed17d8d4f6bd8aa82dc07b83bd0d5e32301d12f Mon Sep 17 00:00:00 2001 From: Peter Wagner Date: Tue, 13 Oct 2020 09:04:52 -0400 Subject: [PATCH 3/4] script/extract --- README.md | 4 +--- script/extract | 7 +++++++ 2 files changed, 8 insertions(+), 3 deletions(-) create mode 100755 script/extract diff --git a/README.md b/README.md index 575751c..2f3254b 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,4 @@ these packages importable, but everything is otherwise unchanged. ## Re-extracting -The `extract/` directory contains the code to do this. Run the Go program, then -copy the contents of the `out/github.com/dependabot/gomodules-extracted` -directory to the top level of this repo. +Run `script/extract`. diff --git a/script/extract b/script/extract new file mode 100755 index 0000000..dcd166e --- /dev/null +++ b/script/extract @@ -0,0 +1,7 @@ +#!/bin/sh + +(cd extract && go run .) && rsync -rcv extract/out/github.com/dependabot/gomodules-extracted/* . + +curl -sL https://raw.githubusercontent.com/golang/go/master/LICENSE | tee _internal_/LICENSE > cmd/LICENSE +git add _internal_ cmd + From 3e07cd3debc708e7f1569a05e17adfdc05c842c4 Mon Sep 17 00:00:00 2001 From: Peter Wagner Date: Tue, 13 Oct 2020 09:14:48 -0400 Subject: [PATCH 4/4] re-extract 2020-10-13 --- cmd/go/_internal_/cfg/zdefaultcc.go | 4 +- cmd/go/_internal_/robustio/robustio_darwin.go | 21 +++++ cmd/go/_internal_/robustio/robustio_flaky.go | 92 +++++++++++++++++++ 3 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 cmd/go/_internal_/robustio/robustio_darwin.go create mode 100644 cmd/go/_internal_/robustio/robustio_flaky.go diff --git a/cmd/go/_internal_/cfg/zdefaultcc.go b/cmd/go/_internal_/cfg/zdefaultcc.go index 5f45a57..7297f75 100644 --- a/cmd/go/_internal_/cfg/zdefaultcc.go +++ b/cmd/go/_internal_/cfg/zdefaultcc.go @@ -7,10 +7,10 @@ const DefaultPkgConfig = `pkg-config` func DefaultCC(goos, goarch string) string { switch goos + `/` + goarch { } - return "gcc" + return "clang" } func DefaultCXX(goos, goarch string) string { switch goos + `/` + goarch { } - return "g++" + return "clang++" } diff --git a/cmd/go/_internal_/robustio/robustio_darwin.go b/cmd/go/_internal_/robustio/robustio_darwin.go new file mode 100644 index 0000000..99fd8eb --- /dev/null +++ b/cmd/go/_internal_/robustio/robustio_darwin.go @@ -0,0 +1,21 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package robustio + +import ( + "errors" + "syscall" +) + +const errFileNotFound = syscall.ENOENT + +// isEphemeralError returns true if err may be resolved by waiting. +func isEphemeralError(err error) bool { + var errno syscall.Errno + if errors.As(err, &errno) { + return errno == errFileNotFound + } + return false +} diff --git a/cmd/go/_internal_/robustio/robustio_flaky.go b/cmd/go/_internal_/robustio/robustio_flaky.go new file mode 100644 index 0000000..3f0d75e --- /dev/null +++ b/cmd/go/_internal_/robustio/robustio_flaky.go @@ -0,0 +1,92 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build windows darwin + +package robustio + +import ( + "errors" + "io/ioutil" + "math/rand" + "os" + "syscall" + "time" +) + +const arbitraryTimeout = 2000 * time.Millisecond + +// retry retries ephemeral errors from f up to an arbitrary timeout +// to work around filesystem flakiness on Windows and Darwin. +func retry(f func() (err error, mayRetry bool)) error { + var ( + bestErr error + lowestErrno syscall.Errno + start time.Time + nextSleep time.Duration = 1 * time.Millisecond + ) + for { + err, mayRetry := f() + if err == nil || !mayRetry { + return err + } + + var errno syscall.Errno + if errors.As(err, &errno) && (lowestErrno == 0 || errno < lowestErrno) { + bestErr = err + lowestErrno = errno + } else if bestErr == nil { + bestErr = err + } + + if start.IsZero() { + start = time.Now() + } else if d := time.Since(start) + nextSleep; d >= arbitraryTimeout { + break + } + time.Sleep(nextSleep) + nextSleep += time.Duration(rand.Int63n(int64(nextSleep))) + } + + return bestErr +} + +// rename is like os.Rename, but retries ephemeral errors. +// +// On Windows it wraps os.Rename, which (as of 2019-06-04) uses MoveFileEx with +// MOVEFILE_REPLACE_EXISTING. +// +// Windows also provides a different system call, ReplaceFile, +// that provides similar semantics, but perhaps preserves more metadata. (The +// documentation on the differences between the two is very sparse.) +// +// Empirical error rates with MoveFileEx are lower under modest concurrency, so +// for now we're sticking with what the os package already provides. +func rename(oldpath, newpath string) (err error) { + return retry(func() (err error, mayRetry bool) { + err = os.Rename(oldpath, newpath) + return err, isEphemeralError(err) + }) +} + +// readFile is like ioutil.ReadFile, but retries ephemeral errors. +func readFile(filename string) ([]byte, error) { + var b []byte + err := retry(func() (err error, mayRetry bool) { + b, err = ioutil.ReadFile(filename) + + // Unlike in rename, we do not retry errFileNotFound here: it can occur + // as a spurious error, but the file may also genuinely not exist, so the + // increase in robustness is probably not worth the extra latency. + return err, isEphemeralError(err) && !errors.Is(err, errFileNotFound) + }) + return b, err +} + +func removeAll(path string) error { + return retry(func() (err error, mayRetry bool) { + err = os.RemoveAll(path) + return err, isEphemeralError(err) + }) +}