From 5fad2f86f86456d973545d35095f237637145698 Mon Sep 17 00:00:00 2001 From: Jeff Widman Date: Tue, 27 Apr 2021 22:21:28 -0700 Subject: [PATCH 1/2] Update modules to the go 1.16.3 code This is the output of running `script/extract` locally on my Mac which is running `go 1.16.3`. I suspect this pulls in a few fixes/changes in how go module machinery works. I was uncertain whether to use tip of master on `go`, but thought better to use a stable version as that'd likely be more predictable. --- _internal_/cfg/cfg.go | 1 + _internal_/execabs/execabs.go | 70 + _internal_/goroot/gc.go | 2 +- cmd/_internal_/browser/browser.go | 2 +- cmd/_internal_/objabi/flag.go | 39 + cmd/_internal_/objabi/funcdata.go | 15 +- cmd/_internal_/objabi/funcid.go | 28 +- cmd/_internal_/objabi/head.go | 2 +- cmd/_internal_/objabi/line.go | 33 +- cmd/_internal_/objabi/path.go | 22 + cmd/_internal_/objabi/reloctype.go | 22 + cmd/_internal_/objabi/reloctype_string.go | 70 +- cmd/_internal_/objabi/symkind.go | 7 +- cmd/_internal_/objabi/symkind_string.go | 21 +- cmd/_internal_/objabi/util.go | 30 +- cmd/_internal_/objabi/zbootstrap.go | 2 +- cmd/_internal_/traceviewer/format.go | 38 + cmd/go/_internal_/auth/netrc.go | 3 +- cmd/go/_internal_/base/base.go | 19 +- cmd/go/_internal_/base/flag.go | 37 +- cmd/go/_internal_/base/goflags.go | 60 +- cmd/go/_internal_/base/signal.go | 2 +- cmd/go/_internal_/cfg/cfg.go | 22 +- cmd/go/_internal_/cfg/zosarch.go | 5 +- cmd/go/_internal_/fsys/fsys.go | 689 +++++++++ cmd/go/_internal_/imports/build.go | 5 + cmd/go/_internal_/imports/read.go | 4 +- cmd/go/_internal_/imports/scan.go | 15 +- cmd/go/_internal_/imports/tags.go | 23 +- .../_internal_/filelock/filelock.go | 5 +- .../_internal_/filelock/filelock_unix.go | 6 +- cmd/go/_internal_/lockedfile/lockedfile.go | 14 +- .../lockedfile/lockedfile_filelock.go | 6 +- cmd/go/_internal_/modconv/convert.go | 59 +- cmd/go/_internal_/modfetch/cache.go | 99 +- .../_internal_/modfetch/codehost/codehost.go | 24 +- cmd/go/_internal_/modfetch/codehost/git.go | 29 +- cmd/go/_internal_/modfetch/codehost/vcs.go | 10 +- cmd/go/_internal_/modfetch/coderepo.go | 90 +- cmd/go/_internal_/modfetch/fetch.go | 498 +++--- cmd/go/_internal_/modfetch/insecure.go | 6 +- cmd/go/_internal_/modfetch/proxy.go | 85 +- cmd/go/_internal_/modfetch/pseudo.go | 12 + cmd/go/_internal_/modfetch/repo.go | 68 +- cmd/go/_internal_/modfetch/sumdb.go | 15 +- cmd/go/_internal_/modinfo/info.go | 17 +- cmd/go/_internal_/modload/build.go | 99 +- cmd/go/_internal_/modload/buildlist.go | 278 ++++ cmd/go/_internal_/modload/help.go | 500 +----- cmd/go/_internal_/modload/import.go | 414 +++-- cmd/go/_internal_/modload/init.go | 543 +++++-- cmd/go/_internal_/modload/list.go | 73 +- cmd/go/_internal_/modload/load.go | 1079 ++++++++----- cmd/go/_internal_/modload/modfile.go | 445 +++++- cmd/go/_internal_/modload/mvs.go | 224 +-- cmd/go/_internal_/modload/query.go | 834 +++++++--- cmd/go/_internal_/modload/search.go | 57 +- cmd/go/_internal_/modload/stat_unix.go | 5 +- cmd/go/_internal_/modload/vendor.go | 12 +- cmd/go/_internal_/mvs/errors.go | 101 ++ cmd/go/_internal_/mvs/mvs.go | 141 +- cmd/go/_internal_/par/queue.go | 88 ++ cmd/go/_internal_/renameio/renameio.go | 11 +- cmd/go/_internal_/robustio/robustio.go | 2 +- cmd/go/_internal_/robustio/robustio_flaky.go | 5 +- cmd/go/_internal_/search/search.go | 14 +- cmd/go/_internal_/str/path.go | 45 - cmd/go/_internal_/str/str.go | 14 + cmd/go/_internal_/trace/trace.go | 206 +++ cmd/go/_internal_/vcs/discovery.go | 97 ++ cmd/go/_internal_/vcs/vcs.go | 1363 +++++++++++++++++ cmd/go/_internal_/web/api.go | 11 +- cmd/go/_internal_/web/http.go | 7 + 73 files changed, 6608 insertions(+), 2391 deletions(-) create mode 100644 _internal_/execabs/execabs.go create mode 100644 cmd/_internal_/traceviewer/format.go create mode 100644 cmd/go/_internal_/fsys/fsys.go create mode 100644 cmd/go/_internal_/modload/buildlist.go create mode 100644 cmd/go/_internal_/mvs/errors.go create mode 100644 cmd/go/_internal_/par/queue.go create mode 100644 cmd/go/_internal_/trace/trace.go create mode 100644 cmd/go/_internal_/vcs/discovery.go create mode 100644 cmd/go/_internal_/vcs/vcs.go diff --git a/_internal_/cfg/cfg.go b/_internal_/cfg/cfg.go index bdbe9df..5530213 100644 --- a/_internal_/cfg/cfg.go +++ b/_internal_/cfg/cfg.go @@ -58,6 +58,7 @@ const KnownEnv = ` GOSUMDB GOTMPDIR GOTOOLDIR + GOVCS GOWASM GO_EXTLINK_ENABLED PKG_CONFIG diff --git a/_internal_/execabs/execabs.go b/_internal_/execabs/execabs.go new file mode 100644 index 0000000..a73ebec --- /dev/null +++ b/_internal_/execabs/execabs.go @@ -0,0 +1,70 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package execabs is a drop-in replacement for os/exec +// that requires PATH lookups to find absolute paths. +// That is, execabs.Command("cmd") runs the same PATH lookup +// as exec.Command("cmd"), but if the result is a path +// which is relative, the Run and Start methods will report +// an error instead of running the executable. +package execabs + +import ( + "context" + "fmt" + "os/exec" + "path/filepath" + "reflect" + "unsafe" +) + +var ErrNotFound = exec.ErrNotFound + +type ( + Cmd = exec.Cmd + Error = exec.Error + ExitError = exec.ExitError +) + +func relError(file, path string) error { + return fmt.Errorf("%s resolves to executable relative to current directory (.%c%s)", file, filepath.Separator, path) +} + +func LookPath(file string) (string, error) { + path, err := exec.LookPath(file) + if err != nil { + return "", err + } + if filepath.Base(file) == file && !filepath.IsAbs(path) { + return "", relError(file, path) + } + return path, nil +} + +func fixCmd(name string, cmd *exec.Cmd) { + if filepath.Base(name) == name && !filepath.IsAbs(cmd.Path) { + // exec.Command was called with a bare binary name and + // exec.LookPath returned a path which is not absolute. + // Set cmd.lookPathErr and clear cmd.Path so that it + // cannot be run. + lookPathErr := (*error)(unsafe.Pointer(reflect.ValueOf(cmd).Elem().FieldByName("lookPathErr").Addr().Pointer())) + if *lookPathErr == nil { + *lookPathErr = relError(name, cmd.Path) + } + cmd.Path = "" + } +} + +func CommandContext(ctx context.Context, name string, arg ...string) *exec.Cmd { + cmd := exec.CommandContext(ctx, name, arg...) + fixCmd(name, cmd) + return cmd + +} + +func Command(name string, arg ...string) *exec.Cmd { + cmd := exec.Command(name, arg...) + fixCmd(name, cmd) + return cmd +} diff --git a/_internal_/goroot/gc.go b/_internal_/goroot/gc.go index e51fb66..ab5b3cb 100644 --- a/_internal_/goroot/gc.go +++ b/_internal_/goroot/gc.go @@ -7,8 +7,8 @@ package goroot import ( + exec "github.com/dependabot/gomodules-extracted/_internal_/execabs" "os" - "os/exec" "path/filepath" "strings" "sync" diff --git a/cmd/_internal_/browser/browser.go b/cmd/_internal_/browser/browser.go index 6867c85..39beb75 100644 --- a/cmd/_internal_/browser/browser.go +++ b/cmd/_internal_/browser/browser.go @@ -6,8 +6,8 @@ package browser import ( + exec "github.com/dependabot/gomodules-extracted/_internal_/execabs" "os" - "os/exec" "runtime" "time" ) diff --git a/cmd/_internal_/objabi/flag.go b/cmd/_internal_/objabi/flag.go index 6870812..27c4370 100644 --- a/cmd/_internal_/objabi/flag.go +++ b/cmd/_internal_/objabi/flag.go @@ -5,6 +5,7 @@ package objabi import ( + "bytes" "flag" "fmt" "io" @@ -59,6 +60,9 @@ func expandArgs(in []string) (out []string) { log.Fatal(err) } args := strings.Split(strings.TrimSpace(strings.Replace(string(slurp), "\r", "", -1)), "\n") + for i, arg := range args { + args[i] = DecodeArg(arg) + } out = append(out, expandArgs(args)...) } else if out != nil { out = append(out, s) @@ -160,3 +164,38 @@ func (f fn1) Set(s string) error { } func (f fn1) String() string { return "" } + +// DecodeArg decodes an argument. +// +// This function is public for testing with the parallel encoder. +func DecodeArg(arg string) string { + // If no encoding, fastpath out. + if !strings.ContainsAny(arg, "\\\n") { + return arg + } + + // We can't use strings.Builder as this must work at bootstrap. + var b bytes.Buffer + var wasBS bool + for _, r := range arg { + if wasBS { + switch r { + case '\\': + b.WriteByte('\\') + case 'n': + b.WriteByte('\n') + default: + // This shouldn't happen. The only backslashes that reach here + // should encode '\n' and '\\' exclusively. + panic("badly formatted input") + } + } else if r == '\\' { + wasBS = true + continue + } else { + b.WriteRune(r) + } + wasBS = false + } + return b.String() +} diff --git a/cmd/_internal_/objabi/funcdata.go b/cmd/_internal_/objabi/funcdata.go index 4a3d5cd..e178ca0 100644 --- a/cmd/_internal_/objabi/funcdata.go +++ b/cmd/_internal_/objabi/funcdata.go @@ -11,17 +11,15 @@ package objabi // ../../../runtime/symtab.go. const ( - PCDATA_RegMapIndex = 0 // if !go115ReduceLiveness - PCDATA_UnsafePoint = 0 // if go115ReduceLiveness + PCDATA_UnsafePoint = 0 PCDATA_StackMapIndex = 1 PCDATA_InlTreeIndex = 2 FUNCDATA_ArgsPointerMaps = 0 FUNCDATA_LocalsPointerMaps = 1 - FUNCDATA_RegPointerMaps = 2 // if !go115ReduceLiveness - FUNCDATA_StackObjects = 3 - FUNCDATA_InlTree = 4 - FUNCDATA_OpenCodedDeferInfo = 5 + FUNCDATA_StackObjects = 2 + FUNCDATA_InlTree = 3 + FUNCDATA_OpenCodedDeferInfo = 4 // ArgsSizeUnknown is set in Func.argsize to mark all functions // whose argument size is unknown (C vararg functions, and @@ -32,11 +30,6 @@ const ( // Special PCDATA values. const ( - // PCDATA_RegMapIndex values. - // - // Only if !go115ReduceLiveness. - PCDATA_RegMapUnsafe = -2 // Unsafe for async preemption - // PCDATA_UnsafePoint values. PCDATA_UnsafePointSafe = -1 // Safe for async preemption PCDATA_UnsafePointUnsafe = -2 // Unsafe for async preemption diff --git a/cmd/_internal_/objabi/funcid.go b/cmd/_internal_/objabi/funcid.go index a0a75d5..0d2785a 100644 --- a/cmd/_internal_/objabi/funcid.go +++ b/cmd/_internal_/objabi/funcid.go @@ -4,11 +4,6 @@ package objabi -import ( - "strconv" - "strings" -) - // A FuncID identifies particular functions that need to be treated // specially by the runtime. // Note that in some situations involving plugins, there may be multiple @@ -31,7 +26,7 @@ const ( FuncID_gcBgMarkWorker FuncID_systemstack_switch FuncID_systemstack - FuncID_cgocallback_gofunc + FuncID_cgocallback FuncID_gogo FuncID_externalthreadhandler FuncID_debugCallV1 @@ -44,7 +39,10 @@ const ( // Get the function ID for the named function in the named file. // The function should be package-qualified. -func GetFuncID(name, file string) FuncID { +func GetFuncID(name string, isWrapper bool) FuncID { + if isWrapper { + return FuncID_wrapper + } switch name { case "runtime.main": return FuncID_runtime_main @@ -72,8 +70,8 @@ func GetFuncID(name, file string) FuncID { return FuncID_systemstack_switch case "runtime.systemstack": return FuncID_systemstack - case "runtime.cgocallback_gofunc": - return FuncID_cgocallback_gofunc + case "runtime.cgocallback": + return FuncID_cgocallback case "runtime.gogo": return FuncID_gogo case "runtime.externalthreadhandler": @@ -98,17 +96,5 @@ func GetFuncID(name, file string) FuncID { // Don't show in the call stack (used when invoking defer functions) return FuncID_wrapper } - if file == "" { - return FuncID_wrapper - } - if strings.HasPrefix(name, "runtime.call") { - if _, err := strconv.Atoi(name[12:]); err == nil { - // runtime.callXX reflect call wrappers. - return FuncID_wrapper - } - } - if strings.HasSuffix(name, "-fm") { - return FuncID_wrapper - } return FuncID_normal } diff --git a/cmd/_internal_/objabi/head.go b/cmd/_internal_/objabi/head.go index a6c73e1..169c5d2 100644 --- a/cmd/_internal_/objabi/head.go +++ b/cmd/_internal_/objabi/head.go @@ -54,7 +54,7 @@ func (h *HeadType) Set(s string) error { switch s { case "aix": *h = Haix - case "darwin": + case "darwin", "ios": *h = Hdarwin case "dragonfly": *h = Hdragonfly diff --git a/cmd/_internal_/objabi/line.go b/cmd/_internal_/objabi/line.go index 178c836..0733b65 100644 --- a/cmd/_internal_/objabi/line.go +++ b/cmd/_internal_/objabi/line.go @@ -37,25 +37,36 @@ func AbsFile(dir, file, rewrites string) string { abs = filepath.Join(dir, file) } + abs, rewritten := ApplyRewrites(abs, rewrites) + if !rewritten && hasPathPrefix(abs, GOROOT) { + abs = "$GOROOT" + abs[len(GOROOT):] + } + + if abs == "" { + abs = "??" + } + return abs +} + +// ApplyRewrites returns the filename for file in the given directory, +// as rewritten by the rewrites argument. +// +// The rewrites argument is a ;-separated list of rewrites. +// Each rewrite is of the form "prefix" or "prefix=>replace", +// where prefix must match a leading sequence of path elements +// and is either removed entirely or replaced by the replacement. +func ApplyRewrites(file, rewrites string) (string, bool) { start := 0 for i := 0; i <= len(rewrites); i++ { if i == len(rewrites) || rewrites[i] == ';' { - if new, ok := applyRewrite(abs, rewrites[start:i]); ok { - abs = new - goto Rewritten + if new, ok := applyRewrite(file, rewrites[start:i]); ok { + return new, true } start = i + 1 } } - if hasPathPrefix(abs, GOROOT) { - abs = "$GOROOT" + abs[len(GOROOT):] - } -Rewritten: - if abs == "" { - abs = "??" - } - return abs + return file, false } // applyRewrite applies the rewrite to the path, diff --git a/cmd/_internal_/objabi/path.go b/cmd/_internal_/objabi/path.go index 2a42179..fd1c998 100644 --- a/cmd/_internal_/objabi/path.go +++ b/cmd/_internal_/objabi/path.go @@ -39,3 +39,25 @@ func PathToPrefix(s string) string { return string(p) } + +// IsRuntimePackagePath examines 'pkgpath' and returns TRUE if it +// belongs to the collection of "runtime-related" packages, including +// "runtime" itself, "reflect", "syscall", and the +// "runtime/internal/*" packages. The compiler and/or assembler in +// some cases need to be aware of when they are building such a +// package, for example to enable features such as ABI selectors in +// assembly sources. +func IsRuntimePackagePath(pkgpath string) bool { + rval := false + switch pkgpath { + case "runtime": + rval = true + case "reflect": + rval = true + case "syscall": + rval = true + default: + rval = strings.HasPrefix(pkgpath, "runtime/internal") + } + return rval +} diff --git a/cmd/_internal_/objabi/reloctype.go b/cmd/_internal_/objabi/reloctype.go index 980d3b3..b9f517f 100644 --- a/cmd/_internal_/objabi/reloctype.go +++ b/cmd/_internal_/objabi/reloctype.go @@ -89,6 +89,17 @@ const ( // should be linked into the final binary, even if there are no other // direct references. (This is used for types reachable by reflection.) R_USETYPE + // R_USEIFACE marks a type is converted to an interface in the function this + // relocation is applied to. The target is a type descriptor. + // This is a marker relocation (0-sized), for the linker's reachabililty + // analysis. + R_USEIFACE + // R_USEIFACEMETHOD marks an interface method that is used in the function + // this relocation is applied to. The target is an interface type descriptor. + // The addend is the offset of the method in the type descriptor. + // This is a marker relocation (0-sized), for the linker's reachabililty + // analysis. + R_USEIFACEMETHOD // R_METHODOFF resolves to a 32-bit offset from the beginning of the section // holding the data being relocated to the referenced symbol. // It is a variant of R_ADDROFF used when linking from the uncommonType of a @@ -145,6 +156,9 @@ const ( // R_ARM64_LDST8 sets a LD/ST immediate value to bits [11:0] of a local address. R_ARM64_LDST8 + // R_ARM64_LDST16 sets a LD/ST immediate value to bits [11:1] of a local address. + R_ARM64_LDST16 + // R_ARM64_LDST32 sets a LD/ST immediate value to bits [11:2] of a local address. R_ARM64_LDST32 @@ -212,6 +226,14 @@ const ( // AUIPC + S-type instruction pair. R_RISCV_PCREL_STYPE + // R_RISCV_TLS_IE_ITYPE resolves a 32-bit TLS initial-exec TOC offset + // address using an AUIPC + I-type instruction pair. + R_RISCV_TLS_IE_ITYPE + + // R_RISCV_TLS_IE_STYPE resolves a 32-bit TLS initial-exec TOC offset + // address using an AUIPC + S-type instruction pair. + R_RISCV_TLS_IE_STYPE + // R_PCRELDBL relocates s390x 2-byte aligned PC-relative addresses. // TODO(mundaym): remove once variants can be serialized - see issue 14218. R_PCRELDBL diff --git a/cmd/_internal_/objabi/reloctype_string.go b/cmd/_internal_/objabi/reloctype_string.go index 83dfe71..658a44f 100644 --- a/cmd/_internal_/objabi/reloctype_string.go +++ b/cmd/_internal_/objabi/reloctype_string.go @@ -4,9 +4,75 @@ package objabi import "strconv" -const _RelocType_name = "R_ADDRR_ADDRPOWERR_ADDRARM64R_ADDRMIPSR_ADDROFFR_WEAKADDROFFR_SIZER_CALLR_CALLARMR_CALLARM64R_CALLINDR_CALLPOWERR_CALLMIPSR_CALLRISCVR_CONSTR_PCRELR_TLS_LER_TLS_IER_GOTOFFR_PLT0R_PLT1R_PLT2R_USEFIELDR_USETYPER_METHODOFFR_POWER_TOCR_GOTPCRELR_JMPMIPSR_DWARFSECREFR_DWARFFILEREFR_ARM64_TLS_LER_ARM64_TLS_IER_ARM64_GOTPCRELR_ARM64_GOTR_ARM64_PCRELR_ARM64_LDST8R_ARM64_LDST32R_ARM64_LDST64R_ARM64_LDST128R_POWER_TLS_LER_POWER_TLS_IER_POWER_TLSR_ADDRPOWER_DSR_ADDRPOWER_GOTR_ADDRPOWER_PCRELR_ADDRPOWER_TOCRELR_ADDRPOWER_TOCREL_DSR_RISCV_PCREL_ITYPER_RISCV_PCREL_STYPER_PCRELDBLR_ADDRMIPSUR_ADDRMIPSTLSR_ADDRCUOFFR_WASMIMPORTR_XCOFFREF" +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[R_ADDR-1] + _ = x[R_ADDRPOWER-2] + _ = x[R_ADDRARM64-3] + _ = x[R_ADDRMIPS-4] + _ = x[R_ADDROFF-5] + _ = x[R_WEAKADDROFF-6] + _ = x[R_SIZE-7] + _ = x[R_CALL-8] + _ = x[R_CALLARM-9] + _ = x[R_CALLARM64-10] + _ = x[R_CALLIND-11] + _ = x[R_CALLPOWER-12] + _ = x[R_CALLMIPS-13] + _ = x[R_CALLRISCV-14] + _ = x[R_CONST-15] + _ = x[R_PCREL-16] + _ = x[R_TLS_LE-17] + _ = x[R_TLS_IE-18] + _ = x[R_GOTOFF-19] + _ = x[R_PLT0-20] + _ = x[R_PLT1-21] + _ = x[R_PLT2-22] + _ = x[R_USEFIELD-23] + _ = x[R_USETYPE-24] + _ = x[R_USEIFACE-25] + _ = x[R_USEIFACEMETHOD-26] + _ = x[R_METHODOFF-27] + _ = x[R_POWER_TOC-28] + _ = x[R_GOTPCREL-29] + _ = x[R_JMPMIPS-30] + _ = x[R_DWARFSECREF-31] + _ = x[R_DWARFFILEREF-32] + _ = x[R_ARM64_TLS_LE-33] + _ = x[R_ARM64_TLS_IE-34] + _ = x[R_ARM64_GOTPCREL-35] + _ = x[R_ARM64_GOT-36] + _ = x[R_ARM64_PCREL-37] + _ = x[R_ARM64_LDST8-38] + _ = x[R_ARM64_LDST16-39] + _ = x[R_ARM64_LDST32-40] + _ = x[R_ARM64_LDST64-41] + _ = x[R_ARM64_LDST128-42] + _ = x[R_POWER_TLS_LE-43] + _ = x[R_POWER_TLS_IE-44] + _ = x[R_POWER_TLS-45] + _ = x[R_ADDRPOWER_DS-46] + _ = x[R_ADDRPOWER_GOT-47] + _ = x[R_ADDRPOWER_PCREL-48] + _ = x[R_ADDRPOWER_TOCREL-49] + _ = x[R_ADDRPOWER_TOCREL_DS-50] + _ = x[R_RISCV_PCREL_ITYPE-51] + _ = x[R_RISCV_PCREL_STYPE-52] + _ = x[R_RISCV_TLS_IE_ITYPE-53] + _ = x[R_RISCV_TLS_IE_STYPE-54] + _ = x[R_PCRELDBL-55] + _ = x[R_ADDRMIPSU-56] + _ = x[R_ADDRMIPSTLS-57] + _ = x[R_ADDRCUOFF-58] + _ = x[R_WASMIMPORT-59] + _ = x[R_XCOFFREF-60] +} + +const _RelocType_name = "R_ADDRR_ADDRPOWERR_ADDRARM64R_ADDRMIPSR_ADDROFFR_WEAKADDROFFR_SIZER_CALLR_CALLARMR_CALLARM64R_CALLINDR_CALLPOWERR_CALLMIPSR_CALLRISCVR_CONSTR_PCRELR_TLS_LER_TLS_IER_GOTOFFR_PLT0R_PLT1R_PLT2R_USEFIELDR_USETYPER_USEIFACER_USEIFACEMETHODR_METHODOFFR_POWER_TOCR_GOTPCRELR_JMPMIPSR_DWARFSECREFR_DWARFFILEREFR_ARM64_TLS_LER_ARM64_TLS_IER_ARM64_GOTPCRELR_ARM64_GOTR_ARM64_PCRELR_ARM64_LDST8R_ARM64_LDST16R_ARM64_LDST32R_ARM64_LDST64R_ARM64_LDST128R_POWER_TLS_LER_POWER_TLS_IER_POWER_TLSR_ADDRPOWER_DSR_ADDRPOWER_GOTR_ADDRPOWER_PCRELR_ADDRPOWER_TOCRELR_ADDRPOWER_TOCREL_DSR_RISCV_PCREL_ITYPER_RISCV_PCREL_STYPER_RISCV_TLS_IE_ITYPER_RISCV_TLS_IE_STYPER_PCRELDBLR_ADDRMIPSUR_ADDRMIPSTLSR_ADDRCUOFFR_WASMIMPORTR_XCOFFREF" -var _RelocType_index = [...]uint16{0, 6, 17, 28, 38, 47, 60, 66, 72, 81, 92, 101, 112, 122, 133, 140, 147, 155, 163, 171, 177, 183, 189, 199, 208, 219, 230, 240, 249, 262, 276, 290, 304, 320, 331, 344, 357, 371, 385, 400, 414, 428, 439, 453, 468, 485, 503, 524, 543, 562, 572, 583, 596, 607, 619, 629} +var _RelocType_index = [...]uint16{0, 6, 17, 28, 38, 47, 60, 66, 72, 81, 92, 101, 112, 122, 133, 140, 147, 155, 163, 171, 177, 183, 189, 199, 208, 218, 234, 245, 256, 266, 275, 288, 302, 316, 330, 346, 357, 370, 383, 397, 411, 425, 440, 454, 468, 479, 493, 508, 525, 543, 564, 583, 602, 622, 642, 652, 663, 676, 687, 699, 709} func (i RelocType) String() string { i -= 1 diff --git a/cmd/_internal_/objabi/symkind.go b/cmd/_internal_/objabi/symkind.go index 129a3e9..f2cebf9 100644 --- a/cmd/_internal_/objabi/symkind.go +++ b/cmd/_internal_/objabi/symkind.go @@ -56,7 +56,12 @@ const ( // Thread-local data that is initially all 0s STLSBSS // Debugging data - SDWARFINFO + SDWARFCUINFO + SDWARFCONST + SDWARFFCN + SDWARFABSFCN + SDWARFTYPE + SDWARFVAR SDWARFRANGE SDWARFLOC SDWARFLINES diff --git a/cmd/_internal_/objabi/symkind_string.go b/cmd/_internal_/objabi/symkind_string.go index 919a666..1b1c394 100644 --- a/cmd/_internal_/objabi/symkind_string.go +++ b/cmd/_internal_/objabi/symkind_string.go @@ -16,17 +16,22 @@ func _() { _ = x[SBSS-5] _ = x[SNOPTRBSS-6] _ = x[STLSBSS-7] - _ = x[SDWARFINFO-8] - _ = x[SDWARFRANGE-9] - _ = x[SDWARFLOC-10] - _ = x[SDWARFLINES-11] - _ = x[SABIALIAS-12] - _ = x[SLIBFUZZER_EXTRA_COUNTER-13] + _ = x[SDWARFCUINFO-8] + _ = x[SDWARFCONST-9] + _ = x[SDWARFFCN-10] + _ = x[SDWARFABSFCN-11] + _ = x[SDWARFTYPE-12] + _ = x[SDWARFVAR-13] + _ = x[SDWARFRANGE-14] + _ = x[SDWARFLOC-15] + _ = x[SDWARFLINES-16] + _ = x[SABIALIAS-17] + _ = x[SLIBFUZZER_EXTRA_COUNTER-18] } -const _SymKind_name = "SxxxSTEXTSRODATASNOPTRDATASDATASBSSSNOPTRBSSSTLSBSSSDWARFINFOSDWARFRANGESDWARFLOCSDWARFLINESSABIALIASSLIBFUZZER_EXTRA_COUNTER" +const _SymKind_name = "SxxxSTEXTSRODATASNOPTRDATASDATASBSSSNOPTRBSSSTLSBSSSDWARFCUINFOSDWARFCONSTSDWARFFCNSDWARFABSFCNSDWARFTYPESDWARFVARSDWARFRANGESDWARFLOCSDWARFLINESSABIALIASSLIBFUZZER_EXTRA_COUNTER" -var _SymKind_index = [...]uint8{0, 4, 9, 16, 26, 31, 35, 44, 51, 61, 72, 81, 92, 101, 125} +var _SymKind_index = [...]uint8{0, 4, 9, 16, 26, 31, 35, 44, 51, 63, 74, 83, 95, 105, 114, 125, 134, 145, 154, 178} func (i SymKind) String() string { if i >= SymKind(len(_SymKind_index)-1) { diff --git a/cmd/_internal_/objabi/util.go b/cmd/_internal_/objabi/util.go index abdfe99..feca814 100644 --- a/cmd/_internal_/objabi/util.go +++ b/cmd/_internal_/objabi/util.go @@ -25,7 +25,6 @@ var ( GOARCH = envOr("GOARCH", defaultGOARCH) GOOS = envOr("GOOS", defaultGOOS) GO386 = envOr("GO386", defaultGO386) - GOAMD64 = goamd64() GOARM = goarm() GOMIPS = gomips() GOMIPS64 = gomips64() @@ -37,17 +36,16 @@ var ( const ( ElfRelocOffset = 256 - MachoRelocOffset = 2048 // reserve enough space for ELF relocations - Go115AMD64 = "alignedjumps" // Should be "alignedjumps" or "normaljumps"; this replaces environment variable introduced in CL 219357. + MachoRelocOffset = 2048 // reserve enough space for ELF relocations ) -// TODO(1.16): assuming no issues in 1.15 release, remove this and related constant. -func goamd64() string { - return Go115AMD64 -} - func goarm() int { - switch v := envOr("GOARM", defaultGOARM); v { + def := defaultGOARM + if GOOS == "android" && GOARCH == "arm" { + // Android arm devices always support GOARM=7. + def = "7" + } + switch v := envOr("GOARM", def); v { case "5": return 5 case "6": @@ -131,12 +129,16 @@ func init() { addexp(f) } } -} -func Framepointer_enabled(goos, goarch string) bool { - return framepointer_enabled != 0 && (goarch == "amd64" || goarch == "arm64" && (goos == "linux" || goos == "darwin")) + // regabi is only supported on amd64. + if GOARCH != "amd64" { + Regabi_enabled = 0 + } } +// Note: must agree with runtime.framepointer_enabled. +var Framepointer_enabled = GOARCH == "amd64" || GOARCH == "arm64" && (GOOS == "linux" || GOOS == "darwin" || GOOS == "ios") + func addexp(s string) { // Could do general integer parsing here, but the runtime copy doesn't yet. v := 1 @@ -159,10 +161,10 @@ func addexp(s string) { } var ( - framepointer_enabled int = 1 Fieldtrack_enabled int Preemptibleloops_enabled int Staticlockranking_enabled int + Regabi_enabled int ) // Toolchain experiments. @@ -174,9 +176,9 @@ var exper = []struct { val *int }{ {"fieldtrack", &Fieldtrack_enabled}, - {"framepointer", &framepointer_enabled}, {"preemptibleloops", &Preemptibleloops_enabled}, {"staticlockranking", &Staticlockranking_enabled}, + {"regabi", &Regabi_enabled}, } var defaultExpstring = Expstring() diff --git a/cmd/_internal_/objabi/zbootstrap.go b/cmd/_internal_/objabi/zbootstrap.go index 8ec1ff1..3c67a6e 100644 --- a/cmd/_internal_/objabi/zbootstrap.go +++ b/cmd/_internal_/objabi/zbootstrap.go @@ -13,6 +13,6 @@ const defaultGOOS = runtime.GOOS const defaultGOARCH = runtime.GOARCH const defaultGO_EXTLINK_ENABLED = `` const defaultGO_LDSO = `` -const version = `go1.15.2` +const version = `go1.16.3` const stackGuardMultiplierDefault = 1 const goexperiment = `` diff --git a/cmd/_internal_/traceviewer/format.go b/cmd/_internal_/traceviewer/format.go new file mode 100644 index 0000000..7b92935 --- /dev/null +++ b/cmd/_internal_/traceviewer/format.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package traceviewer provides definitions of the JSON data structures +// used by the Chrome trace viewer. +// +// The official description of the format is in this file: +// https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview +package traceviewer + +type Data struct { + Events []*Event `json:"traceEvents"` + Frames map[string]Frame `json:"stackFrames"` + TimeUnit string `json:"displayTimeUnit"` +} + +type Event struct { + Name string `json:"name,omitempty"` + Phase string `json:"ph"` + Scope string `json:"s,omitempty"` + Time float64 `json:"ts"` + Dur float64 `json:"dur,omitempty"` + PID uint64 `json:"pid"` + TID uint64 `json:"tid"` + ID uint64 `json:"id,omitempty"` + BindPoint string `json:"bp,omitempty"` + Stack int `json:"sf,omitempty"` + EndStack int `json:"esf,omitempty"` + Arg interface{} `json:"args,omitempty"` + Cname string `json:"cname,omitempty"` + Category string `json:"cat,omitempty"` +} + +type Frame struct { + Name string `json:"name"` + Parent int `json:"parent,omitempty"` +} diff --git a/cmd/go/_internal_/auth/netrc.go b/cmd/go/_internal_/auth/netrc.go index a97827e..114d228 100644 --- a/cmd/go/_internal_/auth/netrc.go +++ b/cmd/go/_internal_/auth/netrc.go @@ -5,7 +5,6 @@ package auth import ( - "io/ioutil" "os" "path/filepath" "runtime" @@ -99,7 +98,7 @@ func readNetrc() { return } - data, err := ioutil.ReadFile(path) + data, err := os.ReadFile(path) if err != nil { if !os.IsNotExist(err) { netrcErr = err diff --git a/cmd/go/_internal_/base/base.go b/cmd/go/_internal_/base/base.go index 7695003..bd6f15c 100644 --- a/cmd/go/_internal_/base/base.go +++ b/cmd/go/_internal_/base/base.go @@ -7,11 +7,12 @@ package base import ( + "context" "flag" "fmt" + exec "github.com/dependabot/gomodules-extracted/_internal_/execabs" "log" "os" - "os/exec" "strings" "sync" @@ -24,7 +25,7 @@ import ( type Command struct { // Run runs the command. // The args are the arguments after the command name. - Run func(cmd *Command, args []string) + Run func(ctx context.Context, cmd *Command, args []string) // UsageLine is the one-line usage message. // The words between "go" and the first flag or argument in the line are taken to be the command name. @@ -55,6 +56,20 @@ var Go = &Command{ // Commands initialized in package main } +// hasFlag reports whether a command or any of its subcommands contain the given +// flag. +func hasFlag(c *Command, name string) bool { + if f := c.Flag.Lookup(name); f != nil { + return true + } + for _, sub := range c.Commands { + if hasFlag(sub, name) { + return true + } + } + return false +} + // LongName returns the command's long name: all the words in the usage line between "go" and a flag or argument, func (c *Command) LongName() string { name := c.UsageLine diff --git a/cmd/go/_internal_/base/flag.go b/cmd/go/_internal_/base/flag.go index ea6e0dc..dccd87d 100644 --- a/cmd/go/_internal_/base/flag.go +++ b/cmd/go/_internal_/base/flag.go @@ -8,6 +8,7 @@ import ( "flag" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/fsys" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/str" ) @@ -28,13 +29,43 @@ func (v *StringsFlag) String() string { return "" } +// explicitStringFlag is like a regular string flag, but it also tracks whether +// the string was set explicitly to a non-empty value. +type explicitStringFlag struct { + value *string + explicit *bool +} + +func (f explicitStringFlag) String() string { + if f.value == nil { + return "" + } + return *f.value +} + +func (f explicitStringFlag) Set(v string) error { + *f.value = v + if v != "" { + *f.explicit = true + } + return nil +} + // AddBuildFlagsNX adds the -n and -x build flags to the flag set. func AddBuildFlagsNX(flags *flag.FlagSet) { flags.BoolVar(&cfg.BuildN, "n", false, "") flags.BoolVar(&cfg.BuildX, "x", false, "") } -// AddLoadFlags adds the -mod build flag to the flag set. -func AddLoadFlags(flags *flag.FlagSet) { - flags.StringVar(&cfg.BuildMod, "mod", "", "") +// AddModFlag adds the -mod build flag to the flag set. +func AddModFlag(flags *flag.FlagSet) { + flags.Var(explicitStringFlag{value: &cfg.BuildMod, explicit: &cfg.BuildModExplicit}, "mod", "") +} + +// AddModCommonFlags adds the module-related flags common to build commands +// and 'go mod' subcommands. +func AddModCommonFlags(flags *flag.FlagSet) { + flags.BoolVar(&cfg.ModCacheRW, "modcacherw", false, "") + flags.StringVar(&cfg.ModFile, "modfile", "", "") + flags.StringVar(&fsys.OverlayFile, "overlay", "", "") } diff --git a/cmd/go/_internal_/base/goflags.go b/cmd/go/_internal_/base/goflags.go index 2956ea6..2b02184 100644 --- a/cmd/go/_internal_/base/goflags.go +++ b/cmd/go/_internal_/base/goflags.go @@ -13,15 +13,7 @@ import ( "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" ) -var ( - goflags []string // cached $GOFLAGS list; can be -x or --x form - knownFlag = make(map[string]bool) // flags allowed to appear in $GOFLAGS; no leading dashes -) - -// AddKnownFlag adds name to the list of known flags for use in $GOFLAGS. -func AddKnownFlag(name string) { - knownFlag[name] = true -} +var goflags []string // cached $GOFLAGS list; can be -x or --x form // GOFLAGS returns the flags from $GOFLAGS. // The list can be assumed to contain one string per flag, @@ -38,22 +30,12 @@ func InitGOFLAGS() { return } - // Build list of all flags for all commands. - // If no command has that flag, then we report the problem. - // This catches typos while still letting users record flags in GOFLAGS - // that only apply to a subset of go commands. - // Commands using CustomFlags can report their flag names - // by calling AddKnownFlag instead. - var walkFlags func(*Command) - walkFlags = func(cmd *Command) { - for _, sub := range cmd.Commands { - walkFlags(sub) - } - cmd.Flag.VisitAll(func(f *flag.Flag) { - knownFlag[f.Name] = true - }) + goflags = strings.Fields(cfg.Getenv("GOFLAGS")) + if len(goflags) == 0 { + // nothing to do; avoid work on later InitGOFLAGS call + goflags = []string{} + return } - walkFlags(Go) // Ignore bad flag in go env and go bug, because // they are what people reach for when debugging @@ -61,11 +43,6 @@ func InitGOFLAGS() { // (Both will show the GOFLAGS setting if let succeed.) hideErrors := cfg.CmdName == "env" || cfg.CmdName == "bug" - goflags = strings.Fields(cfg.Getenv("GOFLAGS")) - if goflags == nil { - goflags = []string{} // avoid work on later InitGOFLAGS call - } - // Each of the words returned by strings.Fields must be its own flag. // To set flag arguments use -x=value instead of -x value. // For boolean flags, -x is fine instead of -x=true. @@ -85,7 +62,7 @@ func InitGOFLAGS() { if i := strings.Index(name, "="); i >= 0 { name = name[:i] } - if !knownFlag[name] { + if !hasFlag(Go, name) { if hideErrors { continue } @@ -115,7 +92,11 @@ func SetFromGOFLAGS(flags *flag.FlagSet) { } for _, goflag := range goflags { name, value, hasValue := goflag, "", false - if i := strings.Index(goflag, "="); i >= 0 { + // Ignore invalid flags like '=' or '=value'. + // If it is not reported in InitGOFlags it means we don't want to report it. + if i := strings.Index(goflag, "="); i == 0 { + continue + } else if i > 0 { name, value, hasValue = goflag[:i], goflag[i+1:], true } if strings.HasPrefix(name, "--") { @@ -153,3 +134,20 @@ func SetFromGOFLAGS(flags *flag.FlagSet) { } } } + +// InGOFLAGS returns whether GOFLAGS contains the given flag, such as "-mod". +func InGOFLAGS(flag string) bool { + for _, goflag := range GOFLAGS() { + name := goflag + if strings.HasPrefix(name, "--") { + name = name[1:] + } + if i := strings.Index(name, "="); i >= 0 { + name = name[:i] + } + if name == flag { + return true + } + } + return false +} diff --git a/cmd/go/_internal_/base/signal.go b/cmd/go/_internal_/base/signal.go index 54d1187..05befcf 100644 --- a/cmd/go/_internal_/base/signal.go +++ b/cmd/go/_internal_/base/signal.go @@ -15,7 +15,7 @@ var Interrupted = make(chan struct{}) // processSignals setups signal handler. func processSignals() { - sig := make(chan os.Signal) + sig := make(chan os.Signal, 1) signal.Notify(sig, signalsToIgnore...) go func() { <-sig diff --git a/cmd/go/_internal_/cfg/cfg.go b/cmd/go/_internal_/cfg/cfg.go index f40bef3..f7b6334 100644 --- a/cmd/go/_internal_/cfg/cfg.go +++ b/cmd/go/_internal_/cfg/cfg.go @@ -11,13 +11,15 @@ import ( "fmt" "go/build" "github.com/dependabot/gomodules-extracted/_internal_/cfg" - "io/ioutil" + "io" "os" "path/filepath" "runtime" "strings" "sync" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/fsys" + "github.com/dependabot/gomodules-extracted/cmd/_internal_/objabi" ) @@ -27,7 +29,8 @@ var ( BuildBuildmode string // -buildmode flag BuildContext = defaultContext() BuildMod string // -mod flag - BuildModReason string // reason -mod flag is set, if set by default + BuildModExplicit bool // whether -mod was set explicitly + BuildModReason string // reason -mod was set, if set by default BuildI bool // -i flag BuildLinkshared bool // -linkshared flag BuildMSan bool // -msan flag @@ -48,9 +51,12 @@ var ( ModCacheRW bool // -modcacherw flag ModFile string // -modfile flag + Insecure bool // -insecure flag + CmdName string // "build", "install", "list", "mod tidy", etc. DebugActiongraph string // -debug-actiongraph flag (undocumented, unstable) + DebugTrace string // -debug-trace flag ) func defaultContext() build.Context { @@ -100,6 +106,15 @@ func defaultContext() build.Context { // Nothing to do here. } + ctxt.OpenFile = func(path string) (io.ReadCloser, error) { + return fsys.Open(path) + } + ctxt.ReadDir = fsys.ReadDir + ctxt.IsDir = func(path string) bool { + isDir, err := fsys.IsDir(path) + return err == nil && isDir + } + return ctxt } @@ -171,7 +186,7 @@ func initEnvCache() { if file == "" { return } - data, err := ioutil.ReadFile(file) + data, err := os.ReadFile(file) if err != nil { return } @@ -252,6 +267,7 @@ var ( GONOPROXY = envOr("GONOPROXY", GOPRIVATE) GONOSUMDB = envOr("GONOSUMDB", GOPRIVATE) GOINSECURE = Getenv("GOINSECURE") + GOVCS = Getenv("GOVCS") ) var SumdbDir = gopathDir("pkg/sumdb") diff --git a/cmd/go/_internal_/cfg/zosarch.go b/cmd/go/_internal_/cfg/zosarch.go index 9dbd520..14ca54e 100644 --- a/cmd/go/_internal_/cfg/zosarch.go +++ b/cmd/go/_internal_/cfg/zosarch.go @@ -16,6 +16,8 @@ var OSArchSupportsCgo = map[string]bool{ "freebsd/arm": true, "freebsd/arm64": true, "illumos/amd64": true, + "ios/amd64": true, + "ios/arm64": true, "js/wasm": false, "linux/386": true, "linux/amd64": true, @@ -27,7 +29,7 @@ var OSArchSupportsCgo = map[string]bool{ "linux/mipsle": true, "linux/ppc64": false, "linux/ppc64le": true, - "linux/riscv64": false, + "linux/riscv64": true, "linux/s390x": true, "linux/sparc64": true, "netbsd/386": true, @@ -38,6 +40,7 @@ var OSArchSupportsCgo = map[string]bool{ "openbsd/amd64": true, "openbsd/arm": true, "openbsd/arm64": true, + "openbsd/mips64": false, "plan9/386": false, "plan9/amd64": false, "plan9/arm": false, diff --git a/cmd/go/_internal_/fsys/fsys.go b/cmd/go/_internal_/fsys/fsys.go new file mode 100644 index 0000000..d6a99d7 --- /dev/null +++ b/cmd/go/_internal_/fsys/fsys.go @@ -0,0 +1,689 @@ +// Package fsys is an abstraction for reading files that +// allows for virtual overlays on top of the files on disk. +package fsys + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "time" +) + +// OverlayFile is the path to a text file in the OverlayJSON format. +// It is the value of the -overlay flag. +var OverlayFile string + +// OverlayJSON is the format overlay files are expected to be in. +// The Replace map maps from overlaid paths to replacement paths: +// the Go command will forward all reads trying to open +// each overlaid path to its replacement path, or consider the overlaid +// path not to exist if the replacement path is empty. +type OverlayJSON struct { + Replace map[string]string +} + +type node struct { + actualFilePath string // empty if a directory + children map[string]*node // path element → file or directory +} + +func (n *node) isDir() bool { + return n.actualFilePath == "" && n.children != nil +} + +func (n *node) isDeleted() bool { + return n.actualFilePath == "" && n.children == nil +} + +// TODO(matloob): encapsulate these in an io/fs-like interface +var overlay map[string]*node // path -> file or directory node +var cwd string // copy of base.Cwd to avoid dependency + +// Canonicalize a path for looking it up in the overlay. +// Important: filepath.Join(cwd, path) doesn't always produce +// the correct absolute path if path is relative, because on +// Windows producing the correct absolute path requires making +// a syscall. So this should only be used when looking up paths +// in the overlay, or canonicalizing the paths in the overlay. +func canonicalize(path string) string { + if path == "" { + return "" + } + if filepath.IsAbs(path) { + return filepath.Clean(path) + } + + if v := filepath.VolumeName(cwd); v != "" && path[0] == filepath.Separator { + // On Windows filepath.Join(cwd, path) doesn't always work. In general + // filepath.Abs needs to make a syscall on Windows. Elsewhere in cmd/go + // use filepath.Join(cwd, path), but cmd/go specifically supports Windows + // paths that start with "\" which implies the path is relative to the + // volume of the working directory. See golang.org/issue/8130. + return filepath.Join(v, path) + } + + // Make the path absolute. + return filepath.Join(cwd, path) +} + +// Init initializes the overlay, if one is being used. +func Init(wd string) error { + if overlay != nil { + // already initialized + return nil + } + + cwd = wd + + if OverlayFile == "" { + return nil + } + + b, err := os.ReadFile(OverlayFile) + if err != nil { + return fmt.Errorf("reading overlay file: %v", err) + } + + var overlayJSON OverlayJSON + if err := json.Unmarshal(b, &overlayJSON); err != nil { + return fmt.Errorf("parsing overlay JSON: %v", err) + } + + return initFromJSON(overlayJSON) +} + +func initFromJSON(overlayJSON OverlayJSON) error { + // Canonicalize the paths in in the overlay map. + // Use reverseCanonicalized to check for collisions: + // no two 'from' paths should canonicalize to the same path. + overlay = make(map[string]*node) + reverseCanonicalized := make(map[string]string) // inverse of canonicalize operation, to check for duplicates + // Build a table of file and directory nodes from the replacement map. + + // Remove any potential non-determinism from iterating over map by sorting it. + replaceFrom := make([]string, 0, len(overlayJSON.Replace)) + for k := range overlayJSON.Replace { + replaceFrom = append(replaceFrom, k) + } + sort.Strings(replaceFrom) + + for _, from := range replaceFrom { + to := overlayJSON.Replace[from] + // Canonicalize paths and check for a collision. + if from == "" { + return fmt.Errorf("empty string key in overlay file Replace map") + } + cfrom := canonicalize(from) + if to != "" { + // Don't canonicalize "", meaning to delete a file, because then it will turn into ".". + to = canonicalize(to) + } + if otherFrom, seen := reverseCanonicalized[cfrom]; seen { + return fmt.Errorf( + "paths %q and %q both canonicalize to %q in overlay file Replace map", otherFrom, from, cfrom) + } + reverseCanonicalized[cfrom] = from + from = cfrom + + // Create node for overlaid file. + dir, base := filepath.Dir(from), filepath.Base(from) + if n, ok := overlay[from]; ok { + // All 'from' paths in the overlay are file paths. Since the from paths + // are in a map, they are unique, so if the node already exists we added + // it below when we create parent directory nodes. That is, that + // both a file and a path to one of its parent directories exist as keys + // in the Replace map. + // + // This only applies if the overlay directory has any files or directories + // in it: placeholder directories that only contain deleted files don't + // count. They are safe to be overwritten with actual files. + for _, f := range n.children { + if !f.isDeleted() { + return fmt.Errorf("invalid overlay: path %v is used as both file and directory", from) + } + } + } + overlay[from] = &node{actualFilePath: to} + + // Add parent directory nodes to overlay structure. + childNode := overlay[from] + for { + dirNode := overlay[dir] + if dirNode == nil || dirNode.isDeleted() { + dirNode = &node{children: make(map[string]*node)} + overlay[dir] = dirNode + } + if childNode.isDeleted() { + // Only create one parent for a deleted file: + // the directory only conditionally exists if + // there are any non-deleted children, so + // we don't create their parents. + if dirNode.isDir() { + dirNode.children[base] = childNode + } + break + } + if !dirNode.isDir() { + // This path already exists as a file, so it can't be a parent + // directory. See comment at error above. + return fmt.Errorf("invalid overlay: path %v is used as both file and directory", dir) + } + dirNode.children[base] = childNode + parent := filepath.Dir(dir) + if parent == dir { + break // reached the top; there is no parent + } + dir, base = parent, filepath.Base(dir) + childNode = dirNode + } + } + + return nil +} + +// IsDir returns true if path is a directory on disk or in the +// overlay. +func IsDir(path string) (bool, error) { + path = canonicalize(path) + + if _, ok := parentIsOverlayFile(path); ok { + return false, nil + } + + if n, ok := overlay[path]; ok { + return n.isDir(), nil + } + + fi, err := os.Stat(path) + if err != nil { + return false, err + } + + return fi.IsDir(), nil +} + +// parentIsOverlayFile returns whether name or any of +// its parents are files in the overlay, and the first parent found, +// including name itself, that's a file in the overlay. +func parentIsOverlayFile(name string) (string, bool) { + if overlay != nil { + // Check if name can't possibly be a directory because + // it or one of its parents is overlaid with a file. + // TODO(matloob): Maybe save this to avoid doing it every time? + prefix := name + for { + node := overlay[prefix] + if node != nil && !node.isDir() { + return prefix, true + } + parent := filepath.Dir(prefix) + if parent == prefix { + break + } + prefix = parent + } + } + + return "", false +} + +// errNotDir is used to communicate from ReadDir to IsDirWithGoFiles +// that the argument is not a directory, so that IsDirWithGoFiles doesn't +// return an error. +var errNotDir = errors.New("not a directory") + +// readDir reads a dir on disk, returning an error that is errNotDir if the dir is not a directory. +// Unfortunately, the error returned by ioutil.ReadDir if dir is not a directory +// can vary depending on the OS (Linux, Mac, Windows return ENOTDIR; BSD returns EINVAL). +func readDir(dir string) ([]fs.FileInfo, error) { + fis, err := ioutil.ReadDir(dir) + if err == nil { + return fis, nil + } + + if os.IsNotExist(err) { + return nil, err + } + if dirfi, staterr := os.Stat(dir); staterr == nil && !dirfi.IsDir() { + return nil, &fs.PathError{Op: "ReadDir", Path: dir, Err: errNotDir} + } + return nil, err +} + +// ReadDir provides a slice of fs.FileInfo entries corresponding +// to the overlaid files in the directory. +func ReadDir(dir string) ([]fs.FileInfo, error) { + dir = canonicalize(dir) + if _, ok := parentIsOverlayFile(dir); ok { + return nil, &fs.PathError{Op: "ReadDir", Path: dir, Err: errNotDir} + } + + dirNode := overlay[dir] + if dirNode == nil { + return readDir(dir) + } + if dirNode.isDeleted() { + return nil, &fs.PathError{Op: "ReadDir", Path: dir, Err: fs.ErrNotExist} + } + diskfis, err := readDir(dir) + if err != nil && !os.IsNotExist(err) && !errors.Is(err, errNotDir) { + return nil, err + } + + // Stat files in overlay to make composite list of fileinfos + files := make(map[string]fs.FileInfo) + for _, f := range diskfis { + files[f.Name()] = f + } + for name, to := range dirNode.children { + switch { + case to.isDir(): + files[name] = fakeDir(name) + case to.isDeleted(): + delete(files, name) + default: + // This is a regular file. + f, err := os.Lstat(to.actualFilePath) + if err != nil { + files[name] = missingFile(name) + continue + } else if f.IsDir() { + return nil, fmt.Errorf("for overlay of %q to %q: overlay Replace entries can't point to dirctories", + filepath.Join(dir, name), to.actualFilePath) + } + // Add a fileinfo for the overlaid file, so that it has + // the original file's name, but the overlaid file's metadata. + files[name] = fakeFile{name, f} + } + } + sortedFiles := diskfis[:0] + for _, f := range files { + sortedFiles = append(sortedFiles, f) + } + sort.Slice(sortedFiles, func(i, j int) bool { return sortedFiles[i].Name() < sortedFiles[j].Name() }) + return sortedFiles, nil +} + +// OverlayPath returns the path to the overlaid contents of the +// file, the empty string if the overlay deletes the file, or path +// itself if the file is not in the overlay, the file is a directory +// in the overlay, or there is no overlay. +// It returns true if the path is overlaid with a regular file +// or deleted, and false otherwise. +func OverlayPath(path string) (string, bool) { + if p, ok := overlay[canonicalize(path)]; ok && !p.isDir() { + return p.actualFilePath, ok + } + + return path, false +} + +// Open opens the file at or overlaid on the given path. +func Open(path string) (*os.File, error) { + return OpenFile(path, os.O_RDONLY, 0) +} + +// OpenFile opens the file at or overlaid on the given path with the flag and perm. +func OpenFile(path string, flag int, perm os.FileMode) (*os.File, error) { + cpath := canonicalize(path) + if node, ok := overlay[cpath]; ok { + // Opening a file in the overlay. + if node.isDir() { + return nil, &fs.PathError{Op: "OpenFile", Path: path, Err: errors.New("fsys.OpenFile doesn't support opening directories yet")} + } + // We can't open overlaid paths for write. + if perm != os.FileMode(os.O_RDONLY) { + return nil, &fs.PathError{Op: "OpenFile", Path: path, Err: errors.New("overlaid files can't be opened for write")} + } + return os.OpenFile(node.actualFilePath, flag, perm) + } + if parent, ok := parentIsOverlayFile(filepath.Dir(cpath)); ok { + // The file is deleted explicitly in the Replace map, + // or implicitly because one of its parent directories was + // replaced by a file. + return nil, &fs.PathError{ + Op: "Open", + Path: path, + Err: fmt.Errorf("file %s does not exist: parent directory %s is replaced by a file in overlay", path, parent), + } + } + return os.OpenFile(cpath, flag, perm) +} + +// IsDirWithGoFiles reports whether dir is a directory containing Go files +// either on disk or in the overlay. +func IsDirWithGoFiles(dir string) (bool, error) { + fis, err := ReadDir(dir) + if os.IsNotExist(err) || errors.Is(err, errNotDir) { + return false, nil + } + if err != nil { + return false, err + } + + var firstErr error + for _, fi := range fis { + if fi.IsDir() { + continue + } + + // TODO(matloob): this enforces that the "from" in the map + // has a .go suffix, but the actual destination file + // doesn't need to have a .go suffix. Is this okay with the + // compiler? + if !strings.HasSuffix(fi.Name(), ".go") { + continue + } + if fi.Mode().IsRegular() { + return true, nil + } + + // fi is the result of an Lstat, so it doesn't follow symlinks. + // But it's okay if the file is a symlink pointing to a regular + // file, so use os.Stat to follow symlinks and check that. + actualFilePath, _ := OverlayPath(filepath.Join(dir, fi.Name())) + fi, err := os.Stat(actualFilePath) + if err == nil && fi.Mode().IsRegular() { + return true, nil + } + if err != nil && firstErr == nil { + firstErr = err + } + } + + // No go files found in directory. + return false, firstErr +} + +// walk recursively descends path, calling walkFn. Copied, with some +// modifications from path/filepath.walk. +func walk(path string, info fs.FileInfo, walkFn filepath.WalkFunc) error { + if !info.IsDir() { + return walkFn(path, info, nil) + } + + fis, readErr := ReadDir(path) + walkErr := walkFn(path, info, readErr) + // If readErr != nil, walk can't walk into this directory. + // walkErr != nil means walkFn want walk to skip this directory or stop walking. + // Therefore, if one of readErr and walkErr isn't nil, walk will return. + if readErr != nil || walkErr != nil { + // The caller's behavior is controlled by the return value, which is decided + // by walkFn. walkFn may ignore readErr and return nil. + // If walkFn returns SkipDir, it will be handled by the caller. + // So walk should return whatever walkFn returns. + return walkErr + } + + for _, fi := range fis { + filename := filepath.Join(path, fi.Name()) + if walkErr = walk(filename, fi, walkFn); walkErr != nil { + if !fi.IsDir() || walkErr != filepath.SkipDir { + return walkErr + } + } + } + return nil +} + +// Walk walks the file tree rooted at root, calling walkFn for each file or +// directory in the tree, including root. +func Walk(root string, walkFn filepath.WalkFunc) error { + info, err := Lstat(root) + if err != nil { + err = walkFn(root, nil, err) + } else { + err = walk(root, info, walkFn) + } + if err == filepath.SkipDir { + return nil + } + return err +} + +// lstat implements a version of os.Lstat that operates on the overlay filesystem. +func Lstat(path string) (fs.FileInfo, error) { + return overlayStat(path, os.Lstat, "lstat") +} + +// Stat implements a version of os.Stat that operates on the overlay filesystem. +func Stat(path string) (fs.FileInfo, error) { + return overlayStat(path, os.Stat, "stat") +} + +// overlayStat implements lstat or Stat (depending on whether os.Lstat or os.Stat is passed in). +func overlayStat(path string, osStat func(string) (fs.FileInfo, error), opName string) (fs.FileInfo, error) { + cpath := canonicalize(path) + + if _, ok := parentIsOverlayFile(filepath.Dir(cpath)); ok { + return nil, &fs.PathError{Op: opName, Path: cpath, Err: fs.ErrNotExist} + } + + node, ok := overlay[cpath] + if !ok { + // The file or directory is not overlaid. + return osStat(path) + } + + switch { + case node.isDeleted(): + return nil, &fs.PathError{Op: "lstat", Path: cpath, Err: fs.ErrNotExist} + case node.isDir(): + return fakeDir(filepath.Base(path)), nil + default: + fi, err := osStat(node.actualFilePath) + if err != nil { + return nil, err + } + return fakeFile{name: filepath.Base(path), real: fi}, nil + } +} + +// fakeFile provides an fs.FileInfo implementation for an overlaid file, +// so that the file has the name of the overlaid file, but takes all +// other characteristics of the replacement file. +type fakeFile struct { + name string + real fs.FileInfo +} + +func (f fakeFile) Name() string { return f.name } +func (f fakeFile) Size() int64 { return f.real.Size() } +func (f fakeFile) Mode() fs.FileMode { return f.real.Mode() } +func (f fakeFile) ModTime() time.Time { return f.real.ModTime() } +func (f fakeFile) IsDir() bool { return f.real.IsDir() } +func (f fakeFile) Sys() interface{} { return f.real.Sys() } + +// missingFile provides an fs.FileInfo for an overlaid file where the +// destination file in the overlay doesn't exist. It returns zero values +// for the fileInfo methods other than Name, set to the file's name, and Mode +// set to ModeIrregular. +type missingFile string + +func (f missingFile) Name() string { return string(f) } +func (f missingFile) Size() int64 { return 0 } +func (f missingFile) Mode() fs.FileMode { return fs.ModeIrregular } +func (f missingFile) ModTime() time.Time { return time.Unix(0, 0) } +func (f missingFile) IsDir() bool { return false } +func (f missingFile) Sys() interface{} { return nil } + +// fakeDir provides an fs.FileInfo implementation for directories that are +// implicitly created by overlaid files. Each directory in the +// path of an overlaid file is considered to exist in the overlay filesystem. +type fakeDir string + +func (f fakeDir) Name() string { return string(f) } +func (f fakeDir) Size() int64 { return 0 } +func (f fakeDir) Mode() fs.FileMode { return fs.ModeDir | 0500 } +func (f fakeDir) ModTime() time.Time { return time.Unix(0, 0) } +func (f fakeDir) IsDir() bool { return true } +func (f fakeDir) Sys() interface{} { return nil } + +// Glob is like filepath.Glob but uses the overlay file system. +func Glob(pattern string) (matches []string, err error) { + // Check pattern is well-formed. + if _, err := filepath.Match(pattern, ""); err != nil { + return nil, err + } + if !hasMeta(pattern) { + if _, err = Lstat(pattern); err != nil { + return nil, nil + } + return []string{pattern}, nil + } + + dir, file := filepath.Split(pattern) + volumeLen := 0 + if runtime.GOOS == "windows" { + volumeLen, dir = cleanGlobPathWindows(dir) + } else { + dir = cleanGlobPath(dir) + } + + if !hasMeta(dir[volumeLen:]) { + return glob(dir, file, nil) + } + + // Prevent infinite recursion. See issue 15879. + if dir == pattern { + return nil, filepath.ErrBadPattern + } + + var m []string + m, err = Glob(dir) + if err != nil { + return + } + for _, d := range m { + matches, err = glob(d, file, matches) + if err != nil { + return + } + } + return +} + +// cleanGlobPath prepares path for glob matching. +func cleanGlobPath(path string) string { + switch path { + case "": + return "." + case string(filepath.Separator): + // do nothing to the path + return path + default: + return path[0 : len(path)-1] // chop off trailing separator + } +} + +func volumeNameLen(path string) int { + isSlash := func(c uint8) bool { + return c == '\\' || c == '/' + } + if len(path) < 2 { + return 0 + } + // with drive letter + c := path[0] + if path[1] == ':' && ('a' <= c && c <= 'z' || 'A' <= c && c <= 'Z') { + return 2 + } + // is it UNC? https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx + if l := len(path); l >= 5 && isSlash(path[0]) && isSlash(path[1]) && + !isSlash(path[2]) && path[2] != '.' { + // first, leading `\\` and next shouldn't be `\`. its server name. + for n := 3; n < l-1; n++ { + // second, next '\' shouldn't be repeated. + if isSlash(path[n]) { + n++ + // third, following something characters. its share name. + if !isSlash(path[n]) { + if path[n] == '.' { + break + } + for ; n < l; n++ { + if isSlash(path[n]) { + break + } + } + return n + } + break + } + } + } + return 0 +} + +// cleanGlobPathWindows is windows version of cleanGlobPath. +func cleanGlobPathWindows(path string) (prefixLen int, cleaned string) { + vollen := volumeNameLen(path) + switch { + case path == "": + return 0, "." + case vollen+1 == len(path) && os.IsPathSeparator(path[len(path)-1]): // /, \, C:\ and C:/ + // do nothing to the path + return vollen + 1, path + case vollen == len(path) && len(path) == 2: // C: + return vollen, path + "." // convert C: into C:. + default: + if vollen >= len(path) { + vollen = len(path) - 1 + } + return vollen, path[0 : len(path)-1] // chop off trailing separator + } +} + +// glob searches for files matching pattern in the directory dir +// and appends them to matches. If the directory cannot be +// opened, it returns the existing matches. New matches are +// added in lexicographical order. +func glob(dir, pattern string, matches []string) (m []string, e error) { + m = matches + fi, err := Stat(dir) + if err != nil { + return // ignore I/O error + } + if !fi.IsDir() { + return // ignore I/O error + } + + list, err := ReadDir(dir) + if err != nil { + return // ignore I/O error + } + + var names []string + for _, info := range list { + names = append(names, info.Name()) + } + sort.Strings(names) + + for _, n := range names { + matched, err := filepath.Match(pattern, n) + if err != nil { + return m, err + } + if matched { + m = append(m, filepath.Join(dir, n)) + } + } + return +} + +// hasMeta reports whether path contains any of the magic characters +// recognized by filepath.Match. +func hasMeta(path string) bool { + magicChars := `*?[` + if runtime.GOOS != "windows" { + magicChars = `*?[\` + } + return strings.ContainsAny(path, magicChars) +} diff --git a/cmd/go/_internal_/imports/build.go b/cmd/go/_internal_/imports/build.go index a5b0d88..8e7d61d 100644 --- a/cmd/go/_internal_/imports/build.go +++ b/cmd/go/_internal_/imports/build.go @@ -141,6 +141,9 @@ func matchTag(name string, tags map[string]bool, want bool) bool { if name == "solaris" { have = have || tags["illumos"] } + if name == "darwin" { + have = have || tags["ios"] + } return have == want } @@ -158,6 +161,7 @@ func matchTag(name string, tags map[string]bool, want bool) bool { // Exceptions: // if GOOS=android, then files with GOOS=linux are also matched. // if GOOS=illumos, then files with GOOS=solaris are also matched. +// if GOOS=ios, then files with GOOS=darwin are also matched. // // If tags["*"] is true, then MatchFile will consider all possible // GOOS and GOARCH to be available and will consequently @@ -208,6 +212,7 @@ var KnownOS = map[string]bool{ "freebsd": true, "hurd": true, "illumos": true, + "ios": true, "js": true, "linux": true, "nacl": true, // legacy; don't remove diff --git a/cmd/go/_internal_/imports/read.go b/cmd/go/_internal_/imports/read.go index 1623e1a..7a3cd76 100644 --- a/cmd/go/_internal_/imports/read.go +++ b/cmd/go/_internal_/imports/read.go @@ -198,7 +198,7 @@ func (r *importReader) readImport(imports *[]string) { r.readString(imports) } -// ReadComments is like ioutil.ReadAll, except that it only reads the leading +// ReadComments is like io.ReadAll, except that it only reads the leading // block of comments in the file. func ReadComments(f io.Reader) ([]byte, error) { r := &importReader{b: bufio.NewReader(f)} @@ -210,7 +210,7 @@ func ReadComments(f io.Reader) ([]byte, error) { return r.buf, r.err } -// ReadImports is like ioutil.ReadAll, except that it expects a Go file as input +// ReadImports is like io.ReadAll, except that it expects a Go file as input // and stops reading the input once the imports have completed. func ReadImports(f io.Reader, reportSyntaxError bool, imports *[]string) ([]byte, error) { r := &importReader{b: bufio.NewReader(f)} diff --git a/cmd/go/_internal_/imports/scan.go b/cmd/go/_internal_/imports/scan.go index 06c7c1a..4d307bb 100644 --- a/cmd/go/_internal_/imports/scan.go +++ b/cmd/go/_internal_/imports/scan.go @@ -6,16 +6,17 @@ package imports import ( "fmt" - "io/ioutil" - "os" + "io/fs" "path/filepath" "sort" "strconv" "strings" + + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/fsys" ) func ScanDir(dir string, tags map[string]bool) ([]string, []string, error) { - infos, err := ioutil.ReadDir(dir) + infos, err := fsys.ReadDir(dir) if err != nil { return nil, nil, err } @@ -25,14 +26,14 @@ func ScanDir(dir string, tags map[string]bool) ([]string, []string, error) { // If the directory entry is a symlink, stat it to obtain the info for the // link target instead of the link itself. - if info.Mode()&os.ModeSymlink != 0 { - info, err = os.Stat(filepath.Join(dir, name)) + if info.Mode()&fs.ModeSymlink != 0 { + info, err = fsys.Stat(filepath.Join(dir, name)) if err != nil { continue // Ignore broken symlinks. } } - if info.Mode().IsRegular() && !strings.HasPrefix(name, "_") && strings.HasSuffix(name, ".go") && MatchFile(name, tags) { + if info.Mode().IsRegular() && !strings.HasPrefix(name, "_") && !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".go") && MatchFile(name, tags) { files = append(files, filepath.Join(dir, name)) } } @@ -49,7 +50,7 @@ func scanFiles(files []string, tags map[string]bool, explicitFiles bool) ([]stri numFiles := 0 Files: for _, name := range files { - r, err := os.Open(name) + r, err := fsys.Open(name) if err != nil { return nil, nil, err } diff --git a/cmd/go/_internal_/imports/tags.go b/cmd/go/_internal_/imports/tags.go index c6ad8ef..75571f7 100644 --- a/cmd/go/_internal_/imports/tags.go +++ b/cmd/go/_internal_/imports/tags.go @@ -4,17 +4,23 @@ package imports -import "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" +import ( + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" + "sync" +) -var tags map[string]bool +var ( + tags map[string]bool + tagsOnce sync.Once +) // Tags returns a set of build tags that are true for the target platform. // It includes GOOS, GOARCH, the compiler, possibly "cgo", // release tags like "go1.13", and user-specified build tags. func Tags() map[string]bool { - if tags == nil { + tagsOnce.Do(func() { tags = loadTags() - } + }) return tags } @@ -36,14 +42,17 @@ func loadTags() map[string]bool { return tags } -var anyTags map[string]bool +var ( + anyTags map[string]bool + anyTagsOnce sync.Once +) // AnyTags returns a special set of build tags that satisfy nearly all // build tag expressions. Only "ignore" and malformed build tag requirements // are considered false. func AnyTags() map[string]bool { - if anyTags == nil { + anyTagsOnce.Do(func() { anyTags = map[string]bool{"*": true} - } + }) return anyTags } diff --git a/cmd/go/_internal_/lockedfile/_internal_/filelock/filelock.go b/cmd/go/_internal_/lockedfile/_internal_/filelock/filelock.go index aba3eed..05f27c3 100644 --- a/cmd/go/_internal_/lockedfile/_internal_/filelock/filelock.go +++ b/cmd/go/_internal_/lockedfile/_internal_/filelock/filelock.go @@ -9,6 +9,7 @@ package filelock import ( "errors" + "io/fs" "os" ) @@ -24,7 +25,7 @@ type File interface { Fd() uintptr // Stat returns the FileInfo structure describing file. - Stat() (os.FileInfo, error) + Stat() (fs.FileInfo, error) } // Lock places an advisory write lock on the file, blocking until it can be @@ -87,7 +88,7 @@ var ErrNotSupported = errors.New("operation not supported") // underlyingError returns the underlying error for known os error types. func underlyingError(err error) error { switch err := err.(type) { - case *os.PathError: + case *fs.PathError: return err.Err case *os.LinkError: return err.Err diff --git a/cmd/go/_internal_/lockedfile/_internal_/filelock/filelock_unix.go b/cmd/go/_internal_/lockedfile/_internal_/filelock/filelock_unix.go index 877921c..6f3ad07 100644 --- a/cmd/go/_internal_/lockedfile/_internal_/filelock/filelock_unix.go +++ b/cmd/go/_internal_/lockedfile/_internal_/filelock/filelock_unix.go @@ -2,12 +2,12 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux netbsd openbsd +// +build darwin dragonfly freebsd illumos linux netbsd openbsd package filelock import ( - "os" + "io/fs" "syscall" ) @@ -26,7 +26,7 @@ func lock(f File, lt lockType) (err error) { } } if err != nil { - return &os.PathError{ + return &fs.PathError{ Op: lt.String(), Path: f.Name(), Err: err, diff --git a/cmd/go/_internal_/lockedfile/lockedfile.go b/cmd/go/_internal_/lockedfile/lockedfile.go index decda06..3039c3d 100644 --- a/cmd/go/_internal_/lockedfile/lockedfile.go +++ b/cmd/go/_internal_/lockedfile/lockedfile.go @@ -9,7 +9,7 @@ package lockedfile import ( "fmt" "io" - "io/ioutil" + "io/fs" "os" "runtime" ) @@ -35,7 +35,7 @@ type osFile struct { // OpenFile is like os.OpenFile, but returns a locked file. // If flag includes os.O_WRONLY or os.O_RDWR, the file is write-locked; // otherwise, it is read-locked. -func OpenFile(name string, flag int, perm os.FileMode) (*File, error) { +func OpenFile(name string, flag int, perm fs.FileMode) (*File, error) { var ( f = new(File) err error @@ -82,10 +82,10 @@ func Edit(name string) (*File, error) { // non-nil error. func (f *File) Close() error { if f.closed { - return &os.PathError{ + return &fs.PathError{ Op: "close", Path: f.Name(), - Err: os.ErrClosed, + Err: fs.ErrClosed, } } f.closed = true @@ -103,12 +103,12 @@ func Read(name string) ([]byte, error) { } defer f.Close() - return ioutil.ReadAll(f) + return io.ReadAll(f) } // Write opens the named file (creating it with the given permissions if needed), // then write-locks it and overwrites it with the given content. -func Write(name string, content io.Reader, perm os.FileMode) (err error) { +func Write(name string, content io.Reader, perm fs.FileMode) (err error) { f, err := OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) if err != nil { return err @@ -135,7 +135,7 @@ func Transform(name string, t func([]byte) ([]byte, error)) (err error) { } defer f.Close() - old, err := ioutil.ReadAll(f) + old, err := io.ReadAll(f) if err != nil { return err } diff --git a/cmd/go/_internal_/lockedfile/lockedfile_filelock.go b/cmd/go/_internal_/lockedfile/lockedfile_filelock.go index 5313c32..c69156d 100644 --- a/cmd/go/_internal_/lockedfile/lockedfile_filelock.go +++ b/cmd/go/_internal_/lockedfile/lockedfile_filelock.go @@ -7,18 +7,20 @@ package lockedfile import ( + "io/fs" "os" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/fsys" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/lockedfile/_internal_/filelock" ) -func openFile(name string, flag int, perm os.FileMode) (*os.File, error) { +func openFile(name string, flag int, perm fs.FileMode) (*os.File, error) { // On BSD systems, we could add the O_SHLOCK or O_EXLOCK flag to the OpenFile // call instead of locking separately, but we have to support separate locking // calls for Linux and Windows anyway, so it's simpler to use that approach // consistently. - f, err := os.OpenFile(name, flag&^os.O_TRUNC, perm) + f, err := fsys.OpenFile(name, flag&^os.O_TRUNC, perm) if err != nil { return nil, err } diff --git a/cmd/go/_internal_/modconv/convert.go b/cmd/go/_internal_/modconv/convert.go index 7e8e36d..9d643c2 100644 --- a/cmd/go/_internal_/modconv/convert.go +++ b/cmd/go/_internal_/modconv/convert.go @@ -7,13 +7,12 @@ package modconv import ( "fmt" "os" + "runtime" "sort" "strings" - "sync" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/par" "golang.org/x/mod/modfile" "golang.org/x/mod/module" @@ -42,46 +41,54 @@ func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error { // Convert requirements block, which may use raw SHA1 hashes as versions, // to valid semver requirement list, respecting major versions. - var ( - work par.Work - mu sync.Mutex - need = make(map[string]string) - replace = make(map[string]*modfile.Replace) - ) + versions := make([]module.Version, len(mf.Require)) + replace := make(map[string]*modfile.Replace) for _, r := range mf.Replace { replace[r.New.Path] = r replace[r.Old.Path] = r } - for _, r := range mf.Require { + + type token struct{} + sem := make(chan token, runtime.GOMAXPROCS(0)) + for i, r := range mf.Require { m := r.Mod if m.Path == "" { continue } if re, ok := replace[m.Path]; ok { - work.Add(re.New) - continue + m = re.New } - work.Add(r.Mod) + sem <- token{} + go func(i int, m module.Version) { + defer func() { <-sem }() + repo, info, err := modfetch.ImportRepoRev(m.Path, m.Version) + if err != nil { + fmt.Fprintf(os.Stderr, "go: converting %s: stat %s@%s: %v\n", base.ShortPath(file), m.Path, m.Version, err) + return + } + + path := repo.ModulePath() + versions[i].Path = path + versions[i].Version = info.Version + }(i, m) + } + // Fill semaphore channel to wait for all tasks to finish. + for n := cap(sem); n > 0; n-- { + sem <- token{} } - work.Do(10, func(item interface{}) { - r := item.(module.Version) - repo, info, err := modfetch.ImportRepoRev(r.Path, r.Version) - if err != nil { - fmt.Fprintf(os.Stderr, "go: converting %s: stat %s@%s: %v\n", base.ShortPath(file), r.Path, r.Version, err) - return + need := map[string]string{} + for _, v := range versions { + if v.Path == "" { + continue } - mu.Lock() - path := repo.ModulePath() // Don't use semver.Max here; need to preserve +incompatible suffix. - if v, ok := need[path]; !ok || semver.Compare(v, info.Version) < 0 { - need[path] = info.Version + if needv, ok := need[v.Path]; !ok || semver.Compare(needv, v.Version) < 0 { + need[v.Path] = v.Version } - mu.Unlock() - }) - - var paths []string + } + paths := make([]string, 0, len(need)) for path := range need { paths = append(paths, path) } diff --git a/cmd/go/_internal_/modfetch/cache.go b/cmd/go/_internal_/modfetch/cache.go index 833ad24..a44b587 100644 --- a/cmd/go/_internal_/modfetch/cache.go +++ b/cmd/go/_internal_/modfetch/cache.go @@ -10,10 +10,11 @@ import ( "errors" "fmt" "io" - "io/ioutil" + "io/fs" "os" "path/filepath" "strings" + "sync" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" @@ -59,7 +60,7 @@ func CachePath(m module.Version, suffix string) (string, error) { // DownloadDir returns the directory to which m should have been downloaded. // An error will be returned if the module path or version cannot be escaped. -// An error satisfying errors.Is(err, os.ErrNotExist) will be returned +// An error satisfying errors.Is(err, fs.ErrNotExist) will be returned // along with the directory if the directory does not exist or if the directory // is not completely populated. func DownloadDir(m module.Version) (string, error) { @@ -83,6 +84,7 @@ func DownloadDir(m module.Version) (string, error) { return "", err } + // Check whether the directory itself exists. dir := filepath.Join(cfg.GOMODCACHE, enc+"@"+encVer) if fi, err := os.Stat(dir); os.IsNotExist(err) { return dir, err @@ -91,6 +93,9 @@ func DownloadDir(m module.Version) (string, error) { } else if !fi.IsDir() { return dir, &DownloadDirPartialError{dir, errors.New("not a directory")} } + + // Check if a .partial file exists. This is created at the beginning of + // a download and removed after the zip is extracted. partialPath, err := CachePath(m, "partial") if err != nil { return dir, err @@ -100,20 +105,33 @@ func DownloadDir(m module.Version) (string, error) { } else if !os.IsNotExist(err) { return dir, err } + + // Check if a .ziphash file exists. It should be created before the + // zip is extracted, but if it was deleted (by another program?), we need + // to re-calculate it. + ziphashPath, err := CachePath(m, "ziphash") + if err != nil { + return dir, err + } + if _, err := os.Stat(ziphashPath); os.IsNotExist(err) { + return dir, &DownloadDirPartialError{dir, errors.New("ziphash file is missing")} + } else if err != nil { + return dir, err + } return dir, nil } // DownloadDirPartialError is returned by DownloadDir if a module directory // exists but was not completely populated. // -// DownloadDirPartialError is equivalent to os.ErrNotExist. +// DownloadDirPartialError is equivalent to fs.ErrNotExist. type DownloadDirPartialError struct { Dir string Err error } func (e *DownloadDirPartialError) Error() string { return fmt.Sprintf("%s: %v", e.Dir, e.Err) } -func (e *DownloadDirPartialError) Is(err error) bool { return err == os.ErrNotExist } +func (e *DownloadDirPartialError) Is(err error) bool { return err == fs.ErrNotExist } // lockVersion locks a file within the module cache that guards the downloading // and extraction of the zipfile for the given module version. @@ -155,16 +173,30 @@ func SideLock() (unlock func(), err error) { type cachingRepo struct { path string cache par.Cache // cache for all operations - r Repo + + once sync.Once + initRepo func() (Repo, error) + r Repo } -func newCachingRepo(r Repo) *cachingRepo { +func newCachingRepo(path string, initRepo func() (Repo, error)) *cachingRepo { return &cachingRepo{ - r: r, - path: r.ModulePath(), + path: path, + initRepo: initRepo, } } +func (r *cachingRepo) repo() Repo { + r.once.Do(func() { + var err error + r.r, err = r.initRepo() + if err != nil { + r.r = errRepo{r.path, err} + } + }) + return r.r +} + func (r *cachingRepo) ModulePath() string { return r.path } @@ -175,7 +207,7 @@ func (r *cachingRepo) Versions(prefix string) ([]string, error) { err error } c := r.cache.Do("versions:"+prefix, func() interface{} { - list, err := r.r.Versions(prefix) + list, err := r.repo().Versions(prefix) return cached{list, err} }).(cached) @@ -197,7 +229,7 @@ func (r *cachingRepo) Stat(rev string) (*RevInfo, error) { return cachedInfo{info, nil} } - info, err = r.r.Stat(rev) + info, err = r.repo().Stat(rev) if err == nil { // If we resolved, say, 1234abcde to v0.0.0-20180604122334-1234abcdef78, // then save the information under the proper version, for future use. @@ -224,7 +256,7 @@ func (r *cachingRepo) Stat(rev string) (*RevInfo, error) { func (r *cachingRepo) Latest() (*RevInfo, error) { c := r.cache.Do("latest:", func() interface{} { - info, err := r.r.Latest() + info, err := r.repo().Latest() // Save info for likely future Stat call. if err == nil { @@ -258,7 +290,7 @@ func (r *cachingRepo) GoMod(version string) ([]byte, error) { return cached{text, nil} } - text, err = r.r.GoMod(version) + text, err = r.repo().GoMod(version) if err == nil { if err := checkGoMod(r.path, version, text); err != nil { return cached{text, err} @@ -277,26 +309,11 @@ func (r *cachingRepo) GoMod(version string) ([]byte, error) { } func (r *cachingRepo) Zip(dst io.Writer, version string) error { - return r.r.Zip(dst, version) -} - -// Stat is like Lookup(path).Stat(rev) but avoids the -// repository path resolution in Lookup if the result is -// already cached on local disk. -func Stat(proxy, path, rev string) (*RevInfo, error) { - _, info, err := readDiskStat(path, rev) - if err == nil { - return info, nil - } - repo, err := Lookup(proxy, path) - if err != nil { - return nil, err - } - return repo.Stat(rev) + return r.repo().Zip(dst, version) } -// InfoFile is like Stat but returns the name of the file containing -// the cached information. +// InfoFile is like Lookup(path).Stat(version) but returns the name of the file +// containing the cached information. func InfoFile(path, version string) (string, error) { if !semver.IsValid(version) { return "", fmt.Errorf("invalid version %q", version) @@ -307,10 +324,7 @@ func InfoFile(path, version string) (string, error) { } err := TryProxies(func(proxy string) error { - repo, err := Lookup(proxy, path) - if err == nil { - _, err = repo.Stat(version) - } + _, err := Lookup(proxy, path).Stat(version) return err }) if err != nil { @@ -336,11 +350,7 @@ func GoMod(path, rev string) ([]byte, error) { rev = info.Version } else { err := TryProxies(func(proxy string) error { - repo, err := Lookup(proxy, path) - if err != nil { - return err - } - info, err := repo.Stat(rev) + info, err := Lookup(proxy, path).Stat(rev) if err == nil { rev = info.Version } @@ -357,11 +367,8 @@ func GoMod(path, rev string) ([]byte, error) { return data, nil } - err = TryProxies(func(proxy string) error { - repo, err := Lookup(proxy, path) - if err == nil { - data, err = repo.GoMod(rev) - } + err = TryProxies(func(proxy string) (err error) { + data, err = Lookup(proxy, path).GoMod(rev) return err }) return data, err @@ -492,7 +499,7 @@ func readDiskStatByHash(path, rev string) (file string, info *RevInfo, err error for _, name := range names { if strings.HasSuffix(name, suffix) { v := strings.TrimSuffix(name, ".info") - if IsPseudoVersion(v) && semver.Max(maxVersion, v) == v { + if IsPseudoVersion(v) && semver.Compare(v, maxVersion) > 0 { maxVersion = v file, info, err = readDiskStat(path, strings.TrimSuffix(name, ".info")) } @@ -607,7 +614,7 @@ func rewriteVersionList(dir string) { } defer unlock() - infos, err := ioutil.ReadDir(dir) + infos, err := os.ReadDir(dir) if err != nil { return } diff --git a/cmd/go/_internal_/modfetch/codehost/codehost.go b/cmd/go/_internal_/modfetch/codehost/codehost.go index 40fd7f1..5089436 100644 --- a/cmd/go/_internal_/modfetch/codehost/codehost.go +++ b/cmd/go/_internal_/modfetch/codehost/codehost.go @@ -10,10 +10,10 @@ import ( "bytes" "crypto/sha256" "fmt" + exec "github.com/dependabot/gomodules-extracted/_internal_/execabs" "io" - "io/ioutil" + "io/fs" "os" - "os/exec" "path/filepath" "strings" "sync" @@ -79,9 +79,8 @@ type Repo interface { ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, err error) // RecentTag returns the most recent tag on rev or one of its predecessors - // with the given prefix and major version. - // An empty major string matches any major version. - RecentTag(rev, prefix, major string) (tag string, err error) + // with the given prefix. allowed may be used to filter out unwanted versions. + RecentTag(rev, prefix string, allowed func(string) bool) (tag string, err error) // DescendsFrom reports whether rev or any of its ancestors has the given tag. // @@ -106,7 +105,7 @@ type FileRev struct { Err error // error if any; os.IsNotExist(Err)==true if rev exists but file does not exist in that rev } -// UnknownRevisionError is an error equivalent to os.ErrNotExist, but for a +// UnknownRevisionError is an error equivalent to fs.ErrNotExist, but for a // revision rather than a file. type UnknownRevisionError struct { Rev string @@ -116,10 +115,10 @@ func (e *UnknownRevisionError) Error() string { return "unknown revision " + e.Rev } func (UnknownRevisionError) Is(err error) bool { - return err == os.ErrNotExist + return err == fs.ErrNotExist } -// ErrNoCommits is an error equivalent to os.ErrNotExist indicating that a given +// ErrNoCommits is an error equivalent to fs.ErrNotExist indicating that a given // repository or module contains no commits. var ErrNoCommits error = noCommitsError{} @@ -129,7 +128,7 @@ func (noCommitsError) Error() string { return "no commits" } func (noCommitsError) Is(err error) bool { - return err == os.ErrNotExist + return err == fs.ErrNotExist } // AllHex reports whether the revision rev is entirely lower-case hexadecimal digits. @@ -189,7 +188,7 @@ func WorkDir(typ, name string) (dir, lockfile string, err error) { } defer unlock() - data, err := ioutil.ReadFile(dir + ".info") + data, err := os.ReadFile(dir + ".info") info, err2 := os.Stat(dir) if err == nil && err2 == nil && info.IsDir() { // Info file and directory both already exist: reuse. @@ -211,7 +210,7 @@ func WorkDir(typ, name string) (dir, lockfile string, err error) { if err := os.MkdirAll(dir, 0777); err != nil { return "", "", err } - if err := ioutil.WriteFile(dir+".info", []byte(key), 0666); err != nil { + if err := os.WriteFile(dir+".info", []byte(key), 0666); err != nil { os.RemoveAll(dir) return "", "", err } @@ -264,6 +263,9 @@ func RunWithStdin(dir string, stdin io.Reader, cmdline ...interface{}) ([]byte, } cmd := str.StringList(cmdline...) + if os.Getenv("TESTGOVCS") == "panic" { + panic(fmt.Sprintf("use of vcs: %v", cmd)) + } if cfg.BuildX { text := new(strings.Builder) if dir != "" { diff --git a/cmd/go/_internal_/modfetch/codehost/git.go b/cmd/go/_internal_/modfetch/codehost/git.go index 993c170..3ac38b9 100644 --- a/cmd/go/_internal_/modfetch/codehost/git.go +++ b/cmd/go/_internal_/modfetch/codehost/git.go @@ -8,11 +8,11 @@ import ( "bytes" "errors" "fmt" + exec "github.com/dependabot/gomodules-extracted/_internal_/execabs" "io" - "io/ioutil" + "io/fs" "net/url" "os" - "os/exec" "path/filepath" "sort" "strconv" @@ -34,13 +34,13 @@ func LocalGitRepo(remote string) (Repo, error) { } // A notExistError wraps another error to retain its original text -// but makes it opaquely equivalent to os.ErrNotExist. +// but makes it opaquely equivalent to fs.ErrNotExist. type notExistError struct { err error } func (e notExistError) Error() string { return e.err.Error() } -func (notExistError) Is(err error) bool { return err == os.ErrNotExist } +func (notExistError) Is(err error) bool { return err == fs.ErrNotExist } const gitWorkDirType = "git3" @@ -188,7 +188,7 @@ func (r *gitRepo) loadRefs() { // For HTTP and HTTPS, that's easy to detect: we'll try to fetch the URL // ourselves and see what code it serves. if u, err := url.Parse(r.remoteURL); err == nil && (u.Scheme == "http" || u.Scheme == "https") { - if _, err := web.GetBytes(u); errors.Is(err, os.ErrNotExist) { + if _, err := web.GetBytes(u); errors.Is(err, fs.ErrNotExist) { gitErr = notExistError{gitErr} } } @@ -505,7 +505,7 @@ func (r *gitRepo) ReadFile(rev, file string, maxSize int64) ([]byte, error) { } out, err := Run(r.dir, "git", "cat-file", "blob", info.Name+":"+file) if err != nil { - return nil, os.ErrNotExist + return nil, fs.ErrNotExist } return out, nil } @@ -629,9 +629,9 @@ func (r *gitRepo) readFileRevs(tags []string, file string, fileMap map[string]*F case "tag", "commit": switch fileType { default: - f.Err = &os.PathError{Path: tag + ":" + file, Op: "read", Err: fmt.Errorf("unexpected non-blob type %q", fileType)} + f.Err = &fs.PathError{Path: tag + ":" + file, Op: "read", Err: fmt.Errorf("unexpected non-blob type %q", fileType)} case "missing": - f.Err = &os.PathError{Path: tag + ":" + file, Op: "read", Err: os.ErrNotExist} + f.Err = &fs.PathError{Path: tag + ":" + file, Op: "read", Err: fs.ErrNotExist} case "blob": f.Data = fileData } @@ -644,7 +644,7 @@ func (r *gitRepo) readFileRevs(tags []string, file string, fileMap map[string]*F return missing, nil } -func (r *gitRepo) RecentTag(rev, prefix, major string) (tag string, err error) { +func (r *gitRepo) RecentTag(rev, prefix string, allowed func(string) bool) (tag string, err error) { info, err := r.Stat(rev) if err != nil { return "", err @@ -680,7 +680,10 @@ func (r *gitRepo) RecentTag(rev, prefix, major string) (tag string, err error) { // NOTE: Do not replace the call to semver.Compare with semver.Max. // We want to return the actual tag, not a canonicalized version of it, // and semver.Max currently canonicalizes (see golang.org/issue/32700). - if c := semver.Canonical(semtag); c != "" && strings.HasPrefix(semtag, c) && (major == "" || semver.Major(c) == major) && semver.Compare(semtag, highest) > 0 { + if c := semver.Canonical(semtag); c == "" || !strings.HasPrefix(semtag, c) || !allowed(semtag) { + continue + } + if semver.Compare(semtag, highest) > 0 { highest = semtag } } @@ -823,12 +826,12 @@ func (r *gitRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, archive, err := Run(r.dir, "git", "-c", "core.autocrlf=input", "-c", "core.eol=lf", "archive", "--format=zip", "--prefix=prefix/", info.Name, args) if err != nil { if bytes.Contains(err.(*RunError).Stderr, []byte("did not match any files")) { - return nil, os.ErrNotExist + return nil, fs.ErrNotExist } return nil, err } - return ioutil.NopCloser(bytes.NewReader(archive)), nil + return io.NopCloser(bytes.NewReader(archive)), nil } // ensureGitAttributes makes sure export-subst and export-ignore features are @@ -859,7 +862,7 @@ func ensureGitAttributes(repoDir string) (err error) { } }() - b, err := ioutil.ReadAll(f) + b, err := io.ReadAll(f) if err != nil { return err } diff --git a/cmd/go/_internal_/modfetch/codehost/vcs.go b/cmd/go/_internal_/modfetch/codehost/vcs.go index 7abe7f5..9447864 100644 --- a/cmd/go/_internal_/modfetch/codehost/vcs.go +++ b/cmd/go/_internal_/modfetch/codehost/vcs.go @@ -9,7 +9,7 @@ import ( "fmt" "github.com/dependabot/gomodules-extracted/_internal_/lazyregexp" "io" - "io/ioutil" + "io/fs" "os" "path/filepath" "sort" @@ -377,7 +377,7 @@ func (r *vcsRepo) ReadFile(rev, file string, maxSize int64) ([]byte, error) { out, err := Run(r.dir, r.cmd.readFile(rev, file, r.remote)) if err != nil { - return nil, os.ErrNotExist + return nil, fs.ErrNotExist } return out, nil } @@ -395,7 +395,7 @@ func (r *vcsRepo) ReadFileRevs(revs []string, file string, maxSize int64) (map[s return nil, vcsErrorf("ReadFileRevs not implemented") } -func (r *vcsRepo) RecentTag(rev, prefix, major string) (tag string, err error) { +func (r *vcsRepo) RecentTag(rev, prefix string, allowed func(string) bool) (tag string, err error) { // We don't technically need to lock here since we're returning an error // uncondititonally, but doing so anyway will help to avoid baking in // lock-inversion bugs. @@ -432,7 +432,7 @@ func (r *vcsRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, if rev == "latest" { rev = r.cmd.latest } - f, err := ioutil.TempFile("", "go-readzip-*.zip") + f, err := os.CreateTemp("", "go-readzip-*.zip") if err != nil { return nil, err } @@ -567,7 +567,7 @@ func bzrParseStat(rev, out string) (*RevInfo, error) { func fossilParseStat(rev, out string) (*RevInfo, error) { for _, line := range strings.Split(out, "\n") { - if strings.HasPrefix(line, "uuid:") { + if strings.HasPrefix(line, "uuid:") || strings.HasPrefix(line, "hash:") { f := strings.Fields(line) if len(f) != 5 || len(f[1]) != 40 || f[4] != "UTC" { return nil, vcsErrorf("unexpected response from fossil info: %q", line) diff --git a/cmd/go/_internal_/modfetch/coderepo.go b/cmd/go/_internal_/modfetch/coderepo.go index d29dfe9..45f5621 100644 --- a/cmd/go/_internal_/modfetch/coderepo.go +++ b/cmd/go/_internal_/modfetch/coderepo.go @@ -10,7 +10,7 @@ import ( "errors" "fmt" "io" - "io/ioutil" + "io/fs" "os" "path" "sort" @@ -419,9 +419,14 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e tagPrefix = r.codeDir + "/" } + isRetracted, err := r.retractedVersions() + if err != nil { + isRetracted = func(string) bool { return false } + } + // tagToVersion returns the version obtained by trimming tagPrefix from tag. - // If the tag is invalid or a pseudo-version, tagToVersion returns an empty - // version. + // If the tag is invalid, retracted, or a pseudo-version, tagToVersion returns + // an empty version. tagToVersion := func(tag string) (v string, tagIsCanonical bool) { if !strings.HasPrefix(tag, tagPrefix) { return "", false @@ -436,6 +441,9 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e if v == "" || !strings.HasPrefix(trimmed, v) { return "", false // Invalid or incomplete version (just vX or vX.Y). } + if isRetracted(v) { + return "", false + } if v == trimmed { tagIsCanonical = true } @@ -500,15 +508,24 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e return checkGoMod() } + // Find the highest tagged version in the revision's history, subject to + // major version and +incompatible constraints. Use that version as the + // pseudo-version base so that the pseudo-version sorts higher. Ignore + // retracted versions. + allowedMajor := func(major string) func(v string) bool { + return func(v string) bool { + return (major == "" || semver.Major(v) == major) && !isRetracted(v) + } + } if pseudoBase == "" { var tag string if r.pseudoMajor != "" || canUseIncompatible() { - tag, _ = r.code.RecentTag(info.Name, tagPrefix, r.pseudoMajor) + tag, _ = r.code.RecentTag(info.Name, tagPrefix, allowedMajor(r.pseudoMajor)) } else { // Allow either v1 or v0, but not incompatible higher versions. - tag, _ = r.code.RecentTag(info.Name, tagPrefix, "v1") + tag, _ = r.code.RecentTag(info.Name, tagPrefix, allowedMajor("v1")) if tag == "" { - tag, _ = r.code.RecentTag(info.Name, tagPrefix, "v0") + tag, _ = r.code.RecentTag(info.Name, tagPrefix, allowedMajor("v0")) } } pseudoBase, _ = tagToVersion(tag) // empty if the tag is invalid @@ -869,6 +886,57 @@ func (r *codeRepo) modPrefix(rev string) string { return r.modPath + "@" + rev } +func (r *codeRepo) retractedVersions() (func(string) bool, error) { + versions, err := r.Versions("") + if err != nil { + return nil, err + } + + for i, v := range versions { + if strings.HasSuffix(v, "+incompatible") { + versions = versions[:i] + break + } + } + if len(versions) == 0 { + return func(string) bool { return false }, nil + } + + var highest string + for i := len(versions) - 1; i >= 0; i-- { + v := versions[i] + if semver.Prerelease(v) == "" { + highest = v + break + } + } + if highest == "" { + highest = versions[len(versions)-1] + } + + data, err := r.GoMod(highest) + if err != nil { + return nil, err + } + f, err := modfile.ParseLax("go.mod", data, nil) + if err != nil { + return nil, err + } + retractions := make([]modfile.VersionInterval, len(f.Retract)) + for _, r := range f.Retract { + retractions = append(retractions, r.VersionInterval) + } + + return func(v string) bool { + for _, r := range retractions { + if semver.Compare(r.Low, v) <= 0 && semver.Compare(v, r.High) <= 0 { + return true + } + } + return false + }, nil +} + func (r *codeRepo) Zip(dst io.Writer, version string) error { if version != module.CanonicalVersion(version) { return fmt.Errorf("version %s is not canonical", version) @@ -897,7 +965,7 @@ func (r *codeRepo) Zip(dst io.Writer, version string) error { subdir = strings.Trim(subdir, "/") // Spool to local file. - f, err := ioutil.TempFile("", "go-codehost-") + f, err := os.CreateTemp("", "go-codehost-") if err != nil { dl.Close() return err @@ -972,7 +1040,7 @@ type zipFile struct { } func (f zipFile) Path() string { return f.name } -func (f zipFile) Lstat() (os.FileInfo, error) { return f.f.FileInfo(), nil } +func (f zipFile) Lstat() (fs.FileInfo, error) { return f.f.FileInfo(), nil } func (f zipFile) Open() (io.ReadCloser, error) { return f.f.Open() } type dataFile struct { @@ -981,9 +1049,9 @@ type dataFile struct { } func (f dataFile) Path() string { return f.name } -func (f dataFile) Lstat() (os.FileInfo, error) { return dataFileInfo{f}, nil } +func (f dataFile) Lstat() (fs.FileInfo, error) { return dataFileInfo{f}, nil } func (f dataFile) Open() (io.ReadCloser, error) { - return ioutil.NopCloser(bytes.NewReader(f.data)), nil + return io.NopCloser(bytes.NewReader(f.data)), nil } type dataFileInfo struct { @@ -992,7 +1060,7 @@ type dataFileInfo struct { func (fi dataFileInfo) Name() string { return path.Base(fi.f.name) } func (fi dataFileInfo) Size() int64 { return int64(len(fi.f.data)) } -func (fi dataFileInfo) Mode() os.FileMode { return 0644 } +func (fi dataFileInfo) Mode() fs.FileMode { return 0644 } func (fi dataFileInfo) ModTime() time.Time { return time.Time{} } func (fi dataFileInfo) IsDir() bool { return false } func (fi dataFileInfo) Sys() interface{} { return nil } diff --git a/cmd/go/_internal_/modfetch/fetch.go b/cmd/go/_internal_/modfetch/fetch.go index 9bc1ce5..b986080 100644 --- a/cmd/go/_internal_/modfetch/fetch.go +++ b/cmd/go/_internal_/modfetch/fetch.go @@ -7,10 +7,11 @@ package modfetch import ( "archive/zip" "bytes" + "context" "errors" "fmt" "io" - "io/ioutil" + "io/fs" "os" "path/filepath" "sort" @@ -23,6 +24,7 @@ import ( "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/par" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/renameio" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/robustio" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/trace" "golang.org/x/mod/module" "golang.org/x/mod/sumdb/dirhash" @@ -34,7 +36,7 @@ var downloadCache par.Cache // Download downloads the specific module version to the // local download cache and returns the name of the directory // corresponding to the root of the module's file tree. -func Download(mod module.Version) (dir string, err error) { +func Download(ctx context.Context, mod module.Version) (dir string, err error) { if cfg.GOMODCACHE == "" { // modload.Init exits if GOPATH[0] is empty, and cfg.GOMODCACHE // is set to GOPATH[0]/pkg/mod if GOMODCACHE is empty, so this should never happen. @@ -47,7 +49,7 @@ func Download(mod module.Version) (dir string, err error) { err error } c := downloadCache.Do(mod, func() interface{} { - dir, err := download(mod) + dir, err := download(ctx, mod) if err != nil { return cached{"", err} } @@ -57,22 +59,22 @@ func Download(mod module.Version) (dir string, err error) { return c.dir, c.err } -func download(mod module.Version) (dir string, err error) { - // If the directory exists, and no .partial file exists, the module has - // already been completely extracted. .partial files may be created when a - // module zip directory is extracted in place instead of being extracted to a - // temporary directory and renamed. +func download(ctx context.Context, mod module.Version) (dir string, err error) { + ctx, span := trace.StartSpan(ctx, "modfetch.download "+mod.String()) + defer span.Done() + dir, err = DownloadDir(mod) if err == nil { + // The directory has already been completely extracted (no .partial file exists). return dir, nil - } else if dir == "" || !errors.Is(err, os.ErrNotExist) { + } else if dir == "" || !errors.Is(err, fs.ErrNotExist) { return "", err } // To avoid cluttering the cache with extraneous files, // DownloadZip uses the same lockfile as Download. // Invoke DownloadZip before locking the file. - zipfile, err := DownloadZip(mod) + zipfile, err := DownloadZip(ctx, mod) if err != nil { return "", err } @@ -83,6 +85,9 @@ func download(mod module.Version) (dir string, err error) { } defer unlock() + ctx, span = trace.StartSpan(ctx, "unzip "+zipfile) + defer span.Done() + // Check whether the directory was populated while we were waiting on the lock. _, dirErr := DownloadDir(mod) if dirErr == nil { @@ -90,10 +95,11 @@ func download(mod module.Version) (dir string, err error) { } _, dirExists := dirErr.(*DownloadDirPartialError) - // Clean up any remaining temporary directories from previous runs, as well - // as partially extracted diectories created by future versions of cmd/go. - // This is only safe to do because the lock file ensures that their writers - // are no longer active. + // Clean up any remaining temporary directories created by old versions + // (before 1.16), as well as partially extracted directories (indicated by + // DownloadDirPartialError, usually because of a .partial file). This is only + // safe to do because the lock file ensures that their writers are no longer + // active. parentDir := filepath.Dir(dir) tmpPrefix := filepath.Base(dir) + ".tmp-" if old, err := filepath.Glob(filepath.Join(parentDir, tmpPrefix+"*")); err == nil { @@ -111,91 +117,49 @@ func download(mod module.Version) (dir string, err error) { if err != nil { return "", err } - if err := os.Remove(partialPath); err != nil && !os.IsNotExist(err) { - return "", err - } - // Extract the module zip directory. + // Extract the module zip directory at its final location. // - // By default, we extract to a temporary directory, then atomically rename to - // its final location. We use the existence of the source directory to signal - // that it has been extracted successfully (see DownloadDir). If someone - // deletes the entire directory (e.g., as an attempt to prune out file - // corruption), the module cache will still be left in a recoverable - // state. + // To prevent other processes from reading the directory if we crash, + // create a .partial file before extracting the directory, and delete + // the .partial file afterward (all while holding the lock). // - // Unfortunately, os.Rename may fail with ERROR_ACCESS_DENIED on Windows if - // another process opens files in the temporary directory. This is partially - // mitigated by using robustio.Rename, which retries os.Rename for a short - // time. + // Before Go 1.16, we extracted to a temporary directory with a random name + // then renamed it into place with os.Rename. On Windows, this failed with + // ERROR_ACCESS_DENIED when another process (usually an anti-virus scanner) + // opened files in the temporary directory. // - // To avoid this error completely, if unzipInPlace is set, we instead create a - // .partial file (indicating the directory isn't fully extracted), then we - // extract the directory at its final location, then we delete the .partial - // file. This is not the default behavior because older versions of Go may - // simply stat the directory to check whether it exists without looking for a - // .partial file. If multiple versions run concurrently, the older version may - // assume a partially extracted directory is complete. - // TODO(golang.org/issue/36568): when these older versions are no longer - // supported, remove the old default behavior and the unzipInPlace flag. + // Go 1.14.2 and higher respect .partial files. Older versions may use + // partially extracted directories. 'go mod verify' can detect this, + // and 'go clean -modcache' can fix it. if err := os.MkdirAll(parentDir, 0777); err != nil { return "", err } - - if unzipInPlace { - if err := ioutil.WriteFile(partialPath, nil, 0666); err != nil { - return "", err - } - if err := modzip.Unzip(dir, mod, zipfile); err != nil { - fmt.Fprintf(os.Stderr, "-> %s\n", err) - if rmErr := RemoveAll(dir); rmErr == nil { - os.Remove(partialPath) - } - return "", err - } - if err := os.Remove(partialPath); err != nil { - return "", err - } - } else { - tmpDir, err := ioutil.TempDir(parentDir, tmpPrefix) - if err != nil { - return "", err - } - if err := modzip.Unzip(tmpDir, mod, zipfile); err != nil { - fmt.Fprintf(os.Stderr, "-> %s\n", err) - RemoveAll(tmpDir) - return "", err - } - if err := robustio.Rename(tmpDir, dir); err != nil { - RemoveAll(tmpDir) - return "", err + if err := os.WriteFile(partialPath, nil, 0666); err != nil { + return "", err + } + if err := modzip.Unzip(dir, mod, zipfile); err != nil { + fmt.Fprintf(os.Stderr, "-> %s\n", err) + if rmErr := RemoveAll(dir); rmErr == nil { + os.Remove(partialPath) } + return "", err + } + if err := os.Remove(partialPath); err != nil { + return "", err } if !cfg.ModCacheRW { - // Make dir read-only only *after* renaming it. - // os.Rename was observed to fail for read-only directories on macOS. makeDirsReadOnly(dir) } return dir, nil } -var unzipInPlace bool - -func init() { - for _, f := range strings.Split(os.Getenv("GODEBUG"), ",") { - if f == "modcacheunzipinplace=1" { - unzipInPlace = true - break - } - } -} - var downloadZipCache par.Cache // DownloadZip downloads the specific module version to the // local zip cache and returns the name of the zip file. -func DownloadZip(mod module.Version) (zipfile string, err error) { +func DownloadZip(ctx context.Context, mod module.Version) (zipfile string, err error) { // The par.Cache here avoids duplicate work. type cached struct { zipfile string @@ -206,13 +170,16 @@ func DownloadZip(mod module.Version) (zipfile string, err error) { if err != nil { return cached{"", err} } + ziphashfile := zipfile + "hash" - // Skip locking if the zipfile already exists. + // Return without locking if the zip and ziphash files exist. if _, err := os.Stat(zipfile); err == nil { - return cached{zipfile, nil} + if _, err := os.Stat(ziphashfile); err == nil { + return cached{zipfile, nil} + } } - // The zip file does not exist. Acquire the lock and create it. + // The zip or ziphash file does not exist. Acquire the lock and create them. if cfg.CmdName != "mod download" { fmt.Fprintf(os.Stderr, "go: downloading %s %s\n", mod.Path, mod.Version) } @@ -222,15 +189,7 @@ func DownloadZip(mod module.Version) (zipfile string, err error) { } defer unlock() - // Double-check that the zipfile was not created while we were waiting for - // the lock. - if _, err := os.Stat(zipfile); err == nil { - return cached{zipfile, nil} - } - if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil { - return cached{"", err} - } - if err := downloadZip(mod, zipfile); err != nil { + if err := downloadZip(ctx, mod, zipfile); err != nil { return cached{"", err} } return cached{zipfile, nil} @@ -238,7 +197,29 @@ func DownloadZip(mod module.Version) (zipfile string, err error) { return c.zipfile, c.err } -func downloadZip(mod module.Version, zipfile string) (err error) { +func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err error) { + ctx, span := trace.StartSpan(ctx, "modfetch.downloadZip "+zipfile) + defer span.Done() + + // Double-check that the zipfile was not created while we were waiting for + // the lock in DownloadZip. + ziphashfile := zipfile + "hash" + var zipExists, ziphashExists bool + if _, err := os.Stat(zipfile); err == nil { + zipExists = true + } + if _, err := os.Stat(ziphashfile); err == nil { + ziphashExists = true + } + if zipExists && ziphashExists { + return nil + } + + // Create parent directories. + if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil { + return err + } + // Clean up any remaining tempfiles from previous runs. // This is only safe to do because the lock file ensures that their // writers are no longer active. @@ -250,12 +231,18 @@ func downloadZip(mod module.Version, zipfile string) (err error) { } } + // If the zip file exists, the ziphash file must have been deleted + // or lost after a file system crash. Re-hash the zip without downloading. + if zipExists { + return hashZip(mod, zipfile, ziphashfile) + } + // From here to the os.Rename call below is functionally almost equivalent to // renameio.WriteToFile, with one key difference: we want to validate the // contents of the file (by hashing it) before we commit it. Because the file // is zip-compressed, we need an actual file — or at least an io.ReaderAt — to // validate it: we can't just tee the stream as we write it. - f, err := ioutil.TempFile(filepath.Dir(zipfile), filepath.Base(renameio.Pattern(zipfile))) + f, err := os.CreateTemp(filepath.Dir(zipfile), filepath.Base(renameio.Pattern(zipfile))) if err != nil { return err } @@ -266,12 +253,28 @@ func downloadZip(mod module.Version, zipfile string) (err error) { } }() + var unrecoverableErr error err = TryProxies(func(proxy string) error { - repo, err := Lookup(proxy, mod.Path) + if unrecoverableErr != nil { + return unrecoverableErr + } + repo := Lookup(proxy, mod.Path) + err := repo.Zip(f, mod.Version) if err != nil { - return err + // Zip may have partially written to f before failing. + // (Perhaps the server crashed while sending the file?) + // Since we allow fallback on error in some cases, we need to fix up the + // file to be empty again for the next attempt. + if _, err := f.Seek(0, io.SeekStart); err != nil { + unrecoverableErr = err + return err + } + if err := f.Truncate(0); err != nil { + unrecoverableErr = err + return err + } } - return repo.Zip(f, mod.Version) + return err }) if err != nil { return err @@ -306,15 +309,7 @@ func downloadZip(mod module.Version, zipfile string) (err error) { } // Hash the zip file and check the sum before renaming to the final location. - hash, err := dirhash.HashZip(f.Name(), dirhash.DefaultHash) - if err != nil { - return err - } - if err := checkModSum(mod, hash); err != nil { - return err - } - - if err := renameio.WriteFile(zipfile+"hash", []byte(hash), 0666); err != nil { + if err := hashZip(mod, f.Name(), ziphashfile); err != nil { return err } if err := os.Rename(f.Name(), zipfile); err != nil { @@ -326,17 +321,34 @@ func downloadZip(mod module.Version, zipfile string) (err error) { return nil } +// hashZip reads the zip file opened in f, then writes the hash to ziphashfile, +// overwriting that file if it exists. +// +// If the hash does not match go.sum (or the sumdb if enabled), hashZip returns +// an error and does not write ziphashfile. +func hashZip(mod module.Version, zipfile, ziphashfile string) error { + hash, err := dirhash.HashZip(zipfile, dirhash.DefaultHash) + if err != nil { + return err + } + if err := checkModSum(mod, hash); err != nil { + return err + } + return renameio.WriteFile(ziphashfile, []byte(hash), 0666) +} + // makeDirsReadOnly makes a best-effort attempt to remove write permissions for dir // and its transitive contents. func makeDirsReadOnly(dir string) { type pathMode struct { path string - mode os.FileMode + mode fs.FileMode } var dirs []pathMode // in lexical order - filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err == nil && info.Mode()&0222 != 0 { - if info.IsDir() { + filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err == nil && d.IsDir() { + info, err := d.Info() + if err == nil && info.Mode()&0222 != 0 { dirs = append(dirs, pathMode{path, info.Mode()}) } } @@ -353,7 +365,7 @@ func makeDirsReadOnly(dir string) { // any permission changes needed to do so. func RemoveAll(dir string) error { // Module cache has 0555 directories; make them writable in order to remove content. - filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + filepath.WalkDir(dir, func(path string, info fs.DirEntry, err error) error { if err != nil { return nil // ignore errors walking in file system } @@ -374,12 +386,14 @@ type modSum struct { var goSum struct { mu sync.Mutex - m map[module.Version][]string // content of go.sum file (+ go.modverify if present) - checked map[modSum]bool // sums actually checked during execution - dirty bool // whether we added any new sums to m + m map[module.Version][]string // content of go.sum file + status map[modSum]modSumStatus // state of sums in m overwrite bool // if true, overwrite go.sum without incorporating its contents enabled bool // whether to use go.sum at all - modverify string // path to go.modverify, to be deleted +} + +type modSumStatus struct { + used, dirty bool } // initGoSum initializes the go.sum data. @@ -395,7 +409,7 @@ func initGoSum() (bool, error) { } goSum.m = make(map[module.Version][]string) - goSum.checked = make(map[modSum]bool) + goSum.status = make(map[modSum]modSumStatus) data, err := lockedfile.Read(GoSumFile) if err != nil && !os.IsNotExist(err) { return false, err @@ -403,19 +417,6 @@ func initGoSum() (bool, error) { goSum.enabled = true readGoSum(goSum.m, GoSumFile, data) - // Add old go.modverify file. - // We'll delete go.modverify in WriteGoSum. - alt := strings.TrimSuffix(GoSumFile, ".sum") + ".modverify" - if data, err := renameio.ReadFile(alt); err == nil { - migrate := make(map[module.Version][]string) - readGoSum(migrate, alt, data) - for mod, sums := range migrate { - for _, sum := range sums { - addModSumLocked(mod, sum) - } - } - goSum.modverify = alt - } return true, nil } @@ -455,13 +456,30 @@ func readGoSum(dst map[module.Version][]string, file string, data []byte) error return nil } -// checkMod checks the given module's checksum. -func checkMod(mod module.Version) { - if cfg.GOMODCACHE == "" { - // Do not use current directory. - return +// HaveSum returns true if the go.sum file contains an entry for mod. +// The entry's hash must be generated with a known hash algorithm. +// mod.Version may have a "/go.mod" suffix to distinguish sums for +// .mod and .zip files. +func HaveSum(mod module.Version) bool { + goSum.mu.Lock() + defer goSum.mu.Unlock() + inited, err := initGoSum() + if err != nil || !inited { + return false } + for _, h := range goSum.m[mod] { + if !strings.HasPrefix(h, "h1:") { + continue + } + if !goSum.status[modSum{mod, h}].dirty { + return true + } + } + return false +} +// checkMod checks the given module's checksum. +func checkMod(mod module.Version) { // Do the file I/O before acquiring the go.sum lock. ziphash, err := CachePath(mod, "ziphash") if err != nil { @@ -469,10 +487,6 @@ func checkMod(mod module.Version) { } data, err := renameio.ReadFile(ziphash) if err != nil { - if errors.Is(err, os.ErrNotExist) { - // This can happen if someone does rm -rf GOPATH/src/cache/download. So it goes. - return - } base.Fatalf("verifying %v", module.VersionError(mod, err)) } h := strings.TrimSpace(string(data)) @@ -488,7 +502,7 @@ func checkMod(mod module.Version) { // goModSum returns the checksum for the go.mod contents. func goModSum(data []byte) (string, error) { return dirhash.Hash1([]string{"go.mod"}, func(string) (io.ReadCloser, error) { - return ioutil.NopCloser(bytes.NewReader(data)), nil + return io.NopCloser(bytes.NewReader(data)), nil }) } @@ -504,6 +518,9 @@ func checkGoMod(path, version string, data []byte) error { } // checkModSum checks that the recorded checksum for mod is h. +// +// mod.Version may have the additional suffix "/go.mod" to request the checksum +// for the module's go.mod file only. func checkModSum(mod module.Version, h string) error { // We lock goSum when manipulating it, // but we arrange to release the lock when calling checkSumDB, @@ -518,6 +535,11 @@ func checkModSum(mod module.Version, h string) error { return err } done := inited && haveModSumLocked(mod, h) + if inited { + st := goSum.status[modSum{mod, h}] + st.used = true + goSum.status[modSum{mod, h}] = st + } goSum.mu.Unlock() if done { @@ -537,6 +559,9 @@ func checkModSum(mod module.Version, h string) error { if inited { goSum.mu.Lock() addModSumLocked(mod, h) + st := goSum.status[modSum{mod, h}] + st.dirty = true + goSum.status[modSum{mod, h}] = st goSum.mu.Unlock() } return nil @@ -546,7 +571,6 @@ func checkModSum(mod module.Version, h string) error { // If it finds a conflicting pair instead, it calls base.Fatalf. // goSum.mu must be locked. func haveModSumLocked(mod module.Version, h string) bool { - goSum.checked[modSum{mod, h}] = true for _, vh := range goSum.m[mod] { if h == vh { return true @@ -568,15 +592,21 @@ func addModSumLocked(mod module.Version, h string) { fmt.Fprintf(os.Stderr, "warning: verifying %s@%s: unknown hashes in go.sum: %v; adding %v"+hashVersionMismatch, mod.Path, mod.Version, strings.Join(goSum.m[mod], ", "), h) } goSum.m[mod] = append(goSum.m[mod], h) - goSum.dirty = true } // checkSumDB checks the mod, h pair against the Go checksum database. // It calls base.Fatalf if the hash is to be rejected. func checkSumDB(mod module.Version, h string) error { + modWithoutSuffix := mod + noun := "module" + if strings.HasSuffix(mod.Version, "/go.mod") { + noun = "go.mod" + modWithoutSuffix.Version = strings.TrimSuffix(mod.Version, "/go.mod") + } + db, lines, err := lookupSumDB(mod) if err != nil { - return module.VersionError(mod, fmt.Errorf("verifying module: %v", err)) + return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: %v", noun, err)) } have := mod.Path + " " + mod.Version + " " + h @@ -586,7 +616,7 @@ func checkSumDB(mod module.Version, h string) error { return nil } if strings.HasPrefix(line, prefix) { - return module.VersionError(mod, fmt.Errorf("verifying module: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+sumdbMismatch, h, db, line[len(prefix)-len("h1:"):])) + return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+sumdbMismatch, noun, h, db, line[len(prefix)-len("h1:"):])) } } return nil @@ -612,18 +642,35 @@ func Sum(mod module.Version) string { } // WriteGoSum writes the go.sum file if it needs to be updated. -func WriteGoSum() { +// +// keep is used to check whether a newly added sum should be saved in go.sum. +// It should have entries for both module content sums and go.mod sums +// (version ends with "/go.mod"). Existing sums will be preserved unless they +// have been marked for deletion with TrimGoSum. +func WriteGoSum(keep map[module.Version]bool) { goSum.mu.Lock() defer goSum.mu.Unlock() + // If we haven't read the go.sum file yet, don't bother writing it. if !goSum.enabled { - // If we haven't read the go.sum file yet, don't bother writing it: at best, - // we could rename the go.modverify file if it isn't empty, but we haven't - // needed to touch it so far — how important could it be? return } - if !goSum.dirty { - // Don't bother opening the go.sum file if we don't have anything to add. + + // Check whether we need to add sums for which keep[m] is true or remove + // unused sums marked with TrimGoSum. If there are no changes to make, + // just return without opening go.sum. + dirty := false +Outer: + for m, hs := range goSum.m { + for _, h := range hs { + st := goSum.status[modSum{m, h}] + if st.dirty && (!st.used || keep[m]) { + dirty = true + break Outer + } + } + } + if !dirty { return } if cfg.BuildMod == "readonly" { @@ -644,9 +691,10 @@ func WriteGoSum() { // them without good reason. goSum.m = make(map[module.Version][]string, len(goSum.m)) readGoSum(goSum.m, GoSumFile, data) - for ms := range goSum.checked { - addModSumLocked(ms.mod, ms.sum) - goSum.dirty = true + for ms, st := range goSum.status { + if st.used { + addModSumLocked(ms.mod, ms.sum) + } } } @@ -661,7 +709,10 @@ func WriteGoSum() { list := goSum.m[m] sort.Strings(list) for _, h := range list { - fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h) + st := goSum.status[modSum{m, h}] + if !st.dirty || (st.used && keep[m]) { + fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h) + } } } return buf.Bytes(), nil @@ -671,16 +722,16 @@ func WriteGoSum() { base.Fatalf("go: updating go.sum: %v", err) } - goSum.checked = make(map[modSum]bool) - goSum.dirty = false + goSum.status = make(map[modSum]modSumStatus) goSum.overwrite = false - - if goSum.modverify != "" { - os.Remove(goSum.modverify) // best effort - } } -// TrimGoSum trims go.sum to contain only the modules for which keep[m] is true. +// TrimGoSum trims go.sum to contain only the modules needed for reproducible +// builds. +// +// keep is used to check whether a sum should be retained in go.mod. It should +// have entries for both module content sums and go.mod sums (version ends +// with "/go.mod"). func TrimGoSum(keep map[module.Version]bool) { goSum.mu.Lock() defer goSum.mu.Unlock() @@ -692,13 +743,11 @@ func TrimGoSum(keep map[module.Version]bool) { return } - for m := range goSum.m { - // If we're keeping x@v we also keep x@v/go.mod. - // Map x@v/go.mod back to x@v for the keep lookup. - noGoMod := module.Version{Path: m.Path, Version: strings.TrimSuffix(m.Version, "/go.mod")} - if !keep[m] && !keep[noGoMod] { - delete(goSum.m, m) - goSum.dirty = true + for m, hs := range goSum.m { + if !keep[m] { + for _, h := range hs { + goSum.status[modSum{m, h}] = modSumStatus{used: false, dirty: true} + } goSum.overwrite = true } } @@ -738,96 +787,20 @@ var HelpModuleAuth = &base.Command{ UsageLine: "module-auth", Short: "module authentication using go.sum", Long: ` -The go command tries to authenticate every downloaded module, -checking that the bits downloaded for a specific module version today -match bits downloaded yesterday. This ensures repeatable builds -and detects introduction of unexpected changes, malicious or not. - -In each module's root, alongside go.mod, the go command maintains -a file named go.sum containing the cryptographic checksums of the -module's dependencies. - -The form of each line in go.sum is three fields: - - [/go.mod] - -Each known module version results in two lines in the go.sum file. -The first line gives the hash of the module version's file tree. -The second line appends "/go.mod" to the version and gives the hash -of only the module version's (possibly synthesized) go.mod file. -The go.mod-only hash allows downloading and authenticating a -module version's go.mod file, which is needed to compute the -dependency graph, without also downloading all the module's source code. - -The hash begins with an algorithm prefix of the form "h:". -The only defined algorithm prefix is "h1:", which uses SHA-256. - -Module authentication failures - -The go command maintains a cache of downloaded packages and computes -and records the cryptographic checksum of each package at download time. -In normal operation, the go command checks the main module's go.sum file -against these precomputed checksums instead of recomputing them on -each command invocation. The 'go mod verify' command checks that -the cached copies of module downloads still match both their recorded -checksums and the entries in go.sum. - -In day-to-day development, the checksum of a given module version -should never change. Each time a dependency is used by a given main -module, the go command checks its local cached copy, freshly -downloaded or not, against the main module's go.sum. If the checksums -don't match, the go command reports the mismatch as a security error -and refuses to run the build. When this happens, proceed with caution: -code changing unexpectedly means today's build will not match -yesterday's, and the unexpected change may not be beneficial. - -If the go command reports a mismatch in go.sum, the downloaded code -for the reported module version does not match the one used in a -previous build of the main module. It is important at that point -to find out what the right checksum should be, to decide whether -go.sum is wrong or the downloaded code is wrong. Usually go.sum is right: -you want to use the same code you used yesterday. - -If a downloaded module is not yet included in go.sum and it is a publicly -available module, the go command consults the Go checksum database to fetch -the expected go.sum lines. If the downloaded code does not match those -lines, the go command reports the mismatch and exits. Note that the -database is not consulted for module versions already listed in go.sum. - -If a go.sum mismatch is reported, it is always worth investigating why -the code downloaded today differs from what was downloaded yesterday. - -The GOSUMDB environment variable identifies the name of checksum database -to use and optionally its public key and URL, as in: - - GOSUMDB="sum.golang.org" - GOSUMDB="sum.golang.org+" - GOSUMDB="sum.golang.org+ https://sum.golang.org" - -The go command knows the public key of sum.golang.org, and also that the name -sum.golang.google.cn (available inside mainland China) connects to the -sum.golang.org checksum database; use of any other database requires giving -the public key explicitly. -The URL defaults to "https://" followed by the database name. - -GOSUMDB defaults to "sum.golang.org", the Go checksum database run by Google. -See https://sum.golang.org/privacy for the service's privacy policy. - -If GOSUMDB is set to "off", or if "go get" is invoked with the -insecure flag, -the checksum database is not consulted, and all unrecognized modules are -accepted, at the cost of giving up the security guarantee of verified repeatable -downloads for all modules. A better way to bypass the checksum database -for specific modules is to use the GOPRIVATE or GONOSUMDB environment -variables. See 'go help module-private' for details. - -The 'go env -w' command (see 'go help env') can be used to set these variables -for future go command invocations. +When the go command downloads a module zip file or go.mod file into the +module cache, it computes a cryptographic hash and compares it with a known +value to verify the file hasn't changed since it was first downloaded. Known +hashes are stored in a file in the module root directory named go.sum. Hashes +may also be downloaded from the checksum database depending on the values of +GOSUMDB, GOPRIVATE, and GONOSUMDB. + +For details, see https://golang.org/ref/mod#authenticating. `, } -var HelpModulePrivate = &base.Command{ - UsageLine: "module-private", - Short: "module configuration for non-public modules", +var HelpPrivate = &base.Command{ + UsageLine: "private", + Short: "configuration for downloading non-public code", Long: ` The go command defaults to downloading modules from the public Go module mirror at proxy.golang.org. It also defaults to validating downloaded modules, @@ -835,8 +808,8 @@ regardless of source, against the public Go checksum database at sum.golang.org. These defaults work well for publicly available source code. The GOPRIVATE environment variable controls which modules the go command -considers to be private (not available publicly) and should therefore not use the -proxy or checksum database. The variable is a comma-separated list of +considers to be private (not available publicly) and should therefore not use +the proxy or checksum database. The variable is a comma-separated list of glob patterns (in the syntax of Go's path.Match) of module path prefixes. For example, @@ -846,10 +819,6 @@ causes the go command to treat as private any module with a path prefix matching either pattern, including git.corp.example.com/xyzzy, rsc.io/private, and rsc.io/private/quux. -The GOPRIVATE environment variable may be used by other tools as well to -identify non-public modules. For example, an editor could use GOPRIVATE -to decide whether to hyperlink a package import to a godoc.org page. - For fine-grained control over module download and validation, the GONOPROXY and GONOSUMDB environment variables accept the same kind of glob list and override GOPRIVATE for the specific decision of whether to use the proxy @@ -862,13 +831,14 @@ users would configure go using: GOPROXY=proxy.example.com GONOPROXY=none -This would tell the go command and other tools that modules beginning with -a corp.example.com subdomain are private but that the company proxy should -be used for downloading both public and private modules, because -GONOPROXY has been set to a pattern that won't match any modules, -overriding GOPRIVATE. +The GOPRIVATE variable is also used to define the "public" and "private" +patterns for the GOVCS variable; see 'go help vcs'. For that usage, +GOPRIVATE applies even in GOPATH mode. In that case, it matches import paths +instead of module paths. The 'go env -w' command (see 'go help env') can be used to set these variables for future go command invocations. + +For more details, see https://golang.org/ref/mod#private-modules. `, } diff --git a/cmd/go/_internal_/modfetch/insecure.go b/cmd/go/_internal_/modfetch/insecure.go index 6c5f5fd..0adb6ea 100644 --- a/cmd/go/_internal_/modfetch/insecure.go +++ b/cmd/go/_internal_/modfetch/insecure.go @@ -6,11 +6,11 @@ package modfetch import ( "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/get" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/str" + + "golang.org/x/mod/module" ) // allowInsecure reports whether we are allowed to fetch this path in an insecure manner. func allowInsecure(path string) bool { - return get.Insecure || str.GlobsMatchPath(cfg.GOINSECURE, path) + return cfg.Insecure || module.MatchPrefixPatterns(cfg.GOINSECURE, path) } diff --git a/cmd/go/_internal_/modfetch/proxy.go b/cmd/go/_internal_/modfetch/proxy.go index ba295b7..8baf20d 100644 --- a/cmd/go/_internal_/modfetch/proxy.go +++ b/cmd/go/_internal_/modfetch/proxy.go @@ -9,9 +9,8 @@ import ( "errors" "fmt" "io" - "io/ioutil" + "io/fs" "net/url" - "os" "path" pathpkg "path" "path/filepath" @@ -37,65 +36,8 @@ URLs of a specified form. The requests have no query parameters, so even a site serving from a fixed file system (including a file:/// URL) can be a module proxy. -The GET requests sent to a Go module proxy are: - -GET $GOPROXY//@v/list returns a list of known versions of the given -module, one per line. - -GET $GOPROXY//@v/.info returns JSON-formatted metadata -about that version of the given module. - -GET $GOPROXY//@v/.mod returns the go.mod file -for that version of the given module. - -GET $GOPROXY//@v/.zip returns the zip archive -for that version of the given module. - -GET $GOPROXY//@latest returns JSON-formatted metadata about the -latest known version of the given module in the same format as -/@v/.info. The latest version should be the version of -the module the go command may use if /@v/list is empty or no -listed version is suitable. /@latest is optional and may not -be implemented by a module proxy. - -When resolving the latest version of a module, the go command will request -/@v/list, then, if no suitable versions are found, /@latest. -The go command prefers, in order: the semantically highest release version, -the semantically highest pre-release version, and the chronologically -most recent pseudo-version. In Go 1.12 and earlier, the go command considered -pseudo-versions in /@v/list to be pre-release versions, but this is -no longer true since Go 1.13. - -To avoid problems when serving from case-sensitive file systems, -the and elements are case-encoded, replacing every -uppercase letter with an exclamation mark followed by the corresponding -lower-case letter: github.com/Azure encodes as github.com/!azure. - -The JSON-formatted metadata about a given module corresponds to -this Go data structure, which may be expanded in the future: - - type Info struct { - Version string // version string - Time time.Time // commit time - } - -The zip archive for a specific version of a given module is a -standard zip file that contains the file tree corresponding -to the module's source code and related files. The archive uses -slash-separated paths, and every file path in the archive must -begin with @/, where the module and version are -substituted directly, not case-encoded. The root of the module -file tree corresponds to the @/ prefix in the -archive. - -Even when downloading directly from version control systems, -the go command synthesizes explicit info, mod, and zip files -and stores them in its local cache, $GOPATH/pkg/mod/cache/download, -the same as if it had downloaded them directly from a proxy. -The cache layout is the same as the proxy URL space, so -serving $GOPATH/pkg/mod/cache/download at (or copying it to) -https://example.com/proxy would let other users access those -cached module versions with GOPROXY=https://example.com/proxy. +For details on the GOPROXY protocol, see +https://golang.org/ref/mod#goproxy-protocol. `, } @@ -186,7 +128,7 @@ func proxyList() ([]proxySpec, error) { // TryProxies iterates f over each configured proxy (including "noproxy" and // "direct" if applicable) until f returns no error or until f returns an -// error that is not equivalent to os.ErrNotExist on a proxy configured +// error that is not equivalent to fs.ErrNotExist on a proxy configured // not to fall back on errors. // // TryProxies then returns that final error. @@ -222,7 +164,7 @@ func TryProxies(f func(proxy string) error) error { if err == nil { return nil } - isNotExistErr := errors.Is(err, os.ErrNotExist) + isNotExistErr := errors.Is(err, fs.ErrNotExist) if proxy.url == "direct" || (proxy.url == "noproxy" && err != errUseProxy) { bestErr = err @@ -242,8 +184,9 @@ func TryProxies(f func(proxy string) error) error { } type proxyRepo struct { - url *url.URL - path string + url *url.URL + path string + redactedURL string } func newProxyRepo(baseURL, path string) (Repo, error) { @@ -268,10 +211,10 @@ func newProxyRepo(baseURL, path string) (Repo, error) { if err != nil { return nil, err } - + redactedURL := base.Redacted() base.Path = strings.TrimSuffix(base.Path, "/") + "/" + enc base.RawPath = strings.TrimSuffix(base.RawPath, "/") + "/" + pathEscape(enc) - return &proxyRepo{base, path}, nil + return &proxyRepo{base, path, redactedURL}, nil } func (p *proxyRepo) ModulePath() string { @@ -304,7 +247,7 @@ func (p *proxyRepo) getBytes(path string) ([]byte, error) { return nil, err } defer body.Close() - return ioutil.ReadAll(body) + return io.ReadAll(body) } func (p *proxyRepo) getBody(path string) (io.ReadCloser, error) { @@ -413,7 +356,7 @@ func (p *proxyRepo) Stat(rev string) (*RevInfo, error) { } info := new(RevInfo) if err := json.Unmarshal(data, info); err != nil { - return nil, p.versionError(rev, err) + return nil, p.versionError(rev, fmt.Errorf("invalid response from proxy %q: %w", p.redactedURL, err)) } if info.Version != rev && rev == module.CanonicalVersion(rev) && module.Check(p.path, rev) == nil { // If we request a correct, appropriate version for the module path, the @@ -427,14 +370,14 @@ func (p *proxyRepo) Stat(rev string) (*RevInfo, error) { func (p *proxyRepo) Latest() (*RevInfo, error) { data, err := p.getBytes("@latest") if err != nil { - if !errors.Is(err, os.ErrNotExist) { + if !errors.Is(err, fs.ErrNotExist) { return nil, p.versionError("", err) } return p.latest() } info := new(RevInfo) if err := json.Unmarshal(data, info); err != nil { - return nil, p.versionError("", err) + return nil, p.versionError("", fmt.Errorf("invalid response from proxy %q: %w", p.redactedURL, err)) } return info, nil } diff --git a/cmd/go/_internal_/modfetch/pseudo.go b/cmd/go/_internal_/modfetch/pseudo.go index f402cbe..8d6e729 100644 --- a/cmd/go/_internal_/modfetch/pseudo.go +++ b/cmd/go/_internal_/modfetch/pseudo.go @@ -76,6 +76,12 @@ func PseudoVersion(major, older string, t time.Time, rev string) string { return v + incDecimal(patch) + "-0." + segment + build } +// ZeroPseudoVersion returns a pseudo-version with a zero timestamp and +// revision, which may be used as a placeholder. +func ZeroPseudoVersion(major string) string { + return PseudoVersion(major, "", time.Time{}, "000000000000") +} + // incDecimal returns the decimal string incremented by 1. func incDecimal(decimal string) string { // Scan right to left turning 9s to 0s until you find a digit to increment. @@ -120,6 +126,12 @@ func IsPseudoVersion(v string) bool { return strings.Count(v, "-") >= 2 && semver.IsValid(v) && pseudoVersionRE.MatchString(v) } +// IsZeroPseudoVersion returns whether v is a pseudo-version with a zero base, +// timestamp, and revision, as returned by ZeroPseudoVersion. +func IsZeroPseudoVersion(v string) bool { + return v == ZeroPseudoVersion(semver.Major(v)) +} + // PseudoVersionTime returns the time stamp of the pseudo-version v. // It returns an error if v is not a pseudo-version or if the time stamp // embedded in the pseudo-version is not a valid time. diff --git a/cmd/go/_internal_/modfetch/repo.go b/cmd/go/_internal_/modfetch/repo.go index b94916a..5842abe 100644 --- a/cmd/go/_internal_/modfetch/repo.go +++ b/cmd/go/_internal_/modfetch/repo.go @@ -7,18 +7,19 @@ package modfetch import ( "fmt" "io" + "io/fs" "os" "sort" "strconv" "time" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/get" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch/codehost" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/par" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/str" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/vcs" web "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/web" + "golang.org/x/mod/module" "golang.org/x/mod/semver" ) @@ -32,8 +33,17 @@ type Repo interface { // Versions lists all known versions with the given prefix. // Pseudo-versions are not included. + // // Versions should be returned sorted in semver order // (implementations can use SortVersions). + // + // Versions returns a non-nil error only if there was a problem + // fetching the list of versions: it may return an empty list + // along with a nil error if the list of matching versions + // is known to be empty. + // + // If the underlying repository does not exist, + // Versions returns an error matching errors.Is(_, os.NotExist). Versions(prefix string) ([]string, error) // Stat returns information about the revision rev. @@ -188,27 +198,26 @@ type lookupCacheKey struct { // // A successful return does not guarantee that the module // has any defined versions. -func Lookup(proxy, path string) (Repo, error) { +func Lookup(proxy, path string) Repo { if traceRepo { defer logCall("Lookup(%q, %q)", proxy, path)() } type cached struct { - r Repo - err error + r Repo } c := lookupCache.Do(lookupCacheKey{proxy, path}, func() interface{} { - r, err := lookup(proxy, path) - if err == nil { - if traceRepo { + r := newCachingRepo(path, func() (Repo, error) { + r, err := lookup(proxy, path) + if err == nil && traceRepo { r = newLoggingRepo(r) } - r = newCachingRepo(r) - } - return cached{r, err} + return r, err + }) + return cached{r} }).(cached) - return c.r, c.err + return c.r } // lookup returns the module with the given module path. @@ -217,7 +226,7 @@ func lookup(proxy, path string) (r Repo, err error) { return nil, errLookupDisabled } - if str.GlobsMatchPath(cfg.GONOPROXY, path) { + if module.MatchPrefixPatterns(cfg.GONOPROXY, path) { switch proxy { case "noproxy", "direct": return lookupDirect(path) @@ -228,7 +237,7 @@ func lookup(proxy, path string) (r Repo, err error) { switch proxy { case "off": - return nil, errProxyOff + return errRepo{path, errProxyOff}, nil case "direct": return lookupDirect(path) case "noproxy": @@ -261,13 +270,13 @@ func lookupDirect(path string) (Repo, error) { if allowInsecure(path) { security = web.Insecure } - rr, err := get.RepoRootForImportPath(path, get.PreferMod, security) + rr, err := vcs.RepoRootForImportPath(path, vcs.PreferMod, security) if err != nil { // We don't know where to find code for a module with this path. return nil, notExistError{err: err} } - if rr.VCS == "mod" { + if rr.VCS.Name == "mod" { // Fetch module from proxy with base URL rr.Repo. return newProxyRepo(rr.Repo, path) } @@ -279,8 +288,8 @@ func lookupDirect(path string) (Repo, error) { return newCodeRepo(code, rr.Root, path) } -func lookupCodeRepo(rr *get.RepoRoot) (codehost.Repo, error) { - code, err := codehost.NewRepo(rr.VCS, rr.Repo) +func lookupCodeRepo(rr *vcs.RepoRoot) (codehost.Repo, error) { + code, err := codehost.NewRepo(rr.VCS.Cmd, rr.Repo) if err != nil { if _, ok := err.(*codehost.VCSError); ok { return nil, err @@ -306,7 +315,7 @@ func ImportRepoRev(path, rev string) (Repo, *RevInfo, error) { if allowInsecure(path) { security = web.Insecure } - rr, err := get.RepoRootForImportPath(path, get.IgnoreMod, security) + rr, err := vcs.RepoRootForImportPath(path, vcs.IgnoreMod, security) if err != nil { return nil, nil, err } @@ -407,7 +416,24 @@ func (l *loggingRepo) Zip(dst io.Writer, version string) error { return l.r.Zip(dst, version) } -// A notExistError is like os.ErrNotExist, but with a custom message +// errRepo is a Repo that returns the same error for all operations. +// +// It is useful in conjunction with caching, since cache hits will not attempt +// the prohibited operations. +type errRepo struct { + modulePath string + err error +} + +func (r errRepo) ModulePath() string { return r.modulePath } + +func (r errRepo) Versions(prefix string) (tags []string, err error) { return nil, r.err } +func (r errRepo) Stat(rev string) (*RevInfo, error) { return nil, r.err } +func (r errRepo) Latest() (*RevInfo, error) { return nil, r.err } +func (r errRepo) GoMod(version string) ([]byte, error) { return nil, r.err } +func (r errRepo) Zip(dst io.Writer, version string) error { return r.err } + +// A notExistError is like fs.ErrNotExist, but with a custom message type notExistError struct { err error } @@ -421,7 +447,7 @@ func (e notExistError) Error() string { } func (notExistError) Is(target error) bool { - return target == os.ErrNotExist + return target == fs.ErrNotExist } func (e notExistError) Unwrap() error { diff --git a/cmd/go/_internal_/modfetch/sumdb.go b/cmd/go/_internal_/modfetch/sumdb.go index 37e3087..f2d4f99 100644 --- a/cmd/go/_internal_/modfetch/sumdb.go +++ b/cmd/go/_internal_/modfetch/sumdb.go @@ -12,7 +12,8 @@ import ( "bytes" "errors" "fmt" - "io/ioutil" + "io" + "io/fs" "net/url" "os" "path/filepath" @@ -22,9 +23,7 @@ import ( "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/get" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/lockedfile" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/str" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/web" "golang.org/x/mod/module" @@ -34,7 +33,7 @@ import ( // useSumDB reports whether to use the Go checksum database for the given module. func useSumDB(mod module.Version) bool { - return cfg.GOSUMDB != "off" && !get.Insecure && !str.GlobsMatchPath(cfg.GONOSUMDB, mod.Path) + return cfg.GOSUMDB != "off" && !cfg.Insecure && !module.MatchPrefixPatterns(cfg.GONOSUMDB, mod.Path) } // lookupSumDB returns the Go checksum database's go.sum lines for the given module, @@ -184,7 +183,7 @@ func (c *dbClient) initBase() { return nil } }) - if errors.Is(err, os.ErrNotExist) { + if errors.Is(err, fs.ErrNotExist) { // No proxies, or all proxies failed (with 404, 410, or were were allowed // to fall back), or we reached an explicit "direct" or "off". c.base = c.direct @@ -205,7 +204,7 @@ func (c *dbClient) ReadConfig(file string) (data []byte, err error) { } targ := filepath.Join(cfg.SumdbDir, file) data, err = lockedfile.Read(targ) - if errors.Is(err, os.ErrNotExist) { + if errors.Is(err, fs.ErrNotExist) { // Treat non-existent as empty, to bootstrap the "latest" file // the first time we connect to a given database. return []byte{}, nil @@ -229,7 +228,7 @@ func (*dbClient) WriteConfig(file string, old, new []byte) error { return err } defer f.Close() - data, err := ioutil.ReadAll(f) + data, err := io.ReadAll(f) if err != nil { return err } @@ -259,7 +258,7 @@ func (*dbClient) ReadCache(file string) ([]byte, error) { // during which the empty file can be locked for reading. // Treat observing an empty file as file not found. if err == nil && len(data) == 0 { - err = &os.PathError{Op: "read", Path: targ, Err: os.ErrNotExist} + err = &fs.PathError{Op: "read", Path: targ, Err: fs.ErrNotExist} } return data, err } diff --git a/cmd/go/_internal_/modinfo/info.go b/cmd/go/_internal_/modinfo/info.go index 5dcbbcc..805b722 100644 --- a/cmd/go/_internal_/modinfo/info.go +++ b/cmd/go/_internal_/modinfo/info.go @@ -21,6 +21,7 @@ type ModulePublic struct { Dir string `json:",omitempty"` // directory holding local copy of files, if any GoMod string `json:",omitempty"` // path to go.mod file describing module, if any GoVersion string `json:",omitempty"` // go version used in module + Retracted []string `json:",omitempty"` // retraction information, if any (with -retracted or -u) Error *ModuleError `json:",omitempty"` // error loading module } @@ -30,18 +31,26 @@ type ModuleError struct { func (m *ModulePublic) String() string { s := m.Path + versionString := func(mm *ModulePublic) string { + v := mm.Version + if len(mm.Retracted) == 0 { + return v + } + return v + " (retracted)" + } + if m.Version != "" { - s += " " + m.Version + s += " " + versionString(m) if m.Update != nil { - s += " [" + m.Update.Version + "]" + s += " [" + versionString(m.Update) + "]" } } if m.Replace != nil { s += " => " + m.Replace.Path if m.Replace.Version != "" { - s += " " + m.Replace.Version + s += " " + versionString(m.Replace) if m.Replace.Update != nil { - s += " [" + m.Replace.Update.Version + "]" + s += " [" + versionString(m.Replace.Update) + "]" } } } diff --git a/cmd/go/_internal_/modload/build.go b/cmd/go/_internal_/modload/build.go index bef3860..cfc2ef6 100644 --- a/cmd/go/_internal_/modload/build.go +++ b/cmd/go/_internal_/modload/build.go @@ -6,12 +6,13 @@ package modload import ( "bytes" + "context" "encoding/hex" + "errors" "fmt" "github.com/dependabot/gomodules-extracted/_internal_/goroot" "os" "path/filepath" - "runtime/debug" "strings" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" @@ -48,7 +49,7 @@ func findStandardImportPath(path string) string { // PackageModuleInfo returns information about the module that provides // a given package. If modules are not enabled or if the package is in the // standard library or if the package was not successfully loaded with -// ImportPaths or a similar loading function, nil is returned. +// LoadPackages or ImportFromFiles, nil is returned. func PackageModuleInfo(pkgpath string) *modinfo.ModulePublic { if isStandardImportPath(pkgpath) || !Enabled() { return nil @@ -57,21 +58,27 @@ func PackageModuleInfo(pkgpath string) *modinfo.ModulePublic { if !ok { return nil } - return moduleInfo(m, true) + fromBuildList := true + listRetracted := false + return moduleInfo(context.TODO(), m, fromBuildList, listRetracted) } -func ModuleInfo(path string) *modinfo.ModulePublic { +func ModuleInfo(ctx context.Context, path string) *modinfo.ModulePublic { if !Enabled() { return nil } + listRetracted := false if i := strings.Index(path, "@"); i >= 0 { - return moduleInfo(module.Version{Path: path[:i], Version: path[i+1:]}, false) + m := module.Version{Path: path[:i], Version: path[i+1:]} + fromBuildList := false + return moduleInfo(ctx, m, fromBuildList, listRetracted) } - for _, m := range BuildList() { + for _, m := range buildList { if m.Path == path { - return moduleInfo(m, true) + fromBuildList := true + return moduleInfo(ctx, m, fromBuildList, listRetracted) } } @@ -84,12 +91,12 @@ func ModuleInfo(path string) *modinfo.ModulePublic { } // addUpdate fills in m.Update if an updated version is available. -func addUpdate(m *modinfo.ModulePublic) { +func addUpdate(ctx context.Context, m *modinfo.ModulePublic) { if m.Version == "" { return } - if info, err := Query(m.Path, "upgrade", m.Version, Allowed); err == nil && semver.Compare(info.Version, m.Version) > 0 { + if info, err := Query(ctx, m.Path, "upgrade", m.Version, CheckAllowed); err == nil && semver.Compare(info.Version, m.Version) > 0 { m.Update = &modinfo.ModulePublic{ Path: m.Path, Version: info.Version, @@ -99,11 +106,37 @@ func addUpdate(m *modinfo.ModulePublic) { } // addVersions fills in m.Versions with the list of known versions. -func addVersions(m *modinfo.ModulePublic) { - m.Versions, _ = versions(m.Path) +// Excluded versions will be omitted. If listRetracted is false, retracted +// versions will also be omitted. +func addVersions(ctx context.Context, m *modinfo.ModulePublic, listRetracted bool) { + allowed := CheckAllowed + if listRetracted { + allowed = CheckExclusions + } + m.Versions, _ = versions(ctx, m.Path, allowed) +} + +// addRetraction fills in m.Retracted if the module was retracted by its author. +// m.Error is set if there's an error loading retraction information. +func addRetraction(ctx context.Context, m *modinfo.ModulePublic) { + if m.Version == "" { + return + } + + err := CheckRetractions(ctx, module.Version{Path: m.Path, Version: m.Version}) + var rerr *ModuleRetractedError + if errors.As(err, &rerr) { + if len(rerr.Rationale) == 0 { + m.Retracted = []string{"retracted by module author"} + } else { + m.Retracted = rerr.Rationale + } + } else if err != nil && m.Error == nil { + m.Error = &modinfo.ModuleError{Err: err.Error()} + } } -func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { +func moduleInfo(ctx context.Context, m module.Version, fromBuildList, listRetracted bool) *modinfo.ModulePublic { if m == Target { info := &modinfo.ModulePublic{ Path: m.Path, @@ -125,21 +158,22 @@ func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { Version: m.Version, Indirect: fromBuildList && loaded != nil && !loaded.direct[m.Path], } - if loaded != nil { - info.GoVersion = loaded.goVersion[m.Path] + if v, ok := rawGoVersion.Load(m); ok { + info.GoVersion = v.(string) } // completeFromModCache fills in the extra fields in m using the module cache. completeFromModCache := func(m *modinfo.ModulePublic) { + mod := module.Version{Path: m.Path, Version: m.Version} + if m.Version != "" { - if q, err := Query(m.Path, m.Version, "", nil); err != nil { + if q, err := Query(ctx, m.Path, m.Version, "", nil); err != nil { m.Error = &modinfo.ModuleError{Err: err.Error()} } else { m.Version = q.Version m.Time = &q.Time } - mod := module.Version{Path: m.Path, Version: m.Version} gomod, err := modfetch.CachePath(mod, "mod") if err == nil { if info, err := os.Stat(gomod); err == nil && info.Mode().IsRegular() { @@ -150,10 +184,22 @@ func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { if err == nil { m.Dir = dir } + + if listRetracted { + addRetraction(ctx, m) + } + } + + if m.GoVersion == "" { + if summary, err := rawGoModSummary(mod); err == nil && summary.goVersionV != "" { + m.GoVersion = summary.goVersionV[1:] + } } } if !fromBuildList { + // If this was an explicitly-versioned argument to 'go mod download' or + // 'go list -m', report the actual requested version, not its replacement. completeFromModCache(info) // Will set m.Error in vendor mode. return info } @@ -179,7 +225,9 @@ func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { info.Replace = &modinfo.ModulePublic{ Path: r.Path, Version: r.Version, - GoVersion: info.GoVersion, + } + if v, ok := rawGoVersion.Load(m); ok { + info.Replace.GoVersion = v.(string) } if r.Version == "" { if filepath.IsAbs(r.Path) { @@ -193,14 +241,15 @@ func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { completeFromModCache(info.Replace) info.Dir = info.Replace.Dir info.GoMod = info.Replace.GoMod + info.Retracted = info.Replace.Retracted } + info.GoVersion = info.Replace.GoVersion return info } // PackageBuildInfo returns a string containing module version information // for modules providing packages named by path and deps. path and deps must -// name packages that were resolved successfully with ImportPaths or one of -// the Load functions. +// name packages that were resolved successfully with LoadPackages. func PackageBuildInfo(path string, deps []string) string { if isStandardImportPath(path) || !Enabled() { return "" @@ -262,17 +311,13 @@ func mustFindModule(target, path string) module.Version { return Target } - if printStackInDie { - debug.PrintStack() - } base.Fatalf("build %v: cannot find module for path %v", target, path) panic("unreachable") } // findModule searches for the module that contains the package at path. -// If the package was loaded with ImportPaths or one of the other loading -// functions, its containing module and true are returned. Otherwise, -// module.Version{} and false are returend. +// If the package was loaded, its containing module and true are returned. +// Otherwise, module.Version{} and false are returend. func findModule(path string) (module.Version, bool) { if pkg, ok := loaded.pkgCache.Get(path).(*loadPkg); ok { return pkg.mod, pkg.mod != module.Version{} @@ -304,13 +349,13 @@ func ModInfoProg(info string, isgccgo bool) []byte { import _ "unsafe" //go:linkname __debug_modinfo__ runtime.modinfo var __debug_modinfo__ = %q - `, string(infoStart)+info+string(infoEnd))) +`, string(infoStart)+info+string(infoEnd))) } else { return []byte(fmt.Sprintf(`package main import _ "unsafe" //go:linkname __set_debug_modinfo__ runtime.setmodinfo func __set_debug_modinfo__(string) func init() { __set_debug_modinfo__(%q) } - `, string(infoStart)+info+string(infoEnd))) +`, string(infoStart)+info+string(infoEnd))) } } diff --git a/cmd/go/_internal_/modload/buildlist.go b/cmd/go/_internal_/modload/buildlist.go new file mode 100644 index 0000000..b38e7bd --- /dev/null +++ b/cmd/go/_internal_/modload/buildlist.go @@ -0,0 +1,278 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/imports" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/mvs" + "context" + "fmt" + "os" + "strings" + + "golang.org/x/mod/module" +) + +// buildList is the list of modules to use for building packages. +// It is initialized by calling LoadPackages or ImportFromFiles, +// each of which uses loaded.load. +// +// Ideally, exactly ONE of those functions would be called, +// and exactly once. Most of the time, that's true. +// During "go get" it may not be. TODO(rsc): Figure out if +// that restriction can be established, or else document why not. +// +var buildList []module.Version + +// additionalExplicitRequirements is a list of modules paths for which +// WriteGoMod should record explicit requirements, even if they would be +// selected without those requirements. Each path must also appear in buildList. +var additionalExplicitRequirements []string + +// capVersionSlice returns s with its cap reduced to its length. +func capVersionSlice(s []module.Version) []module.Version { + return s[:len(s):len(s)] +} + +// LoadAllModules loads and returns the list of modules matching the "all" +// module pattern, starting with the Target module and in a deterministic +// (stable) order, without loading any packages. +// +// Modules are loaded automatically (and lazily) in LoadPackages: +// LoadAllModules need only be called if LoadPackages is not, +// typically in commands that care about modules but no particular package. +// +// The caller must not modify the returned list, but may append to it. +func LoadAllModules(ctx context.Context) []module.Version { + LoadModFile(ctx) + ReloadBuildList() + WriteGoMod() + return capVersionSlice(buildList) +} + +// Selected returns the selected version of the module with the given path, or +// the empty string if the given module has no selected version +// (either because it is not required or because it is the Target module). +func Selected(path string) (version string) { + if path == Target.Path { + return "" + } + for _, m := range buildList { + if m.Path == path { + return m.Version + } + } + return "" +} + +// EditBuildList edits the global build list by first adding every module in add +// to the existing build list, then adjusting versions (and adding or removing +// requirements as needed) until every module in mustSelect is selected at the +// given version. +// +// (Note that the newly-added modules might not be selected in the resulting +// build list: they could be lower than existing requirements or conflict with +// versions in mustSelect.) +// +// If the versions listed in mustSelect are mutually incompatible (due to one of +// the listed modules requiring a higher version of another), EditBuildList +// returns a *ConstraintError and leaves the build list in its previous state. +func EditBuildList(ctx context.Context, add, mustSelect []module.Version) error { + var upgraded = capVersionSlice(buildList) + if len(add) > 0 { + // First, upgrade the build list with any additions. + // In theory we could just append the additions to the build list and let + // mvs.Downgrade take care of resolving the upgrades too, but the + // diagnostics from Upgrade are currently much better in case of errors. + var err error + upgraded, err = mvs.Upgrade(Target, &mvsReqs{buildList: upgraded}, add...) + if err != nil { + return err + } + } + + downgraded, err := mvs.Downgrade(Target, &mvsReqs{buildList: append(upgraded, mustSelect...)}, mustSelect...) + if err != nil { + return err + } + + final, err := mvs.Upgrade(Target, &mvsReqs{buildList: downgraded}, mustSelect...) + if err != nil { + return err + } + + selected := make(map[string]module.Version, len(final)) + for _, m := range final { + selected[m.Path] = m + } + inconsistent := false + for _, m := range mustSelect { + s, ok := selected[m.Path] + if !ok { + if m.Version != "none" { + panic(fmt.Sprintf("internal error: mvs.BuildList lost %v", m)) + } + continue + } + if s.Version != m.Version { + inconsistent = true + break + } + } + + if !inconsistent { + buildList = final + additionalExplicitRequirements = make([]string, 0, len(mustSelect)) + for _, m := range mustSelect { + if m.Version != "none" { + additionalExplicitRequirements = append(additionalExplicitRequirements, m.Path) + } + } + return nil + } + + // We overshot one or more of the modules in mustSelected, which means that + // Downgrade removed something in mustSelect because it conflicted with + // something else in mustSelect. + // + // Walk the requirement graph to find the conflict. + // + // TODO(bcmills): Ideally, mvs.Downgrade (or a replacement for it) would do + // this directly. + + reqs := &mvsReqs{buildList: final} + reason := map[module.Version]module.Version{} + for _, m := range mustSelect { + reason[m] = m + } + queue := mustSelect[:len(mustSelect):len(mustSelect)] + for len(queue) > 0 { + var m module.Version + m, queue = queue[0], queue[1:] + required, err := reqs.Required(m) + if err != nil { + return err + } + for _, r := range required { + if _, ok := reason[r]; !ok { + reason[r] = reason[m] + queue = append(queue, r) + } + } + } + + var conflicts []Conflict + for _, m := range mustSelect { + s, ok := selected[m.Path] + if !ok { + if m.Version != "none" { + panic(fmt.Sprintf("internal error: mvs.BuildList lost %v", m)) + } + continue + } + if s.Version != m.Version { + conflicts = append(conflicts, Conflict{ + Source: reason[s], + Dep: s, + Constraint: m, + }) + } + } + + return &ConstraintError{ + Conflicts: conflicts, + } +} + +// A ConstraintError describes inconsistent constraints in EditBuildList +type ConstraintError struct { + // Conflict lists the source of the conflict for each version in mustSelect + // that could not be selected due to the requirements of some other version in + // mustSelect. + Conflicts []Conflict +} + +func (e *ConstraintError) Error() string { + b := new(strings.Builder) + b.WriteString("version constraints conflict:") + for _, c := range e.Conflicts { + fmt.Fprintf(b, "\n\t%v requires %v, but %v is requested", c.Source, c.Dep, c.Constraint) + } + return b.String() +} + +// A Conflict documents that Source requires Dep, which conflicts with Constraint. +// (That is, Dep has the same module path as Constraint but a higher version.) +type Conflict struct { + Source module.Version + Dep module.Version + Constraint module.Version +} + +// ReloadBuildList resets the state of loaded packages, then loads and returns +// the build list set by EditBuildList. +func ReloadBuildList() []module.Version { + loaded = loadFromRoots(loaderParams{ + PackageOpts: PackageOpts{ + Tags: imports.Tags(), + }, + listRoots: func() []string { return nil }, + allClosesOverTests: index.allPatternClosesOverTests(), // but doesn't matter because the root list is empty. + }) + return capVersionSlice(buildList) +} + +// TidyBuildList trims the build list to the minimal requirements needed to +// retain the same versions of all packages from the preceding call to +// LoadPackages. +func TidyBuildList() { + used := map[module.Version]bool{Target: true} + for _, pkg := range loaded.pkgs { + used[pkg.mod] = true + } + + keep := []module.Version{Target} + var direct []string + for _, m := range buildList[1:] { + if used[m] { + keep = append(keep, m) + if loaded.direct[m.Path] { + direct = append(direct, m.Path) + } + } else if cfg.BuildV { + if _, ok := index.require[m]; ok { + fmt.Fprintf(os.Stderr, "unused %s\n", m.Path) + } + } + } + + min, err := mvs.Req(Target, direct, &mvsReqs{buildList: keep}) + if err != nil { + base.Fatalf("go: %v", err) + } + buildList = append([]module.Version{Target}, min...) +} + +// checkMultiplePaths verifies that a given module path is used as itself +// or as a replacement for another module, but not both at the same time. +// +// (See https://golang.org/issue/26607 and https://golang.org/issue/34650.) +func checkMultiplePaths() { + firstPath := make(map[module.Version]string, len(buildList)) + for _, mod := range buildList { + src := mod + if rep := Replacement(mod); rep.Path != "" { + src = rep + } + if prev, ok := firstPath[src]; !ok { + firstPath[src] = mod.Path + } else if prev != mod.Path { + base.Errorf("go: %s@%s used for two different module paths (%s and %s)", src.Path, src.Version, prev, mod.Path) + } + } + base.ExitIfErrors() +} diff --git a/cmd/go/_internal_/modload/help.go b/cmd/go/_internal_/modload/help.go index 98a6113..e690561 100644 --- a/cmd/go/_internal_/modload/help.go +++ b/cmd/go/_internal_/modload/help.go @@ -6,410 +6,31 @@ package modload import "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" -// TODO(rsc): The "module code layout" section needs to be written. - var HelpModules = &base.Command{ UsageLine: "modules", Short: "modules, module versions, and more", Long: ` -A module is a collection of related Go packages. -Modules are the unit of source code interchange and versioning. -The go command has direct support for working with modules, -including recording and resolving dependencies on other modules. -Modules replace the old GOPATH-based approach to specifying -which source files are used in a given build. - -Module support - -The go command includes support for Go modules. Module-aware mode is active -by default whenever a go.mod file is found in the current directory or in -any parent directory. - -The quickest way to take advantage of module support is to check out your -repository, create a go.mod file (described in the next section) there, and run -go commands from within that file tree. - -For more fine-grained control, the go command continues to respect -a temporary environment variable, GO111MODULE, which can be set to one -of three string values: off, on, or auto (the default). -If GO111MODULE=on, then the go command requires the use of modules, -never consulting GOPATH. We refer to this as the command -being module-aware or running in "module-aware mode". -If GO111MODULE=off, then the go command never uses -module support. Instead it looks in vendor directories and GOPATH -to find dependencies; we now refer to this as "GOPATH mode." -If GO111MODULE=auto or is unset, then the go command enables or disables -module support based on the current directory. -Module support is enabled only when the current directory contains a -go.mod file or is below a directory containing a go.mod file. - -In module-aware mode, GOPATH no longer defines the meaning of imports -during a build, but it still stores downloaded dependencies (in GOPATH/pkg/mod) -and installed commands (in GOPATH/bin, unless GOBIN is set). - -Defining a module - -A module is defined by a tree of Go source files with a go.mod file -in the tree's root directory. The directory containing the go.mod file -is called the module root. Typically the module root will also correspond -to a source code repository root (but in general it need not). -The module is the set of all Go packages in the module root and its -subdirectories, but excluding subtrees with their own go.mod files. - -The "module path" is the import path prefix corresponding to the module root. -The go.mod file defines the module path and lists the specific versions -of other modules that should be used when resolving imports during a build, -by giving their module paths and versions. - -For example, this go.mod declares that the directory containing it is the root -of the module with path example.com/m, and it also declares that the module -depends on specific versions of golang.org/x/text and gopkg.in/yaml.v2: - - module example.com/m - - require ( - golang.org/x/text v0.3.0 - gopkg.in/yaml.v2 v2.1.0 - ) - -The go.mod file can also specify replacements and excluded versions -that only apply when building the module directly; they are ignored -when the module is incorporated into a larger build. -For more about the go.mod file, see 'go help go.mod'. - -To start a new module, simply create a go.mod file in the root of the -module's directory tree, containing only a module statement. -The 'go mod init' command can be used to do this: - - go mod init example.com/m - -In a project already using an existing dependency management tool like -godep, glide, or dep, 'go mod init' will also add require statements -matching the existing configuration. - -Once the go.mod file exists, no additional steps are required: -go commands like 'go build', 'go test', or even 'go list' will automatically -add new dependencies as needed to satisfy imports. - -The main module and the build list - -The "main module" is the module containing the directory where the go command -is run. The go command finds the module root by looking for a go.mod in the -current directory, or else the current directory's parent directory, -or else the parent's parent directory, and so on. - -The main module's go.mod file defines the precise set of packages available -for use by the go command, through require, replace, and exclude statements. -Dependency modules, found by following require statements, also contribute -to the definition of that set of packages, but only through their go.mod -files' require statements: any replace and exclude statements in dependency -modules are ignored. The replace and exclude statements therefore allow the -main module complete control over its own build, without also being subject -to complete control by dependencies. - -The set of modules providing packages to builds is called the "build list". -The build list initially contains only the main module. Then the go command -adds to the list the exact module versions required by modules already -on the list, recursively, until there is nothing left to add to the list. -If multiple versions of a particular module are added to the list, -then at the end only the latest version (according to semantic version -ordering) is kept for use in the build. - -The 'go list' command provides information about the main module -and the build list. For example: - - go list -m # print path of main module - go list -m -f={{.Dir}} # print root directory of main module - go list -m all # print build list - -Maintaining module requirements - -The go.mod file is meant to be readable and editable by both -programmers and tools. The go command itself automatically updates the go.mod file -to maintain a standard formatting and the accuracy of require statements. - -Any go command that finds an unfamiliar import will look up the module -containing that import and add the latest version of that module -to go.mod automatically. In most cases, therefore, it suffices to -add an import to source code and run 'go build', 'go test', or even 'go list': -as part of analyzing the package, the go command will discover -and resolve the import and update the go.mod file. - -Any go command can determine that a module requirement is -missing and must be added, even when considering only a single -package from the module. On the other hand, determining that a module requirement -is no longer necessary and can be deleted requires a full view of -all packages in the module, across all possible build configurations -(architectures, operating systems, build tags, and so on). -The 'go mod tidy' command builds that view and then -adds any missing module requirements and removes unnecessary ones. - -As part of maintaining the require statements in go.mod, the go command -tracks which ones provide packages imported directly by the current module -and which ones provide packages only used indirectly by other module -dependencies. Requirements needed only for indirect uses are marked with a -"// indirect" comment in the go.mod file. Indirect requirements are -automatically removed from the go.mod file once they are implied by other -direct requirements. Indirect requirements only arise when using modules -that fail to state some of their own dependencies or when explicitly -upgrading a module's dependencies ahead of its own stated requirements. - -Because of this automatic maintenance, the information in go.mod is an -up-to-date, readable description of the build. - -The 'go get' command updates go.mod to change the module versions used in a -build. An upgrade of one module may imply upgrading others, and similarly a -downgrade of one module may imply downgrading others. The 'go get' command -makes these implied changes as well. If go.mod is edited directly, commands -like 'go build' or 'go list' will assume that an upgrade is intended and -automatically make any implied upgrades and update go.mod to reflect them. - -The 'go mod' command provides other functionality for use in maintaining -and understanding modules and go.mod files. See 'go help mod'. - -The -mod build flag provides additional control over updating and use of go.mod. - -If invoked with -mod=readonly, the go command is disallowed from the implicit -automatic updating of go.mod described above. Instead, it fails when any changes -to go.mod are needed. This setting is most useful to check that go.mod does -not need updates, such as in a continuous integration and testing system. -The "go get" command remains permitted to update go.mod even with -mod=readonly, -and the "go mod" commands do not take the -mod flag (or any other build flags). - -If invoked with -mod=vendor, the go command loads packages from the main -module's vendor directory instead of downloading modules to and loading packages -from the module cache. The go command assumes the vendor directory holds -correct copies of dependencies, and it does not compute the set of required -module versions from go.mod files. However, the go command does check that -vendor/modules.txt (generated by 'go mod vendor') contains metadata consistent -with go.mod. - -If invoked with -mod=mod, the go command loads modules from the module cache -even if there is a vendor directory present. - -If the go command is not invoked with a -mod flag and the vendor directory -is present and the "go" version in go.mod is 1.14 or higher, the go command -will act as if it were invoked with -mod=vendor. - -Pseudo-versions - -The go.mod file and the go command more generally use semantic versions as -the standard form for describing module versions, so that versions can be -compared to determine which should be considered earlier or later than another. -A module version like v1.2.3 is introduced by tagging a revision in the -underlying source repository. Untagged revisions can be referred to -using a "pseudo-version" like v0.0.0-yyyymmddhhmmss-abcdefabcdef, -where the time is the commit time in UTC and the final suffix is the prefix -of the commit hash. The time portion ensures that two pseudo-versions can -be compared to determine which happened later, the commit hash identifes -the underlying commit, and the prefix (v0.0.0- in this example) is derived from -the most recent tagged version in the commit graph before this commit. - -There are three pseudo-version forms: - -vX.0.0-yyyymmddhhmmss-abcdefabcdef is used when there is no earlier -versioned commit with an appropriate major version before the target commit. -(This was originally the only form, so some older go.mod files use this form -even for commits that do follow tags.) - -vX.Y.Z-pre.0.yyyymmddhhmmss-abcdefabcdef is used when the most -recent versioned commit before the target commit is vX.Y.Z-pre. - -vX.Y.(Z+1)-0.yyyymmddhhmmss-abcdefabcdef is used when the most -recent versioned commit before the target commit is vX.Y.Z. - -Pseudo-versions never need to be typed by hand: the go command will accept -the plain commit hash and translate it into a pseudo-version (or a tagged -version if available) automatically. This conversion is an example of a -module query. +Modules are how Go manages dependencies. -Module queries +A module is a collection of packages that are released, versioned, and +distributed together. Modules may be downloaded directly from version control +repositories or from module proxy servers. -The go command accepts a "module query" in place of a module version -both on the command line and in the main module's go.mod file. -(After evaluating a query found in the main module's go.mod file, -the go command updates the file to replace the query with its result.) +For a series of tutorials on modules, see +https://golang.org/doc/tutorial/create-module. -A fully-specified semantic version, such as "v1.2.3", -evaluates to that specific version. +For a detailed reference on modules, see https://golang.org/ref/mod. -A semantic version prefix, such as "v1" or "v1.2", -evaluates to the latest available tagged version with that prefix. +By default, the go command may download modules from https://proxy.golang.org. +It may authenticate modules using the checksum database at +https://sum.golang.org. Both services are operated by the Go team at Google. +The privacy policies for these services are available at +https://proxy.golang.org/privacy and https://sum.golang.org/privacy, +respectively. -A semantic version comparison, such as "=v1.5.6", -evaluates to the available tagged version nearest to the comparison target -(the latest version for < and <=, the earliest version for > and >=). - -The string "latest" matches the latest available tagged version, -or else the underlying source repository's latest untagged revision. - -The string "upgrade" is like "latest", but if the module is -currently required at a later version than the version "latest" -would select (for example, a newer pre-release version), "upgrade" -will select the later version instead. - -The string "patch" matches the latest available tagged version -of a module with the same major and minor version numbers as the -currently required version. If no version is currently required, -"patch" is equivalent to "latest". - -A revision identifier for the underlying source repository, such as -a commit hash prefix, revision tag, or branch name, selects that -specific code revision. If the revision is also tagged with a -semantic version, the query evaluates to that semantic version. -Otherwise the query evaluates to a pseudo-version for the commit. -Note that branches and tags with names that are matched by other -query syntax cannot be selected this way. For example, the query -"v2" means the latest version starting with "v2", not the branch -named "v2". - -All queries prefer release versions to pre-release versions. -For example, " good/thing v1.4.5 - -The verbs are - module, to define the module path; - go, to set the expected language version; - require, to require a particular module at a given version or later; - exclude, to exclude a particular module version from use; and - replace, to replace a module version with a different module version. -Exclude and replace apply only in the main module's go.mod and are ignored -in dependencies. See https://research.swtch.com/vgo-mvs for details. - -The leading verb can be factored out of adjacent lines to create a block, -like in Go imports: - - require ( - new/thing v2.3.4 - old/thing v1.2.3 - ) - -The go.mod file is designed both to be edited directly and to be -easily updated by tools. The 'go mod edit' command can be used to -parse and edit the go.mod file from programs and tools. -See 'go help mod edit'. - -The go command automatically updates go.mod each time it uses the -module graph, to make sure go.mod always accurately reflects reality -and is properly formatted. For example, consider this go.mod file: - - module M - - require ( - A v1 - B v1.0.0 - C v1.0.0 - D v1.2.3 - E dev - ) - - exclude D v1.2.3 - -The update rewrites non-canonical version identifiers to semver form, -so A's v1 becomes v1.0.0 and E's dev becomes the pseudo-version for the -latest commit on the dev branch, perhaps v0.0.0-20180523231146-b3f5c0f6e5f1. - -The update modifies requirements to respect exclusions, so the -requirement on the excluded D v1.2.3 is updated to use the next -available version of D, perhaps D v1.2.4 or D v1.3.0. +The go.mod file format is described in detail at +https://golang.org/ref/mod#go-mod-file. -The update removes redundant or misleading requirements. -For example, if A v1.0.0 itself requires B v1.2.0 and C v1.0.0, -then go.mod's requirement of B v1.0.0 is misleading (superseded by -A's need for v1.2.0), and its requirement of C v1.0.0 is redundant -(implied by A's need for the same version), so both will be removed. -If module M contains packages that directly import packages from B or -C, then the requirements will be kept but updated to the actual -versions being used. +To create a new go.mod file, use 'go help init'. For details see +'go help mod init' or https://golang.org/ref/mod#go-mod-init. -Finally, the update reformats the go.mod in a canonical formatting, so -that future mechanical changes will result in minimal diffs. +To add missing module requirements or remove unneeded requirements, +use 'go mod tidy'. For details, see 'go help mod tidy' or +https://golang.org/ref/mod#go-mod-tidy. -Because the module graph defines the meaning of import statements, any -commands that load packages also use and therefore update go.mod, -including go build, go get, go install, go list, go test, go mod graph, -go mod tidy, and go mod why. +To add, upgrade, downgrade, or remove a specific module requirement, use +'go get'. For details, see 'go help module-get' or +https://golang.org/ref/mod#go-get. -The expected language version, set by the go directive, determines -which language features are available when compiling the module. -Language features available in that version will be available for use. -Language features removed in earlier versions, or added in later versions, -will not be available. Note that the language version does not affect -build tags, which are determined by the Go release being used. +To make other changes or to parse go.mod as JSON for use by other tools, +use 'go mod edit'. See 'go help mod edit' or +https://golang.org/ref/mod#go-mod-edit. `, } diff --git a/cmd/go/_internal_/modload/import.go b/cmd/go/_internal_/modload/import.go index e5b17d7..bdfe465 100644 --- a/cmd/go/_internal_/modload/import.go +++ b/cmd/go/_internal_/modload/import.go @@ -5,18 +5,19 @@ package modload import ( + "context" "errors" "fmt" "go/build" "github.com/dependabot/gomodules-extracted/_internal_/goroot" + "io/fs" "os" "path/filepath" "sort" "strings" - "time" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/load" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/fsys" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/par" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/search" @@ -30,23 +31,52 @@ type ImportMissingError struct { Module module.Version QueryErr error + // isStd indicates whether we would expect to find the package in the standard + // library. This is normally true for all dotless import paths, but replace + // directives can cause us to treat the replaced paths as also being in + // modules. + isStd bool + + // replaced the highest replaced version of the module where the replacement + // contains the package. replaced is only set if the replacement is unused. + replaced module.Version + // newMissingVersion is set to a newer version of Module if one is present // in the build list. When set, we can't automatically upgrade. newMissingVersion string } -var _ load.ImportPathError = (*ImportMissingError)(nil) - func (e *ImportMissingError) Error() string { if e.Module.Path == "" { - if search.IsStandardImportPath(e.Path) { + if e.isStd { return fmt.Sprintf("package %s is not in GOROOT (%s)", e.Path, filepath.Join(cfg.GOROOT, "src", e.Path)) } - if e.QueryErr != nil { + if e.QueryErr != nil && e.QueryErr != ErrNoModRoot { return fmt.Sprintf("cannot find module providing package %s: %v", e.Path, e.QueryErr) } - return "cannot find module providing package " + e.Path + if cfg.BuildMod == "mod" || (cfg.BuildMod == "readonly" && allowMissingModuleImports) { + return "cannot find module providing package " + e.Path + } + + if e.replaced.Path != "" { + suggestArg := e.replaced.Path + if !modfetch.IsZeroPseudoVersion(e.replaced.Version) { + suggestArg = e.replaced.String() + } + return fmt.Sprintf("module %s provides package %s and is replaced but not required; to add it:\n\tgo get %s", e.replaced.Path, e.Path, suggestArg) + } + + message := fmt.Sprintf("no required module provides package %s", e.Path) + if e.QueryErr != nil { + return fmt.Sprintf("%s: %v", message, e.QueryErr) + } + return fmt.Sprintf("%s; to add it:\n\tgo get %s", message, e.Path) } + + if e.newMissingVersion != "" { + return fmt.Sprintf("package %s provided by %s at latest version %s but not at required version %s", e.Path, e.Module.Path, e.Module.Version, e.newMissingVersion) + } + return fmt.Sprintf("missing module for import: %s@%s provides %s", e.Module.Path, e.Module.Version, e.Path) } @@ -97,20 +127,95 @@ func (e *AmbiguousImportError) Error() string { return buf.String() } -var _ load.ImportPathError = &AmbiguousImportError{} +// ImportMissingSumError is reported in readonly mode when we need to check +// if a module contains a package, but we don't have a sum for its .zip file. +// We might need sums for multiple modules to verify the package is unique. +// +// TODO(#43653): consolidate multiple errors of this type into a single error +// that suggests a 'go get' command for root packages that transtively import +// packages from modules with missing sums. load.CheckPackageErrors would be +// a good place to consolidate errors, but we'll need to attach the import +// stack here. +type ImportMissingSumError struct { + importPath string + found bool + mods []module.Version + importer, importerVersion string // optional, but used for additional context + importerIsTest bool +} -// Import finds the module and directory in the build list -// containing the package with the given import path. -// The answer must be unique: Import returns an error -// if multiple modules attempt to provide the same package. -// Import can return a module with an empty m.Path, for packages in the standard library. -// Import can return an empty directory string, for fake packages like "C" and "unsafe". +func (e *ImportMissingSumError) Error() string { + var importParen string + if e.importer != "" { + importParen = fmt.Sprintf(" (imported by %s)", e.importer) + } + var message string + if e.found { + message = fmt.Sprintf("missing go.sum entry needed to verify package %s%s is provided by exactly one module", e.importPath, importParen) + } else { + message = fmt.Sprintf("missing go.sum entry for module providing package %s%s", e.importPath, importParen) + } + var hint string + if e.importer == "" { + // Importing package is unknown, or the missing package was named on the + // command line. Recommend 'go mod download' for the modules that could + // provide the package, since that shouldn't change go.mod. + args := make([]string, len(e.mods)) + for i, mod := range e.mods { + args[i] = mod.Path + } + hint = fmt.Sprintf("; to add:\n\tgo mod download %s", strings.Join(args, " ")) + } else { + // Importing package is known (common case). Recommend 'go get' on the + // current version of the importing package. + tFlag := "" + if e.importerIsTest { + tFlag = " -t" + } + version := "" + if e.importerVersion != "" { + version = "@" + e.importerVersion + } + hint = fmt.Sprintf("; to add:\n\tgo get%s %s%s", tFlag, e.importer, version) + } + return message + hint +} + +func (e *ImportMissingSumError) ImportPath() string { + return e.importPath +} + +type invalidImportError struct { + importPath string + err error +} + +func (e *invalidImportError) ImportPath() string { + return e.importPath +} + +func (e *invalidImportError) Error() string { + return e.err.Error() +} + +func (e *invalidImportError) Unwrap() error { + return e.err +} + +// importFromBuildList finds the module and directory in the build list +// containing the package with the given import path. The answer must be unique: +// importFromBuildList returns an error if multiple modules attempt to provide +// the same package. // -// If the package cannot be found in the current build list, -// Import returns an ImportMissingError as the error. -// If Import can identify a module that could be added to supply the package, -// the ImportMissingError records that module. -func Import(path string) (m module.Version, dir string, err error) { +// importFromBuildList can return a module with an empty m.Path, for packages in +// the standard library. +// +// importFromBuildList can return an empty directory string, for fake packages +// like "C" and "unsafe". +// +// If the package cannot be found in buildList, +// importFromBuildList returns an *ImportMissingError. +func importFromBuildList(ctx context.Context, path string, buildList []module.Version) (m module.Version, dir string, err error) { if strings.Contains(path, "@") { return module.Version{}, "", fmt.Errorf("import path should not have @version") } @@ -121,6 +226,10 @@ func Import(path string) (m module.Version, dir string, err error) { // There's no directory for import "C" or import "unsafe". return module.Version{}, "", nil } + // Before any further lookup, check that the path is valid. + if err := module.CheckImportPath(path); err != nil { + return module.Version{}, "", &invalidImportError{importPath: path, err: err} + } // Is the package in the standard library? pathIsStd := search.IsStandardImportPath(path) @@ -160,13 +269,24 @@ func Import(path string) (m module.Version, dir string, err error) { // Check each module on the build list. var dirs []string var mods []module.Version + var sumErrMods []module.Version for _, m := range buildList { if !maybeInModule(path, m.Path) { // Avoid possibly downloading irrelevant modules. continue } - root, isLocal, err := fetch(m) + needSum := true + root, isLocal, err := fetch(ctx, m, needSum) if err != nil { + if sumErr := (*sumMissingError)(nil); errors.As(err, &sumErr) { + // We are missing a sum needed to fetch a module in the build list. + // We can't verify that the package is unique, and we may not find + // the package at all. Keep checking other modules to decide which + // error to report. Multiple sums may be missing if we need to look in + // multiple nested modules to resolve the import. + sumErrMods = append(sumErrMods, m) + continue + } // Report fetch error. // Note that we don't know for sure this module is necessary, // but it certainly _could_ provide the package, and even if we @@ -182,88 +302,84 @@ func Import(path string) (m module.Version, dir string, err error) { dirs = append(dirs, dir) } } - if len(mods) == 1 { - return mods[0], dirs[0], nil - } - if len(mods) > 0 { + if len(mods) > 1 { return module.Version{}, "", &AmbiguousImportError{importPath: path, Dirs: dirs, Modules: mods} } - - // Look up module containing the package, for addition to the build list. - // Goal is to determine the module, download it to dir, and return m, dir, ErrMissing. - if cfg.BuildMod == "readonly" { - var queryErr error - if !pathIsStd { - if cfg.BuildModReason == "" { - queryErr = fmt.Errorf("import lookup disabled by -mod=%s", cfg.BuildMod) - } else { - queryErr = fmt.Errorf("import lookup disabled by -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) - } + if len(sumErrMods) > 0 { + return module.Version{}, "", &ImportMissingSumError{ + importPath: path, + mods: sumErrMods, + found: len(mods) > 0, } - return module.Version{}, "", &ImportMissingError{Path: path, QueryErr: queryErr} } - if modRoot == "" && !allowMissingModuleImports { - return module.Version{}, "", &ImportMissingError{ - Path: path, - QueryErr: errors.New("working directory is not part of a module"), - } + if len(mods) == 1 { + return mods[0], dirs[0], nil } - // Not on build list. - // To avoid spurious remote fetches, next try the latest replacement for each module. - // (golang.org/issue/26241) - if modFile != nil { - latest := map[string]string{} // path -> version - for _, r := range modFile.Replace { - if maybeInModule(path, r.Old.Path) { - // Don't use semver.Max here; need to preserve +incompatible suffix. - v := latest[r.Old.Path] - if semver.Compare(r.Old.Version, v) > 0 { - v = r.Old.Version - } - latest[r.Old.Path] = v - } - } + var queryErr error + if !HasModRoot() { + queryErr = ErrNoModRoot + } + return module.Version{}, "", &ImportMissingError{Path: path, QueryErr: queryErr, isStd: pathIsStd} +} - mods = make([]module.Version, 0, len(latest)) - for p, v := range latest { - // If the replacement didn't specify a version, synthesize a - // pseudo-version with an appropriate major version and a timestamp below - // any real timestamp. That way, if the main module is used from within - // some other module, the user will be able to upgrade the requirement to - // any real version they choose. - if v == "" { - if _, pathMajor, ok := module.SplitPathVersion(p); ok && len(pathMajor) > 0 { - v = modfetch.PseudoVersion(pathMajor[1:], "", time.Time{}, "000000000000") +// queryImport attempts to locate a module that can be added to the current +// build list to provide the package with the given import path. +// +// Unlike QueryPattern, queryImport prefers to add a replaced version of a +// module *before* checking the proxies for a version to add. +func queryImport(ctx context.Context, path string) (module.Version, error) { + // To avoid spurious remote fetches, try the latest replacement for each + // module (golang.org/issue/26241). + if index != nil { + var mods []module.Version + for mp, mv := range index.highestReplaced { + if !maybeInModule(path, mp) { + continue + } + if mv == "" { + // The only replacement is a wildcard that doesn't specify a version, so + // synthesize a pseudo-version with an appropriate major version and a + // timestamp below any real timestamp. That way, if the main module is + // used from within some other module, the user will be able to upgrade + // the requirement to any real version they choose. + if _, pathMajor, ok := module.SplitPathVersion(mp); ok && len(pathMajor) > 0 { + mv = modfetch.ZeroPseudoVersion(pathMajor[1:]) } else { - v = modfetch.PseudoVersion("v0", "", time.Time{}, "000000000000") + mv = modfetch.ZeroPseudoVersion("v0") } } - mods = append(mods, module.Version{Path: p, Version: v}) + mods = append(mods, module.Version{Path: mp, Version: mv}) } // Every module path in mods is a prefix of the import path. - // As in QueryPackage, prefer the longest prefix that satisfies the import. + // As in QueryPattern, prefer the longest prefix that satisfies the import. sort.Slice(mods, func(i, j int) bool { return len(mods[i].Path) > len(mods[j].Path) }) for _, m := range mods { - root, isLocal, err := fetch(m) + needSum := true + root, isLocal, err := fetch(ctx, m, needSum) if err != nil { - // Report fetch error as above. - return module.Version{}, "", err + if sumErr := (*sumMissingError)(nil); errors.As(err, &sumErr) { + return module.Version{}, &ImportMissingSumError{importPath: path} + } + return module.Version{}, err } if _, ok, err := dirInModule(path, m.Path, root, isLocal); err != nil { - return m, "", err + return m, err } else if ok { - return m, "", &ImportMissingError{Path: path, Module: m} + if cfg.BuildMod == "readonly" { + return module.Version{}, &ImportMissingError{Path: path, replaced: m} + } + return m, nil } } if len(mods) > 0 && module.CheckPath(path) != nil { // The package path is not valid to fetch remotely, - // so it can only exist if in a replaced module, + // so it can only exist in a replaced module, // and we know from the above loop that it is not. - return module.Version{}, "", &PackageNotInModuleError{ + return module.Version{}, &PackageNotInModuleError{ Mod: mods[0], Query: "latest", Pattern: path, @@ -272,36 +388,53 @@ func Import(path string) (m module.Version, dir string, err error) { } } - if pathIsStd { + if search.IsStandardImportPath(path) { // This package isn't in the standard library, isn't in any module already // in the build list, and isn't in any other module that the user has // shimmed in via a "replace" directive. // Moreover, the import path is reserved for the standard library, so - // QueryPackage cannot possibly find a module containing this package. + // QueryPattern cannot possibly find a module containing this package. // - // Instead of trying QueryPackage, report an ImportMissingError immediately. - return module.Version{}, "", &ImportMissingError{Path: path} + // Instead of trying QueryPattern, report an ImportMissingError immediately. + return module.Version{}, &ImportMissingError{Path: path, isStd: true} } + if cfg.BuildMod == "readonly" && !allowMissingModuleImports { + // In readonly mode, we can't write go.mod, so we shouldn't try to look up + // the module. If readonly mode was enabled explicitly, include that in + // the error message. + var queryErr error + if cfg.BuildModExplicit { + queryErr = fmt.Errorf("import lookup disabled by -mod=%s", cfg.BuildMod) + } else if cfg.BuildModReason != "" { + queryErr = fmt.Errorf("import lookup disabled by -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) + } + return module.Version{}, &ImportMissingError{Path: path, QueryErr: queryErr} + } + + // Look up module containing the package, for addition to the build list. + // Goal is to determine the module, download it to dir, + // and return m, dir, ImpportMissingError. fmt.Fprintf(os.Stderr, "go: finding module for package %s\n", path) - candidates, err := QueryPackage(path, "latest", Allowed) + candidates, err := QueryPackages(ctx, path, "latest", Selected, CheckAllowed) if err != nil { - if errors.Is(err, os.ErrNotExist) { + if errors.Is(err, fs.ErrNotExist) { // Return "cannot find module providing package […]" instead of whatever - // low-level error QueryPackage produced. - return module.Version{}, "", &ImportMissingError{Path: path, QueryErr: err} + // low-level error QueryPattern produced. + return module.Version{}, &ImportMissingError{Path: path, QueryErr: err} } else { - return module.Version{}, "", err + return module.Version{}, err } } - m = candidates[0].Mod - newMissingVersion := "" - for _, c := range candidates { + + candidate0MissingVersion := "" + for i, c := range candidates { cm := c.Mod + canAdd := true for _, bm := range buildList { if bm.Path == cm.Path && semver.Compare(bm.Version, cm.Version) > 0 { - // QueryPackage proposed that we add module cm to provide the package, + // QueryPattern proposed that we add module cm to provide the package, // but we already depend on a newer version of that module (and we don't // have the package). // @@ -309,13 +442,22 @@ func Import(path string) (m module.Version, dir string, err error) { // version (e.g., v1.0.0) of a module, but we have a newer version // of the same module in the build list (e.g., v1.0.1-beta), and // the package is not present there. - m = cm - newMissingVersion = bm.Version + canAdd = false + if i == 0 { + candidate0MissingVersion = bm.Version + } break } } + if canAdd { + return cm, nil + } + } + return module.Version{}, &ImportMissingError{ + Path: path, + Module: candidates[0].Mod, + newMissingVersion: candidate0MissingVersion, } - return m, "", &ImportMissingError{Path: path, Module: m, newMissingVersion: newMissingVersion} } // maybeInModule reports whether, syntactically, @@ -369,7 +511,7 @@ func dirInModule(path, mpath, mdir string, isLocal bool) (dir string, haveGoFile if isLocal { for d := dir; d != mdir && len(d) > len(mdir); { haveGoMod := haveGoModCache.Do(d, func() interface{} { - fi, err := os.Stat(filepath.Join(d, "go.mod")) + fi, err := fsys.Stat(filepath.Join(d, "go.mod")) return err == nil && !fi.IsDir() }).(bool) @@ -392,57 +534,65 @@ func dirInModule(path, mpath, mdir string, isLocal bool) (dir string, haveGoFile // We don't care about build tags, not even "+build ignore". // We're just looking for a plausible directory. res := haveGoFilesCache.Do(dir, func() interface{} { - ok, err := isDirWithGoFiles(dir) + ok, err := fsys.IsDirWithGoFiles(dir) return goFilesEntry{haveGoFiles: ok, err: err} }).(goFilesEntry) return dir, res.haveGoFiles, res.err } -func isDirWithGoFiles(dir string) (bool, error) { - f, err := os.Open(dir) - if err != nil { - if os.IsNotExist(err) { - return false, nil - } - return false, err - } - defer f.Close() - - names, firstErr := f.Readdirnames(-1) - if firstErr != nil { - if fi, err := f.Stat(); err == nil && !fi.IsDir() { - return false, nil - } - - // Rewrite the error from ReadDirNames to include the path if not present. - // See https://golang.org/issue/38923. - var pe *os.PathError - if !errors.As(firstErr, &pe) { - firstErr = &os.PathError{Op: "readdir", Path: dir, Err: firstErr} - } +// fetch downloads the given module (or its replacement) +// and returns its location. +// +// needSum indicates whether the module may be downloaded in readonly mode +// without a go.sum entry. It should only be false for modules fetched +// speculatively (for example, for incompatible version filtering). The sum +// will still be verified normally. +// +// The isLocal return value reports whether the replacement, +// if any, is local to the filesystem. +func fetch(ctx context.Context, mod module.Version, needSum bool) (dir string, isLocal bool, err error) { + if mod == Target { + return ModRoot(), true, nil } - - for _, name := range names { - if strings.HasSuffix(name, ".go") { - info, err := os.Stat(filepath.Join(dir, name)) - if err == nil && info.Mode().IsRegular() { - // If any .go source file exists, the package exists regardless of - // errors for other source files. Leave further error reporting for - // later. - return true, nil + if r := Replacement(mod); r.Path != "" { + if r.Version == "" { + dir = r.Path + if !filepath.IsAbs(dir) { + dir = filepath.Join(ModRoot(), dir) } - if firstErr == nil { + // Ensure that the replacement directory actually exists: + // dirInModule does not report errors for missing modules, + // so if we don't report the error now, later failures will be + // very mysterious. + if _, err := fsys.Stat(dir); err != nil { if os.IsNotExist(err) { - // If the file was concurrently deleted, or was a broken symlink, - // convert the error to an opaque error instead of one matching - // os.IsNotExist. - err = errors.New(err.Error()) + // Semantically the module version itself “exists” — we just don't + // have its source code. Remove the equivalence to os.ErrNotExist, + // and make the message more concise while we're at it. + err = fmt.Errorf("replacement directory %s does not exist", r.Path) + } else { + err = fmt.Errorf("replacement directory %s: %w", r.Path, err) } - firstErr = err + return dir, true, module.VersionError(mod, err) } + return dir, true, nil } + mod = r } - return false, firstErr + if HasModRoot() && cfg.BuildMod == "readonly" && needSum && !modfetch.HaveSum(mod) { + return "", false, module.VersionError(mod, &sumMissingError{}) + } + + dir, err = modfetch.Download(ctx, mod) + return dir, false, err +} + +type sumMissingError struct { + suggestion string +} + +func (e *sumMissingError) Error() string { + return "missing go.sum entry" + e.suggestion } diff --git a/cmd/go/_internal_/modload/init.go b/cmd/go/_internal_/modload/init.go index 31568f9..e68fc4f 100644 --- a/cmd/go/_internal_/modload/init.go +++ b/cmd/go/_internal_/modload/init.go @@ -6,28 +6,29 @@ package modload import ( "bytes" + "context" "encoding/json" "errors" "fmt" "go/build" "github.com/dependabot/gomodules-extracted/_internal_/lazyregexp" - "io/ioutil" "os" "path" "path/filepath" - "runtime/debug" + "sort" "strconv" "strings" + "sync" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cache" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/load" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/fsys" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/lockedfile" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modconv" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/mvs" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/search" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/str" "golang.org/x/mod/modfile" "golang.org/x/mod/module" @@ -35,7 +36,6 @@ import ( ) var ( - mustUseModules = false initialized bool modRoot string @@ -52,20 +52,42 @@ var ( gopath string - CmdModInit bool // running 'go mod init' - CmdModModule string // module argument for 'go mod init' + // RootMode determines whether a module root is needed. + RootMode Root + + // ForceUseModules may be set to force modules to be enabled when + // GO111MODULE=auto or to report an error when GO111MODULE=off. + ForceUseModules bool allowMissingModuleImports bool ) +type Root int + +const ( + // AutoRoot is the default for most commands. modload.Init will look for + // a go.mod file in the current directory or any parent. If none is found, + // modules may be disabled (GO111MODULE=on) or commands may run in a + // limited module mode. + AutoRoot Root = iota + + // NoRoot is used for commands that run in module mode and ignore any go.mod + // file the current directory or in parent directories. + NoRoot + + // NeedRoot is used for commands that must run in module mode and don't + // make sense without a main module. + NeedRoot +) + // ModFile returns the parsed go.mod file. // -// Note that after calling ImportPaths or LoadBuildList, +// Note that after calling LoadPackages or LoadAllModules, // the require statements in the modfile.File are no longer // the source of truth and will be ignored: edits made directly // will be lost at the next call to WriteGoMod. // To make permanent changes to the require statements -// in go.mod, edit it before calling ImportPaths or LoadBuildList. +// in go.mod, edit it before loading. func ModFile() *modfile.File { Init() if modFile == nil { @@ -92,19 +114,27 @@ func Init() { // Keep in sync with WillBeEnabled. We perform extra validation here, and // there are lots of diagnostics and side effects, so we can't use // WillBeEnabled directly. + var mustUseModules bool env := cfg.Getenv("GO111MODULE") switch env { default: base.Fatalf("go: unknown environment setting GO111MODULE=%s", env) - case "auto", "": - mustUseModules = false - case "on": + case "auto": + mustUseModules = ForceUseModules + case "on", "": mustUseModules = true case "off": + if ForceUseModules { + base.Fatalf("go: modules disabled by GO111MODULE=off; see 'go help modules'") + } mustUseModules = false return } + if err := fsys.Init(base.Cwd); err != nil { + base.Fatalf("go: %v", err) + } + // Disable any prompting for passwords by Git. // Only has an effect for 2.3.0 or later, but avoiding // the prompt in earlier versions is just too hard. @@ -132,15 +162,23 @@ func Init() { os.Setenv("GIT_SSH_COMMAND", "ssh -o ControlMaster=no") } - if CmdModInit { - // Running 'go mod init': go.mod will be created in current directory. - modRoot = base.Cwd + if modRoot != "" { + // modRoot set before Init was called ("go mod init" does this). + // No need to search for go.mod. + } else if RootMode == NoRoot { + if cfg.ModFile != "" && !base.InGOFLAGS("-modfile") { + base.Fatalf("go: -modfile cannot be used with commands that ignore the current module") + } + modRoot = "" } else { modRoot = findModuleRoot(base.Cwd) if modRoot == "" { if cfg.ModFile != "" { base.Fatalf("go: cannot find main module, but -modfile was set.\n\t-modfile cannot be used to set the module root directory.") } + if RootMode == NeedRoot { + base.Fatalf("go: %v", ErrNoModRoot) + } if !mustUseModules { // GO111MODULE is 'auto', and we can't find a module root. // Stay in GOPATH mode. @@ -154,39 +192,27 @@ func Init() { // when it happens. See golang.org/issue/26708. modRoot = "" fmt.Fprintf(os.Stderr, "go: warning: ignoring go.mod in system temp root %v\n", os.TempDir()) + if !mustUseModules { + return + } } } if cfg.ModFile != "" && !strings.HasSuffix(cfg.ModFile, ".mod") { base.Fatalf("go: -modfile=%s: file does not have .mod extension", cfg.ModFile) } - // We're in module mode. Install the hooks to make it work. - - if c := cache.Default(); c == nil { - // With modules, there are no install locations for packages - // other than the build cache. - base.Fatalf("go: cannot use modules with build cache disabled") - } - + // We're in module mode. Set any global variables that need to be set. + cfg.ModulesEnabled = true + setDefaultBuildMod() list := filepath.SplitList(cfg.BuildContext.GOPATH) if len(list) == 0 || list[0] == "" { base.Fatalf("missing $GOPATH") } gopath = list[0] - if _, err := os.Stat(filepath.Join(gopath, "go.mod")); err == nil { + if _, err := fsys.Stat(filepath.Join(gopath, "go.mod")); err == nil { base.Fatalf("$GOPATH/go.mod exists but should not") } - cfg.ModulesEnabled = true - load.ModBinDir = BinDir - load.ModLookup = Lookup - load.ModPackageModuleInfo = PackageModuleInfo - load.ModImportPaths = ImportPaths - load.ModPackageBuildInfo = PackageBuildInfo - load.ModInfoProg = ModInfoProg - load.ModImportFromFiles = ImportFromFiles - load.ModDirImportPath = DirImportPath - if modRoot == "" { // We're in module mode, but not inside a module. // @@ -211,10 +237,6 @@ func Init() { } } -func init() { - load.ModInit = Init -} - // WillBeEnabled checks whether modules should be enabled but does not // initialize modules by installing hooks. If Init has already been called, // WillBeEnabled returns the same result as Enabled. @@ -225,10 +247,12 @@ func init() { // be called until the command is installed and flags are parsed. Instead of // calling Init and Enabled, the main package can call this function. func WillBeEnabled() bool { - if modRoot != "" || mustUseModules { + if modRoot != "" || cfg.ModulesEnabled { + // Already enabled. return true } if initialized { + // Initialized, not enabled. return false } @@ -236,18 +260,14 @@ func WillBeEnabled() bool { // exits, so it can't call this function directly. env := cfg.Getenv("GO111MODULE") switch env { - case "on": + case "on", "": return true - case "auto", "": + case "auto": break default: return false } - if CmdModInit { - // Running 'go mod init': go.mod will be created in current directory. - return true - } if modRoot := findModuleRoot(base.Cwd); modRoot == "" { // GO111MODULE is 'auto', and we can't find a module root. // Stay in GOPATH mode. @@ -269,7 +289,7 @@ func WillBeEnabled() bool { // (usually through MustModRoot). func Enabled() bool { Init() - return modRoot != "" || mustUseModules + return modRoot != "" || cfg.ModulesEnabled } // ModRoot returns the root of the main module. @@ -303,16 +323,7 @@ func ModFilePath() string { return filepath.Join(modRoot, "go.mod") } -// printStackInDie causes die to print a stack trace. -// -// It is enabled by the testgo tag, and helps to diagnose paths that -// unexpectedly require a main module. -var printStackInDie = false - func die() { - if printStackInDie { - debug.PrintStack() - } if cfg.Getenv("GO111MODULE") == "off" { base.Fatalf("go: modules disabled by GO111MODULE=off; see 'go help modules'") } @@ -327,15 +338,21 @@ func die() { } base.Fatalf("go: cannot find main module, but found %s in %s\n\tto create a module there, run:\n\t%sgo mod init", name, dir, cdCmd) } - base.Fatalf("go: cannot find main module; see 'go help modules'") + base.Fatalf("go: %v", ErrNoModRoot) } -// InitMod sets Target and, if there is a main module, parses the initial build -// list from its go.mod file, creating and populating that file if needed. +var ErrNoModRoot = errors.New("go.mod file not found in current directory or any parent directory; see 'go help modules'") + +// LoadModFile sets Target and, if there is a main module, parses the initial +// build list from its go.mod file. // -// As a side-effect, InitMod sets a default for cfg.BuildMod if it does not -// already have an explicit value. -func InitMod() { +// LoadModFile may make changes in memory, like adding a go directive and +// ensuring requirements are consistent. WriteGoMod should be called later to +// write changes out to disk or report errors in readonly mode. +// +// As a side-effect, LoadModFile may change cfg.BuildMod to "vendor" if +// -mod wasn't set explicitly and automatic vendoring should be enabled. +func LoadModFile(ctx context.Context) { if len(buildList) > 0 { return } @@ -348,14 +365,6 @@ func InitMod() { return } - if CmdModInit { - // Running go mod init: do legacy module conversion - legacyModInit() - modFileToBuildList() - WriteGoMod() - return - } - gomod := ModFilePath() data, err := lockedfile.Read(gomod) if err != nil { @@ -363,7 +372,7 @@ func InitMod() { } var fixed bool - f, err := modfile.Parse(gomod, data, fixVersion(&fixed)) + f, err := modfile.Parse(gomod, data, fixVersion(ctx, &fixed)) if err != nil { // Errors returned by modfile.Parse begin with file:line. base.Fatalf("go: errors parsing go.mod:\n%s\n", err) @@ -371,32 +380,133 @@ func InitMod() { modFile = f index = indexModFile(data, f, fixed) - if len(f.Syntax.Stmt) == 0 || f.Module == nil { - // Empty mod file. Must add module path. - path, err := findModulePath(modRoot) - if err != nil { - base.Fatalf("go: %v", err) - } - f.AddModuleStmt(path) + if f.Module == nil { + // No module declaration. Must add module path. + base.Fatalf("go: no module declaration in go.mod. To specify the module path:\n\tgo mod edit -module=example.com/mod") } - if len(f.Syntax.Stmt) == 1 && f.Module != nil { - // Entire file is just a module statement. - // Populate require if possible. - legacyModInit() + if err := checkModulePathLax(f.Module.Mod.Path); err != nil { + base.Fatalf("go: %v", err) } + setDefaultBuildMod() // possibly enable automatic vendoring modFileToBuildList() - setDefaultBuildMod() if cfg.BuildMod == "vendor" { readVendorList() checkVendorConsistency() - } else { - // TODO(golang.org/issue/33326): if cfg.BuildMod != "readonly"? - WriteGoMod() } } +// CreateModFile initializes a new module by creating a go.mod file. +// +// If modPath is empty, CreateModFile will attempt to infer the path from the +// directory location within GOPATH. +// +// If a vendoring configuration file is present, CreateModFile will attempt to +// translate it to go.mod directives. The resulting build list may not be +// exactly the same as in the legacy configuration (for example, we can't get +// packages at multiple versions from the same module). +func CreateModFile(ctx context.Context, modPath string) { + modRoot = base.Cwd + Init() + modFilePath := ModFilePath() + if _, err := fsys.Stat(modFilePath); err == nil { + base.Fatalf("go: %s already exists", modFilePath) + } + + if modPath == "" { + var err error + modPath, err = findModulePath(modRoot) + if err != nil { + base.Fatalf("go: %v", err) + } + } else if err := checkModulePathLax(modPath); err != nil { + base.Fatalf("go: %v", err) + } + + fmt.Fprintf(os.Stderr, "go: creating new go.mod: module %s\n", modPath) + modFile = new(modfile.File) + modFile.AddModuleStmt(modPath) + addGoStmt() // Add the go directive before converted module requirements. + + convertedFrom, err := convertLegacyConfig(modPath) + if convertedFrom != "" { + fmt.Fprintf(os.Stderr, "go: copying requirements from %s\n", base.ShortPath(convertedFrom)) + } + if err != nil { + base.Fatalf("go: %v", err) + } + + modFileToBuildList() + WriteGoMod() + + // Suggest running 'go mod tidy' unless the project is empty. Even if we + // imported all the correct requirements above, we're probably missing + // some sums, so the next build command in -mod=readonly will likely fail. + // + // We look for non-hidden .go files or subdirectories to determine whether + // this is an existing project. Walking the tree for packages would be more + // accurate, but could take much longer. + empty := true + files, _ := os.ReadDir(modRoot) + for _, f := range files { + name := f.Name() + if strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_") { + continue + } + if strings.HasSuffix(name, ".go") || f.IsDir() { + empty = false + break + } + } + if !empty { + fmt.Fprintf(os.Stderr, "go: to add module requirements and sums:\n\tgo mod tidy\n") + } +} + +// checkModulePathLax checks that the path meets some minimum requirements +// to avoid confusing users or the module cache. The requirements are weaker +// than those of module.CheckPath to allow room for weakening module path +// requirements in the future, but strong enough to help users avoid significant +// problems. +func checkModulePathLax(p string) error { + // TODO(matloob): Replace calls of this function in this CL with calls + // to module.CheckImportPath once it's been laxened, if it becomes laxened. + // See golang.org/issue/29101 for a discussion about whether to make CheckImportPath + // more lax or more strict. + + errorf := func(format string, args ...interface{}) error { + return fmt.Errorf("invalid module path %q: %s", p, fmt.Sprintf(format, args...)) + } + + // Disallow shell characters " ' * < > ? ` | to avoid triggering bugs + // with file systems and subcommands. Disallow file path separators : and \ + // because path separators other than / will confuse the module cache. + // See fileNameOK in golang.org/x/mod/module/module.go. + shellChars := "`" + `\"'*<>?|` + fsChars := `\:` + if i := strings.IndexAny(p, shellChars); i >= 0 { + return errorf("contains disallowed shell character %q", p[i]) + } + if i := strings.IndexAny(p, fsChars); i >= 0 { + return errorf("contains disallowed path separator character %q", p[i]) + } + + // Ensure path.IsAbs and build.IsLocalImport are false, and that the path is + // invariant under path.Clean, also to avoid confusing the module cache. + if path.IsAbs(p) { + return errorf("is an absolute path") + } + if build.IsLocalImport(p) { + return errorf("is a local import path") + } + if path.Clean(p) != p { + return errorf("is not clean") + } + + return nil +} + // fixVersion returns a modfile.VersionFixer implemented using the Query function. // // It resolves commit hashes and branch names to versions, @@ -404,7 +514,7 @@ func InitMod() { // and does nothing for versions that already appear to be canonical. // // The VersionFixer sets 'fixed' if it ever returns a non-canonical version. -func fixVersion(fixed *bool) modfile.VersionFixer { +func fixVersion(ctx context.Context, fixed *bool) modfile.VersionFixer { return func(path, vers string) (resolved string, err error) { defer func() { if err == nil && resolved != vers { @@ -431,12 +541,13 @@ func fixVersion(fixed *bool) modfile.VersionFixer { } } if vers != "" && module.CanonicalVersion(vers) == vers { - if err := module.CheckPathMajor(vers, pathMajor); err == nil { - return vers, nil + if err := module.CheckPathMajor(vers, pathMajor); err != nil { + return "", module.VersionError(module.Version{Path: path, Version: vers}, err) } + return vers, nil } - info, err := Query(path, vers, "", nil) + info, err := Query(ctx, path, vers, "", nil) if err != nil { return "", err } @@ -465,88 +576,78 @@ func modFileToBuildList() { list := []module.Version{Target} for _, r := range modFile.Require { - list = append(list, r.Mod) + if index != nil && index.exclude[r.Mod] { + if cfg.BuildMod == "mod" { + fmt.Fprintf(os.Stderr, "go: dropping requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version) + } else { + fmt.Fprintf(os.Stderr, "go: ignoring requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version) + } + } else { + list = append(list, r.Mod) + } } buildList = list } -// setDefaultBuildMod sets a default value for cfg.BuildMod -// if it is currently empty. +// setDefaultBuildMod sets a default value for cfg.BuildMod if the -mod flag +// wasn't provided. setDefaultBuildMod may be called multiple times. func setDefaultBuildMod() { - if cfg.BuildMod != "" { + if cfg.BuildModExplicit { // Don't override an explicit '-mod=' argument. return } - cfg.BuildMod = "mod" + if cfg.CmdName == "get" || strings.HasPrefix(cfg.CmdName, "mod ") { - // Don't set -mod implicitly for commands whose purpose is to - // manipulate the build list. + // 'get' and 'go mod' commands may update go.mod automatically. + // TODO(jayconrod): should this narrower? Should 'go mod download' or + // 'go mod graph' update go.mod by default? + cfg.BuildMod = "mod" return } if modRoot == "" { + cfg.BuildMod = "readonly" return } - if fi, err := os.Stat(filepath.Join(modRoot, "vendor")); err == nil && fi.IsDir() { + if fi, err := fsys.Stat(filepath.Join(modRoot, "vendor")); err == nil && fi.IsDir() { modGo := "unspecified" - if index.goVersion != "" { - if semver.Compare("v"+index.goVersion, "v1.14") >= 0 { + if index != nil && index.goVersionV != "" { + if semver.Compare(index.goVersionV, "v1.14") >= 0 { // The Go version is at least 1.14, and a vendor directory exists. // Set -mod=vendor by default. cfg.BuildMod = "vendor" cfg.BuildModReason = "Go version in go.mod is at least 1.14 and vendor directory exists." return } else { - modGo = index.goVersion + modGo = index.goVersionV[1:] } } - // Since a vendor directory exists, we have a non-trivial reason for - // choosing -mod=mod, although it probably won't be used for anything. - // Record the reason anyway for consistency. - // It may be overridden if we switch to mod=readonly below. - cfg.BuildModReason = fmt.Sprintf("Go version in go.mod is %s.", modGo) + // Since a vendor directory exists, we should record why we didn't use it. + // This message won't normally be shown, but it may appear with import errors. + cfg.BuildModReason = fmt.Sprintf("Go version in go.mod is %s, so vendor directory was not used.", modGo) } - p := ModFilePath() - if fi, err := os.Stat(p); err == nil && !hasWritePerm(p, fi) { - cfg.BuildMod = "readonly" - cfg.BuildModReason = "go.mod file is read-only." - } + cfg.BuildMod = "readonly" } -func legacyModInit() { - if modFile == nil { - path, err := findModulePath(modRoot) - if err != nil { - base.Fatalf("go: %v", err) - } - fmt.Fprintf(os.Stderr, "go: creating new go.mod: module %s\n", path) - modFile = new(modfile.File) - modFile.AddModuleStmt(path) - addGoStmt() // Add the go directive before converted module requirements. - } - +// convertLegacyConfig imports module requirements from a legacy vendoring +// configuration file, if one is present. +func convertLegacyConfig(modPath string) (from string, err error) { for _, name := range altConfigs { cfg := filepath.Join(modRoot, name) - data, err := ioutil.ReadFile(cfg) + data, err := os.ReadFile(cfg) if err == nil { convert := modconv.Converters[name] if convert == nil { - return + return "", nil } - fmt.Fprintf(os.Stderr, "go: copying requirements from %s\n", base.ShortPath(cfg)) cfg = filepath.ToSlash(cfg) - if err := modconv.ConvertLegacyConfig(modFile, cfg, data); err != nil { - base.Fatalf("go: %v", err) - } - if len(modFile.Syntax.Stmt) == 1 { - // Add comment to avoid re-converting every time it runs. - modFile.AddComment("// go: no requirements found in " + name) - } - return + err := modconv.ConvertLegacyConfig(modFile, cfg, data) + return name, err } } + return "", nil } // addGoStmt adds a go directive to the go.mod file if it does not already include one. @@ -588,7 +689,7 @@ func findModuleRoot(dir string) (root string) { // Look for enclosing go.mod. for { - if fi, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil && !fi.IsDir() { + if fi, err := fsys.Stat(filepath.Join(dir, "go.mod")); err == nil && !fi.IsDir() { return dir } d := filepath.Dir(dir) @@ -612,7 +713,7 @@ func findAltConfig(dir string) (root, name string) { } for { for _, name := range altConfigs { - if fi, err := os.Stat(filepath.Join(dir, name)); err == nil && !fi.IsDir() { + if fi, err := fsys.Stat(filepath.Join(dir, name)); err == nil && !fi.IsDir() { return dir, name } } @@ -626,14 +727,6 @@ func findAltConfig(dir string) (root, name string) { } func findModulePath(dir string) (string, error) { - if CmdModModule != "" { - // Running go mod init x/y/z; return x/y/z. - if err := module.CheckImportPath(CmdModModule); err != nil { - return "", err - } - return CmdModModule, nil - } - // TODO(bcmills): once we have located a plausible module path, we should // query version control (if available) to verify that it matches the major // version of the most recent tag. @@ -642,9 +735,9 @@ func findModulePath(dir string) (string, error) { // Cast about for import comments, // first in top-level directory, then in subdirectories. - list, _ := ioutil.ReadDir(dir) + list, _ := os.ReadDir(dir) for _, info := range list { - if info.Mode().IsRegular() && strings.HasSuffix(info.Name(), ".go") { + if info.Type().IsRegular() && strings.HasSuffix(info.Name(), ".go") { if com := findImportComment(filepath.Join(dir, info.Name())); com != "" { return com, nil } @@ -652,9 +745,9 @@ func findModulePath(dir string) (string, error) { } for _, info1 := range list { if info1.IsDir() { - files, _ := ioutil.ReadDir(filepath.Join(dir, info1.Name())) + files, _ := os.ReadDir(filepath.Join(dir, info1.Name())) for _, info2 := range files { - if info2.Mode().IsRegular() && strings.HasSuffix(info2.Name(), ".go") { + if info2.Type().IsRegular() && strings.HasSuffix(info2.Name(), ".go") { if com := findImportComment(filepath.Join(dir, info1.Name(), info2.Name())); com != "" { return path.Dir(com), nil } @@ -664,7 +757,7 @@ func findModulePath(dir string) (string, error) { } // Look for Godeps.json declaring import path. - data, _ := ioutil.ReadFile(filepath.Join(dir, "Godeps/Godeps.json")) + data, _ := os.ReadFile(filepath.Join(dir, "Godeps/Godeps.json")) var cfg1 struct{ ImportPath string } json.Unmarshal(data, &cfg1) if cfg1.ImportPath != "" { @@ -672,7 +765,7 @@ func findModulePath(dir string) (string, error) { } // Look for vendor.json declaring import path. - data, _ = ioutil.ReadFile(filepath.Join(dir, "vendor/vendor.json")) + data, _ = os.ReadFile(filepath.Join(dir, "vendor/vendor.json")) var cfg2 struct{ RootPath string } json.Unmarshal(data, &cfg2) if cfg2.RootPath != "" { @@ -680,16 +773,35 @@ func findModulePath(dir string) (string, error) { } // Look for path in GOPATH. + var badPathErr error for _, gpdir := range filepath.SplitList(cfg.BuildContext.GOPATH) { if gpdir == "" { continue } if rel := search.InDir(dir, filepath.Join(gpdir, "src")); rel != "" && rel != "." { - return filepath.ToSlash(rel), nil + path := filepath.ToSlash(rel) + // TODO(matloob): replace this with module.CheckImportPath + // once it's been laxened. + // Only checkModulePathLax here. There are some unpublishable + // module names that are compatible with checkModulePathLax + // but they already work in GOPATH so don't break users + // trying to do a build with modules. gorelease will alert users + // publishing their modules to fix their paths. + if err := checkModulePathLax(path); err != nil { + badPathErr = err + break + } + return path, nil } } - msg := `cannot determine module path for source directory %s (outside GOPATH, module path must be specified) + reason := "outside GOPATH, module path must be specified" + if badPathErr != nil { + // return a different error message if the module was in GOPATH, but + // the module path determined above would be an invalid path. + reason = fmt.Sprintf("bad module path inferred from directory in GOPATH: %v", badPathErr) + } + msg := `cannot determine module path for source directory %s (%s) Example usage: 'go mod init example.com/m' to initialize a v0 or v1 module @@ -697,7 +809,7 @@ Example usage: Run 'go help mod init' for more information. ` - return "", fmt.Errorf(msg, dir) + return "", fmt.Errorf(msg, dir, reason) } var ( @@ -705,7 +817,7 @@ var ( ) func findImportComment(file string) string { - data, err := ioutil.ReadFile(file) + data, err := os.ReadFile(file) if err != nil { return "" } @@ -738,14 +850,16 @@ func AllowWriteGoMod() { // MinReqs returns a Reqs with minimal additional dependencies of Target, // as will be written to go.mod. func MinReqs() mvs.Reqs { - var retain []string + retain := append([]string{}, additionalExplicitRequirements...) for _, m := range buildList[1:] { _, explicit := index.require[m] if explicit || loaded.direct[m.Path] { retain = append(retain, m.Path) } } - min, err := mvs.Req(Target, retain, Reqs()) + sort.Strings(retain) + str.Uniq(&retain) + min, err := mvs.Req(Target, retain, &mvsReqs{buildList: buildList}) if err != nil { base.Fatalf("go: %v", err) } @@ -791,20 +905,23 @@ func WriteGoMod() { if dirty && cfg.BuildMod == "readonly" { // If we're about to fail due to -mod=readonly, // prefer to report a dirty go.mod over a dirty go.sum - if cfg.BuildModReason != "" { + if cfg.BuildModExplicit { + base.Fatalf("go: updates to go.mod needed, disabled by -mod=readonly") + } else if cfg.BuildModReason != "" { base.Fatalf("go: updates to go.mod needed, disabled by -mod=readonly\n\t(%s)", cfg.BuildModReason) } else { - base.Fatalf("go: updates to go.mod needed, disabled by -mod=readonly") + base.Fatalf("go: updates to go.mod needed; to update it:\n\tgo mod tidy") } } - // Always update go.sum, even if we didn't change go.mod: we may have - // downloaded modules that we didn't have before. - modfetch.WriteGoSum() if !dirty && cfg.CmdName != "mod tidy" { // The go.mod file has the same semantic content that it had before // (but not necessarily the same exact bytes). - // Ignore any intervening edits. + // Don't write go.mod, but write go.sum in case we added or trimmed sums. + // 'go mod init' shouldn't write go.sum, since it will be incomplete. + if cfg.CmdName != "mod init" { + modfetch.WriteGoSum(keepSums(true)) + } return } @@ -815,6 +932,12 @@ func WriteGoMod() { defer func() { // At this point we have determined to make the go.mod file on disk equal to new. index = indexModFile(new, modFile, false) + + // Update go.sum after releasing the side lock and refreshing the index. + // 'go mod init' shouldn't write go.sum, since it will be incomplete. + if cfg.CmdName != "mod init" { + modfetch.WriteGoSum(keepSums(true)) + } }() // Make a best-effort attempt to acquire the side lock, only to exclude @@ -849,3 +972,101 @@ func WriteGoMod() { base.Fatalf("go: updating go.mod: %v", err) } } + +// keepSums returns a set of module sums to preserve in go.sum. The set +// includes entries for all modules used to load packages (according to +// the last load function such as LoadPackages or ImportFromFiles). +// It also contains entries for go.mod files needed for MVS (the version +// of these entries ends with "/go.mod"). +// +// If keepBuildListZips is true, the set also includes sums for zip files for +// all modules in the build list with replacements applied. 'go get' and +// 'go mod download' may add sums to this set when adding a requirement on a +// module without a root package or when downloading a direct or indirect +// dependency. +func keepSums(keepBuildListZips bool) map[module.Version]bool { + // Re-derive the build list using the current list of direct requirements. + // Keep the sum for the go.mod of each visited module version (or its + // replacement). + modkey := func(m module.Version) module.Version { + return module.Version{Path: m.Path, Version: m.Version + "/go.mod"} + } + keep := make(map[module.Version]bool) + var mu sync.Mutex + reqs := &keepSumReqs{ + Reqs: &mvsReqs{buildList: buildList}, + visit: func(m module.Version) { + // If we build using a replacement module, keep the sum for the replacement, + // since that's the code we'll actually use during a build. + mu.Lock() + r := Replacement(m) + if r.Path == "" { + keep[modkey(m)] = true + } else { + keep[modkey(r)] = true + } + mu.Unlock() + }, + } + buildList, err := mvs.BuildList(Target, reqs) + if err != nil { + panic(fmt.Sprintf("unexpected error reloading build list: %v", err)) + } + + actualMods := make(map[string]module.Version) + for _, m := range buildList[1:] { + if r := Replacement(m); r.Path != "" { + actualMods[m.Path] = r + } else { + actualMods[m.Path] = m + } + } + + // Add entries for modules in the build list with paths that are prefixes of + // paths of loaded packages. We need to retain sums for modules needed to + // report ambiguous import errors. We use our re-derived build list, + // since the global build list may have been tidied. + if loaded != nil { + for _, pkg := range loaded.pkgs { + if pkg.testOf != nil || pkg.inStd || module.CheckImportPath(pkg.path) != nil { + continue + } + for prefix := pkg.path; prefix != "."; prefix = path.Dir(prefix) { + if m, ok := actualMods[prefix]; ok { + keep[m] = true + } + } + } + } + + // Add entries for the zip of each module in the build list. + // We might not need all of these (tidy does not add them), but they may be + // added by a specific 'go get' or 'go mod download' command to resolve + // missing import sum errors. + if keepBuildListZips { + for _, m := range actualMods { + keep[m] = true + } + } + + return keep +} + +// keepSumReqs embeds another Reqs implementation. The Required method +// calls visit for each version in the module graph. +type keepSumReqs struct { + mvs.Reqs + visit func(module.Version) +} + +func (r *keepSumReqs) Required(m module.Version) ([]module.Version, error) { + r.visit(m) + return r.Reqs.Required(m) +} + +func TrimGoSum() { + // Don't retain sums for the zip file of every module in the build list. + // We may not need them all to build the main module's packages. + keepBuildListZips := false + modfetch.TrimGoSum(keepSums(keepBuildListZips)) +} diff --git a/cmd/go/_internal_/modload/list.go b/cmd/go/_internal_/modload/list.go index 12fae9c..cb6ad8b 100644 --- a/cmd/go/_internal_/modload/list.go +++ b/cmd/go/_internal_/modload/list.go @@ -5,47 +5,62 @@ package modload import ( + "context" "errors" "fmt" "os" + "runtime" "strings" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modinfo" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/par" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/search" "golang.org/x/mod/module" ) -func ListModules(args []string, listU, listVersions bool) []*modinfo.ModulePublic { - mods := listModules(args, listVersions) - if listU || listVersions { - var work par.Work +func ListModules(ctx context.Context, args []string, listU, listVersions, listRetracted bool) []*modinfo.ModulePublic { + mods := listModules(ctx, args, listVersions, listRetracted) + + type token struct{} + sem := make(chan token, runtime.GOMAXPROCS(0)) + if listU || listVersions || listRetracted { for _, m := range mods { - work.Add(m) + add := func(m *modinfo.ModulePublic) { + sem <- token{} + go func() { + if listU { + addUpdate(ctx, m) + } + if listVersions { + addVersions(ctx, m, listRetracted) + } + if listRetracted || listU { + addRetraction(ctx, m) + } + <-sem + }() + } + + add(m) if m.Replace != nil { - work.Add(m.Replace) + add(m.Replace) } } - work.Do(10, func(item interface{}) { - m := item.(*modinfo.ModulePublic) - if listU { - addUpdate(m) - } - if listVersions { - addVersions(m) - } - }) } + // Fill semaphore channel to wait for all tasks to finish. + for n := cap(sem); n > 0; n-- { + sem <- token{} + } + return mods } -func listModules(args []string, listVersions bool) []*modinfo.ModulePublic { - LoadBuildList() +func listModules(ctx context.Context, args []string, listVersions, listRetracted bool) []*modinfo.ModulePublic { + LoadAllModules(ctx) if len(args) == 0 { - return []*modinfo.ModulePublic{moduleInfo(buildList[0], true)} + return []*modinfo.ModulePublic{moduleInfo(ctx, buildList[0], true, listRetracted)} } var mods []*modinfo.ModulePublic @@ -58,7 +73,7 @@ func listModules(args []string, listVersions bool) []*modinfo.ModulePublic { base.Fatalf("go: cannot use relative path %s to specify module", arg) } if !HasModRoot() && (arg == "all" || strings.Contains(arg, "...")) { - base.Fatalf("go: cannot match %q: working directory is not part of a module", arg) + base.Fatalf("go: cannot match %q: %v", arg, ErrNoModRoot) } if i := strings.Index(arg, "@"); i >= 0 { path := arg[:i] @@ -71,7 +86,13 @@ func listModules(args []string, listVersions bool) []*modinfo.ModulePublic { } } - info, err := Query(path, vers, current, nil) + allowed := CheckAllowed + if IsRevisionQuery(vers) || listRetracted { + // Allow excluded and retracted versions if the user asked for a + // specific revision or used 'go list -retracted'. + allowed = nil + } + info, err := Query(ctx, path, vers, current, allowed) if err != nil { mods = append(mods, &modinfo.ModulePublic{ Path: path, @@ -80,7 +101,8 @@ func listModules(args []string, listVersions bool) []*modinfo.ModulePublic { }) continue } - mods = append(mods, moduleInfo(module.Version{Path: path, Version: info.Version}, false)) + mod := moduleInfo(ctx, module.Version{Path: path, Version: info.Version}, false, listRetracted) + mods = append(mods, mod) continue } @@ -105,7 +127,7 @@ func listModules(args []string, listVersions bool) []*modinfo.ModulePublic { matched = true if !matchedBuildList[i] { matchedBuildList[i] = true - mods = append(mods, moduleInfo(m, true)) + mods = append(mods, moduleInfo(ctx, m, true, listRetracted)) } } } @@ -115,9 +137,10 @@ func listModules(args []string, listVersions bool) []*modinfo.ModulePublic { // Don't make the user provide an explicit '@latest' when they're // explicitly asking what the available versions are. // Instead, resolve the module, even if it isn't an existing dependency. - info, err := Query(arg, "latest", "", nil) + info, err := Query(ctx, arg, "latest", "", nil) if err == nil { - mods = append(mods, moduleInfo(module.Version{Path: arg, Version: info.Version}, false)) + mod := moduleInfo(ctx, module.Version{Path: arg, Version: info.Version}, false, listRetracted) + mods = append(mods, mod) } else { mods = append(mods, &modinfo.ModulePublic{ Path: arg, diff --git a/cmd/go/_internal_/modload/load.go b/cmd/go/_internal_/modload/load.go index b7fabe2..0315f89 100644 --- a/cmd/go/_internal_/modload/load.go +++ b/cmd/go/_internal_/modload/load.go @@ -4,63 +4,202 @@ package modload +// This file contains the module-mode package loader, as well as some accessory +// functions pertaining to the package import graph. +// +// There are two exported entry points into package loading — LoadPackages and +// ImportFromFiles — both implemented in terms of loadFromRoots, which itself +// manipulates an instance of the loader struct. +// +// Although most of the loading state is maintained in the loader struct, +// one key piece - the build list - is a global, so that it can be modified +// separate from the loading operation, such as during "go get" +// upgrades/downgrades or in "go mod" operations. +// TODO(#40775): It might be nice to make the loader take and return +// a buildList rather than hard-coding use of the global. +// +// Loading is an iterative process. On each iteration, we try to load the +// requested packages and their transitive imports, then try to resolve modules +// for any imported packages that are still missing. +// +// The first step of each iteration identifies a set of “root” packages. +// Normally the root packages are exactly those matching the named pattern +// arguments. However, for the "all" meta-pattern, the final set of packages is +// computed from the package import graph, and therefore cannot be an initial +// input to loading that graph. Instead, the root packages for the "all" pattern +// are those contained in the main module, and allPatternIsRoot parameter to the +// loader instructs it to dynamically expand those roots to the full "all" +// pattern as loading progresses. +// +// The pkgInAll flag on each loadPkg instance tracks whether that +// package is known to match the "all" meta-pattern. +// A package matches the "all" pattern if: +// - it is in the main module, or +// - it is imported by any test in the main module, or +// - it is imported by another package in "all", or +// - the main module specifies a go version ≤ 1.15, and the package is imported +// by a *test of* another package in "all". +// +// When we implement lazy loading, we will record the modules providing packages +// in "all" even when we are only loading individual packages, so we set the +// pkgInAll flag regardless of the whether the "all" pattern is a root. +// (This is necessary to maintain the “import invariant” described in +// https://golang.org/design/36460-lazy-module-loading.) +// +// Because "go mod vendor" prunes out the tests of vendored packages, the +// behavior of the "all" pattern with -mod=vendor in Go 1.11–1.15 is the same +// as the "all" pattern (regardless of the -mod flag) in 1.16+. +// The allClosesOverTests parameter to the loader indicates whether the "all" +// pattern should close over tests (as in Go 1.11–1.15) or stop at only those +// packages transitively imported by the packages and tests in the main module +// ("all" in Go 1.16+ and "go mod vendor" in Go 1.11+). +// +// Note that it is possible for a loaded package NOT to be in "all" even when we +// are loading the "all" pattern. For example, packages that are transitive +// dependencies of other roots named on the command line must be loaded, but are +// not in "all". (The mod_notall test illustrates this behavior.) +// Similarly, if the LoadTests flag is set but the "all" pattern does not close +// over test dependencies, then when we load the test of a package that is in +// "all" but outside the main module, the dependencies of that test will not +// necessarily themselves be in "all". (That configuration does not arise in Go +// 1.11–1.15, but it will be possible in Go 1.16+.) +// +// Loading proceeds from the roots, using a parallel work-queue with a limit on +// the amount of active work (to avoid saturating disks, CPU cores, and/or +// network connections). Each package is added to the queue the first time it is +// imported by another package. When we have finished identifying the imports of +// a package, we add the test for that package if it is needed. A test may be +// needed if: +// - the package matches a root pattern and tests of the roots were requested, or +// - the package is in the main module and the "all" pattern is requested +// (because the "all" pattern includes the dependencies of tests in the main +// module), or +// - the package is in "all" and the definition of "all" we are using includes +// dependencies of tests (as is the case in Go ≤1.15). +// +// After all available packages have been loaded, we examine the results to +// identify any requested or imported packages that are still missing, and if +// so, which modules we could add to the module graph in order to make the +// missing packages available. We add those to the module graph and iterate, +// until either all packages resolve successfully or we cannot identify any +// module that would resolve any remaining missing package. +// +// If the main module is “tidy” (that is, if "go mod tidy" is a no-op for it) +// and all requested packages are in "all", then loading completes in a single +// iteration. +// TODO(bcmills): We should also be able to load in a single iteration if the +// requested packages all come from modules that are themselves tidy, regardless +// of whether those packages are in "all". Today, that requires two iterations +// if those packages are not found in existing dependencies of the main module. + import ( "bytes" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/imports" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/mvs" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/par" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/search" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/str" + "context" "errors" "fmt" "go/build" + "io/fs" "os" "path" pathpkg "path" "path/filepath" + "reflect" + "runtime" "sort" "strings" + "sync" + "sync/atomic" + + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/fsys" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/imports" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/mvs" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/par" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/search" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/str" "golang.org/x/mod/module" ) -// buildList is the list of modules to use for building packages. -// It is initialized by calling ImportPaths, ImportFromFiles, -// LoadALL, or LoadBuildList, each of which uses loaded.load. -// -// Ideally, exactly ONE of those functions would be called, -// and exactly once. Most of the time, that's true. -// During "go get" it may not be. TODO(rsc): Figure out if -// that restriction can be established, or else document why not. -// -var buildList []module.Version - // loaded is the most recently-used package loader. // It holds details about individual packages. -// -// Note that loaded.buildList is only valid during a load operation; -// afterward, it is copied back into the global buildList, -// which should be used instead. var loaded *loader -// ImportPaths returns the set of packages matching the args (patterns), -// on the target platform. Modules may be added to the build list -// to satisfy new imports. -func ImportPaths(patterns []string) []*search.Match { - matches := ImportPathsQuiet(patterns, imports.Tags()) - search.WarnUnmatched(matches) - return matches +// PackageOpts control the behavior of the LoadPackages function. +type PackageOpts struct { + // Tags are the build tags in effect (as interpreted by the + // cmd/go/internal/imports package). + // If nil, treated as equivalent to imports.Tags(). + Tags map[string]bool + + // ResolveMissingImports indicates that we should attempt to add module + // dependencies as needed to resolve imports of packages that are not found. + // + // For commands that support the -mod flag, resolving imports may still fail + // if the flag is set to "readonly" (the default) or "vendor". + ResolveMissingImports bool + + // AllowPackage, if non-nil, is called after identifying the module providing + // each package. If AllowPackage returns a non-nil error, that error is set + // for the package, and the imports and test of that package will not be + // loaded. + // + // AllowPackage may be invoked concurrently by multiple goroutines, + // and may be invoked multiple times for a given package path. + AllowPackage func(ctx context.Context, path string, mod module.Version) error + + // LoadTests loads the test dependencies of each package matching a requested + // pattern. If ResolveMissingImports is also true, test dependencies will be + // resolved if missing. + LoadTests bool + + // UseVendorAll causes the "all" package pattern to be interpreted as if + // running "go mod vendor" (or building with "-mod=vendor"). + // + // This is a no-op for modules that declare 'go 1.16' or higher, for which this + // is the default (and only) interpretation of the "all" pattern in module mode. + UseVendorAll bool + + // AllowErrors indicates that LoadPackages should not terminate the process if + // an error occurs. + AllowErrors bool + + // SilenceErrors indicates that LoadPackages should not print errors + // that occur while loading packages. SilenceErrors implies AllowErrors. + SilenceErrors bool + + // SilenceMissingStdImports indicates that LoadPackages should not print + // errors or terminate the process if an imported package is missing, and the + // import path looks like it might be in the standard library (perhaps in a + // future version). + SilenceMissingStdImports bool + + // SilenceUnmatchedWarnings suppresses the warnings normally emitted for + // patterns that did not match any packages. + SilenceUnmatchedWarnings bool } -// ImportPathsQuiet is like ImportPaths but does not warn about patterns with -// no matches. It also lets the caller specify a set of build tags to match -// packages. The build tags should typically be imports.Tags() or -// imports.AnyTags(); a nil map has no special meaning. -func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { - updateMatches := func(matches []*search.Match, iterating bool) { +// LoadPackages identifies the set of packages matching the given patterns and +// loads the packages in the import graph rooted at that set. +func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (matches []*search.Match, loadedPackages []string) { + LoadModFile(ctx) + if opts.Tags == nil { + opts.Tags = imports.Tags() + } + + patterns = search.CleanPatterns(patterns) + matches = make([]*search.Match, 0, len(patterns)) + allPatternIsRoot := false + for _, pattern := range patterns { + matches = append(matches, search.NewMatch(pattern)) + if pattern == "all" { + allPatternIsRoot = true + } + } + + updateMatches := func(ld *loader) { for _, m := range matches { switch { case m.IsLocal(): @@ -87,7 +226,7 @@ func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { // indicates that. ModRoot() - if !iterating { + if ld != nil { m.AddError(err) } continue @@ -100,19 +239,18 @@ func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { case strings.Contains(m.Pattern(), "..."): m.Errs = m.Errs[:0] - matchPackages(m, loaded.tags, includeStd, buildList) + matchPackages(ctx, m, opts.Tags, includeStd, buildList) case m.Pattern() == "all": - loaded.testAll = true - if iterating { - // Enumerate the packages in the main module. - // We'll load the dependencies as we find them. + if ld == nil { + // The initial roots are the packages in the main module. + // loadFromRoots will expand that to "all". m.Errs = m.Errs[:0] - matchPackages(m, loaded.tags, omitStd, []module.Version{Target}) + matchPackages(ctx, m, opts.Tags, omitStd, []module.Version{Target}) } else { // Starting with the packages in the main module, // enumerate the full list of "all". - m.Pkgs = loaded.computePatternAll(m.Pkgs) + m.Pkgs = ld.computePatternAll() } case m.Pattern() == "std" || m.Pattern() == "cmd": @@ -126,49 +264,76 @@ func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { } } - InitMod() + loaded = loadFromRoots(loaderParams{ + PackageOpts: opts, - var matches []*search.Match - for _, pattern := range search.CleanPatterns(patterns) { - matches = append(matches, search.NewMatch(pattern)) - } + allClosesOverTests: index.allPatternClosesOverTests() && !opts.UseVendorAll, + allPatternIsRoot: allPatternIsRoot, - loaded = newLoader(tags) - loaded.load(func() []string { - var roots []string - updateMatches(matches, true) - for _, m := range matches { - roots = append(roots, m.Pkgs...) - } - return roots + listRoots: func() (roots []string) { + updateMatches(nil) + for _, m := range matches { + roots = append(roots, m.Pkgs...) + } + return roots + }, }) // One last pass to finalize wildcards. - updateMatches(matches, false) - checkMultiplePaths() - WriteGoMod() + updateMatches(loaded) - return matches -} + // Report errors, if any. + checkMultiplePaths() + for _, pkg := range loaded.pkgs { + if pkg.err != nil { + if sumErr := (*ImportMissingSumError)(nil); errors.As(pkg.err, &sumErr) { + if importer := pkg.stack; importer != nil { + sumErr.importer = importer.path + sumErr.importerVersion = importer.mod.Version + sumErr.importerIsTest = importer.testOf != nil + } + } + silence := opts.SilenceErrors + if stdErr := (*ImportMissingError)(nil); errors.As(pkg.err, &stdErr) && + stdErr.isStd && opts.SilenceMissingStdImports { + silence = true + } -// checkMultiplePaths verifies that a given module path is used as itself -// or as a replacement for another module, but not both at the same time. -// -// (See https://golang.org/issue/26607 and https://golang.org/issue/34650.) -func checkMultiplePaths() { - firstPath := make(map[module.Version]string, len(buildList)) - for _, mod := range buildList { - src := mod - if rep := Replacement(mod); rep.Path != "" { - src = rep + if !silence { + if opts.AllowErrors { + fmt.Fprintf(os.Stderr, "%s: %v\n", pkg.stackText(), pkg.err) + } else { + base.Errorf("%s: %v", pkg.stackText(), pkg.err) + } + } } - if prev, ok := firstPath[src]; !ok { - firstPath[src] = mod.Path - } else if prev != mod.Path { - base.Errorf("go: %s@%s used for two different module paths (%s and %s)", src.Path, src.Version, prev, mod.Path) + if !pkg.isTest() { + loadedPackages = append(loadedPackages, pkg.path) + } + } + if !opts.SilenceErrors { + // Also list errors in matching patterns (such as directory permission + // errors for wildcard patterns). + for _, match := range matches { + for _, err := range match.Errs { + if opts.AllowErrors { + fmt.Fprintf(os.Stderr, "%v\n", err) + } else { + base.Errorf("%v", err) + } + } } } base.ExitIfErrors() + + if !opts.SilenceUnmatchedWarnings { + search.WarnUnmatched(matches) + } + + // Success! Update go.mod (if needed) and return the results. + WriteGoMod() + sort.Strings(loadedPackages) + return matches, loadedPackages } // matchLocalDirs is like m.MatchDirs, but tries to avoid scanning directories @@ -217,11 +382,11 @@ func resolveLocalPackage(dir string) (string, error) { // If the named directory does not exist or contains no Go files, // the package does not exist. // Other errors may affect package loading, but not resolution. - if _, err := os.Stat(absDir); err != nil { + if _, err := fsys.Stat(absDir); err != nil { if os.IsNotExist(err) { // Canonicalize OS-specific errors to errDirectoryNotFound so that error // messages will be easier for users to search for. - return "", &os.PathError{Op: "stat", Path: absDir, Err: errDirectoryNotFound} + return "", &fs.PathError{Op: "stat", Path: absDir, Err: errDirectoryNotFound} } return "", err } @@ -307,7 +472,7 @@ var ( // pathInModuleCache returns the import path of the directory dir, // if dir is in the module cache copy of a module in our build list. func pathInModuleCache(dir string) string { - for _, m := range buildList[1:] { + tryMod := func(m module.Version) (string, bool) { var root string var err error if repl := Replacement(m); repl.Path != "" && repl.Version == "" { @@ -321,13 +486,26 @@ func pathInModuleCache(dir string) string { root, err = modfetch.DownloadDir(m) } if err != nil { - continue + return "", false } - if sub := search.InDir(dir, root); sub != "" { - sub = filepath.ToSlash(sub) - if !strings.Contains(sub, "/vendor/") && !strings.HasPrefix(sub, "vendor/") && !strings.Contains(sub, "@") { - return path.Join(m.Path, filepath.ToSlash(sub)) - } + + sub := search.InDir(dir, root) + if sub == "" { + return "", false + } + sub = filepath.ToSlash(sub) + if strings.Contains(sub, "/vendor/") || strings.HasPrefix(sub, "vendor/") || strings.Contains(sub, "@") { + return "", false + } + + return path.Join(m.Path, filepath.ToSlash(sub)), true + } + + for _, m := range buildList[1:] { + if importPath, ok := tryMod(m); ok { + // checkMultiplePaths ensures that a module can be used for at most one + // requirement, so this must be it. + return importPath } } return "" @@ -335,8 +513,8 @@ func pathInModuleCache(dir string) string { // ImportFromFiles adds modules to the build list as needed // to satisfy the imports in the named Go source files. -func ImportFromFiles(gofiles []string) { - InitMod() +func ImportFromFiles(ctx context.Context, gofiles []string) { + LoadModFile(ctx) tags := imports.Tags() imports, testImports, err := imports.ScanFiles(gofiles, tags) @@ -344,12 +522,17 @@ func ImportFromFiles(gofiles []string) { base.Fatalf("go: %v", err) } - loaded = newLoader(tags) - loaded.load(func() []string { - var roots []string - roots = append(roots, imports...) - roots = append(roots, testImports...) - return roots + loaded = loadFromRoots(loaderParams{ + PackageOpts: PackageOpts{ + Tags: tags, + ResolveMissingImports: true, + }, + allClosesOverTests: index.allPatternClosesOverTests(), + listRoots: func() (roots []string) { + roots = append(roots, imports...) + roots = append(roots, testImports...) + return roots + }, }) WriteGoMod() } @@ -357,9 +540,10 @@ func ImportFromFiles(gofiles []string) { // DirImportPath returns the effective import path for dir, // provided it is within the main module, or else returns ".". func DirImportPath(dir string) string { - if modRoot == "" { + if !HasModRoot() { return "." } + LoadModFile(context.TODO()) if !filepath.IsAbs(dir) { dir = filepath.Join(base.Cwd, dir) @@ -380,130 +564,20 @@ func DirImportPath(dir string) string { return "." } -// LoadBuildList loads and returns the build list from go.mod. -// The loading of the build list happens automatically in ImportPaths: -// LoadBuildList need only be called if ImportPaths is not -// (typically in commands that care about the module but -// no particular package). -func LoadBuildList() []module.Version { - InitMod() - ReloadBuildList() - WriteGoMod() - return buildList -} - -func ReloadBuildList() []module.Version { - loaded = newLoader(imports.Tags()) - loaded.load(func() []string { return nil }) - return buildList -} - -// LoadALL returns the set of all packages in the current module -// and their dependencies in any other modules, without filtering -// due to build tags, except "+build ignore". -// It adds modules to the build list as needed to satisfy new imports. -// This set is useful for deciding whether a particular import is needed -// anywhere in a module. -func LoadALL() []string { - return loadAll(true) -} - -// LoadVendor is like LoadALL but only follows test dependencies -// for tests in the main module. Tests in dependency modules are -// ignored completely. -// This set is useful for identifying the which packages to include in a vendor directory. -func LoadVendor() []string { - return loadAll(false) -} - -func loadAll(testAll bool) []string { - InitMod() - - loaded = newLoader(imports.AnyTags()) - loaded.isALL = true - loaded.testAll = testAll - if !testAll { - loaded.testRoots = true - } - all := TargetPackages("...") - loaded.load(func() []string { return all.Pkgs }) - checkMultiplePaths() - WriteGoMod() - - var paths []string - for _, pkg := range loaded.pkgs { - if pkg.err != nil { - base.Errorf("%s: %v", pkg.stackText(), pkg.err) - continue - } - paths = append(paths, pkg.path) - } - for _, err := range all.Errs { - base.Errorf("%v", err) - } - base.ExitIfErrors() - return paths -} - // TargetPackages returns the list of packages in the target (top-level) module // matching pattern, which may be relative to the working directory, under all // build tag settings. -func TargetPackages(pattern string) *search.Match { +func TargetPackages(ctx context.Context, pattern string) *search.Match { // TargetPackages is relative to the main module, so ensure that the main // module is a thing that can contain packages. + LoadModFile(ctx) ModRoot() m := search.NewMatch(pattern) - matchPackages(m, imports.AnyTags(), omitStd, []module.Version{Target}) + matchPackages(ctx, m, imports.AnyTags(), omitStd, []module.Version{Target}) return m } -// BuildList returns the module build list, -// typically constructed by a previous call to -// LoadBuildList or ImportPaths. -// The caller must not modify the returned list. -func BuildList() []module.Version { - return buildList -} - -// SetBuildList sets the module build list. -// The caller is responsible for ensuring that the list is valid. -// SetBuildList does not retain a reference to the original list. -func SetBuildList(list []module.Version) { - buildList = append([]module.Version{}, list...) -} - -// TidyBuildList trims the build list to the minimal requirements needed to -// retain the same versions of all packages from the preceding Load* or -// ImportPaths* call. -func TidyBuildList() { - used := map[module.Version]bool{Target: true} - for _, pkg := range loaded.pkgs { - used[pkg.mod] = true - } - - keep := []module.Version{Target} - var direct []string - for _, m := range buildList[1:] { - if used[m] { - keep = append(keep, m) - if loaded.direct[m.Path] { - direct = append(direct, m.Path) - } - } else if cfg.BuildV { - if _, ok := index.require[m]; ok { - fmt.Fprintf(os.Stderr, "unused %s\n", m.Path) - } - } - } - - min, err := mvs.Req(Target, direct, &mvsReqs{buildList: keep}) - if err != nil { - base.Fatalf("go: %v", err) - } - buildList = append([]module.Version{Target}, min...) -} - // ImportMap returns the actual package import path // for an import path found in source code. // If the given import path does not appear in the source code @@ -558,12 +632,6 @@ func PackageImports(path string) (imports, testImports []string) { return imports, testImports } -// ModuleUsedDirectly reports whether the main module directly imports -// some package in the module with the given path. -func ModuleUsedDirectly(path string) bool { - return loaded.direct[path] -} - // Lookup returns the source directory, import path, and any loading error for // the package at path as imported from the package in parentDir. // Lookup requires that one of the Load functions in this package has already @@ -599,129 +667,193 @@ func Lookup(parentPath string, parentIsStd bool, path string) (dir, realPath str // the required packages for a particular build, // checking that the packages are available in the module set, // and updating the module set if needed. -// Loading is an iterative process: try to load all the needed packages, -// but if imports are missing, try to resolve those imports, and repeat. -// -// Although most of the loading state is maintained in the loader struct, -// one key piece - the build list - is a global, so that it can be modified -// separate from the loading operation, such as during "go get" -// upgrades/downgrades or in "go mod" operations. -// TODO(rsc): It might be nice to make the loader take and return -// a buildList rather than hard-coding use of the global. type loader struct { - tags map[string]bool // tags for scanDir - testRoots bool // include tests for roots - isALL bool // created with LoadALL - testAll bool // include tests for all packages - forceStdVendor bool // if true, load standard-library dependencies from the vendor subtree + loaderParams + + work *par.Queue // reset on each iteration roots []*loadPkg - pkgs []*loadPkg - work *par.Work // current work queue - pkgCache *par.Cache // map from string to *loadPkg + pkgCache *par.Cache // package path (string) → *loadPkg + pkgs []*loadPkg // transitive closure of loaded packages and tests; populated in buildStacks // computed at end of iterations - direct map[string]bool // imported directly by main module - goVersion map[string]string // go version recorded in each module + direct map[string]bool // imported directly by main module } -// LoadTests controls whether the loaders load tests of the root packages. -var LoadTests bool +// loaderParams configure the packages loaded by, and the properties reported +// by, a loader instance. +type loaderParams struct { + PackageOpts -func newLoader(tags map[string]bool) *loader { - ld := new(loader) - ld.tags = tags - ld.testRoots = LoadTests + allClosesOverTests bool // Does the "all" pattern include the transitive closure of tests of packages in "all"? + allPatternIsRoot bool // Is the "all" pattern an additional root? - // Inside the "std" and "cmd" modules, we prefer to use the vendor directory - // unless the command explicitly changes the module graph. - if !targetInGorootSrc || (cfg.CmdName != "get" && !strings.HasPrefix(cfg.CmdName, "mod ")) { - ld.forceStdVendor = true - } - - return ld + listRoots func() []string } func (ld *loader) reset() { + select { + case <-ld.work.Idle(): + default: + panic("loader.reset when not idle") + } + ld.roots = nil - ld.pkgs = nil - ld.work = new(par.Work) ld.pkgCache = new(par.Cache) + ld.pkgs = nil } // A loadPkg records information about a single loaded package. type loadPkg struct { - path string // import path + // Populated at construction time: + path string // import path + testOf *loadPkg + + // Populated at construction time and updated by (*loader).applyPkgFlags: + flags atomicLoadPkgFlags + + // Populated by (*loader).load: mod module.Version // module providing package dir string // directory containing source code - imports []*loadPkg // packages imported by this one err error // error loading package - stack *loadPkg // package importing this one in minimal import stack for this pkg - test *loadPkg // package with test imports, if we need test - testOf *loadPkg + imports []*loadPkg // packages imported by this one testImports []string // test-only imports, saved for use by pkg.test. + inStd bool + + // Populated by (*loader).pkgTest: + testOnce sync.Once + test *loadPkg + + // Populated by postprocessing in (*loader).buildStacks: + stack *loadPkg // package importing this one in minimal import stack for this pkg +} + +// loadPkgFlags is a set of flags tracking metadata about a package. +type loadPkgFlags int8 + +const ( + // pkgInAll indicates that the package is in the "all" package pattern, + // regardless of whether we are loading the "all" package pattern. + // + // When the pkgInAll flag and pkgImportsLoaded flags are both set, the caller + // who set the last of those flags must propagate the pkgInAll marking to all + // of the imports of the marked package. + // + // A test is marked with pkgInAll if that test would promote the packages it + // imports to be in "all" (such as when the test is itself within the main + // module, or when ld.allClosesOverTests is true). + pkgInAll loadPkgFlags = 1 << iota + + // pkgIsRoot indicates that the package matches one of the root package + // patterns requested by the caller. + // + // If LoadTests is set, then when pkgIsRoot and pkgImportsLoaded are both set, + // the caller who set the last of those flags must populate a test for the + // package (in the pkg.test field). + // + // If the "all" pattern is included as a root, then non-test packages in "all" + // are also roots (and must be marked pkgIsRoot). + pkgIsRoot + + // pkgImportsLoaded indicates that the imports and testImports fields of a + // loadPkg have been populated. + pkgImportsLoaded +) + +// has reports whether all of the flags in cond are set in f. +func (f loadPkgFlags) has(cond loadPkgFlags) bool { + return f&cond == cond +} + +// An atomicLoadPkgFlags stores a loadPkgFlags for which individual flags can be +// added atomically. +type atomicLoadPkgFlags struct { + bits int32 +} + +// update sets the given flags in af (in addition to any flags already set). +// +// update returns the previous flag state so that the caller may determine which +// flags were newly-set. +func (af *atomicLoadPkgFlags) update(flags loadPkgFlags) (old loadPkgFlags) { + for { + old := atomic.LoadInt32(&af.bits) + new := old | int32(flags) + if new == old || atomic.CompareAndSwapInt32(&af.bits, old, new) { + return loadPkgFlags(old) + } + } +} + +// has reports whether all of the flags in cond are set in af. +func (af *atomicLoadPkgFlags) has(cond loadPkgFlags) bool { + return loadPkgFlags(atomic.LoadInt32(&af.bits))&cond == cond +} + +// isTest reports whether pkg is a test of another package. +func (pkg *loadPkg) isTest() bool { + return pkg.testOf != nil } var errMissing = errors.New("cannot find package") -// load attempts to load the build graph needed to process a set of root packages. -// The set of root packages is defined by the addRoots function, -// which must call add(path) with the import path of each root package. -func (ld *loader) load(roots func() []string) { +// loadFromRoots attempts to load the build graph needed to process a set of +// root packages and their dependencies. +// +// The set of root packages is returned by the params.listRoots function, and +// expanded to the full set of packages by tracing imports (and possibly tests) +// as needed. +func loadFromRoots(params loaderParams) *loader { + ld := &loader{ + loaderParams: params, + work: par.NewQueue(runtime.GOMAXPROCS(0)), + } + var err error - reqs := Reqs() + reqs := &mvsReqs{buildList: buildList} buildList, err = mvs.BuildList(Target, reqs) if err != nil { base.Fatalf("go: %v", err) } - added := make(map[string]bool) + addedModuleFor := make(map[string]bool) for { ld.reset() - if roots != nil { - // Note: the returned roots can change on each iteration, - // since the expansion of package patterns depends on the - // build list we're using. - for _, path := range roots() { - ld.work.Add(ld.pkg(path, true)) + + // Load the root packages and their imports. + // Note: the returned roots can change on each iteration, + // since the expansion of package patterns depends on the + // build list we're using. + inRoots := map[*loadPkg]bool{} + for _, path := range ld.listRoots() { + root := ld.pkg(path, pkgIsRoot) + if !inRoots[root] { + ld.roots = append(ld.roots, root) + inRoots[root] = true } } - ld.work.Do(10, ld.doPkg) + + // ld.pkg adds imported packages to the work queue and calls applyPkgFlags, + // which adds tests (and test dependencies) as needed. + // + // When all of the work in the queue has completed, we'll know that the + // transitive closure of dependencies has been loaded. + <-ld.work.Idle() + ld.buildStacks() - numAdded := 0 - haveMod := make(map[module.Version]bool) - for _, m := range buildList { - haveMod[m] = true - } - modAddedBy := make(map[module.Version]*loadPkg) - for _, pkg := range ld.pkgs { - if err, ok := pkg.err.(*ImportMissingError); ok && err.Module.Path != "" { - if err.newMissingVersion != "" { - base.Fatalf("go: %s: package provided by %s at latest version %s but not at required version %s", pkg.stackText(), err.Module.Path, err.Module.Version, err.newMissingVersion) - } - fmt.Fprintf(os.Stderr, "go: found %s in %s %s\n", pkg.path, err.Module.Path, err.Module.Version) - if added[pkg.path] { - base.Fatalf("go: %s: looping trying to add package", pkg.stackText()) - } - added[pkg.path] = true - numAdded++ - if !haveMod[err.Module] { - haveMod[err.Module] = true - modAddedBy[err.Module] = pkg - buildList = append(buildList, err.Module) - } - continue - } - // Leave other errors for Import or load.Packages to report. + + if !ld.ResolveMissingImports || (!HasModRoot() && !allowMissingModuleImports) { + // We've loaded as much as we can without resolving missing imports. + break } - base.ExitIfErrors() - if numAdded == 0 { + modAddedBy := ld.resolveMissingImports(addedModuleFor) + if len(modAddedBy) == 0 { break } // Recompute buildList with all our additions. - reqs = Reqs() + reqs = &mvsReqs{buildList: buildList} buildList, err = mvs.BuildList(Target, reqs) if err != nil { // If an error was found in a newly added module, report the package @@ -742,105 +874,303 @@ func (ld *loader) load(roots func() []string) { for _, pkg := range ld.pkgs { if pkg.mod == Target { for _, dep := range pkg.imports { - if dep.mod.Path != "" { + if dep.mod.Path != "" && dep.mod.Path != Target.Path && index != nil { + _, explicit := index.require[dep.mod] + if allowWriteGoMod && cfg.BuildMod == "readonly" && !explicit { + // TODO(#40775): attach error to package instead of using + // base.Errorf. Ideally, 'go list' should not fail because of this, + // but today, LoadPackages calls WriteGoMod unconditionally, which + // would fail with a less clear message. + base.Errorf("go: %[1]s: package %[2]s imported from implicitly required module; to add missing requirements, run:\n\tgo get %[2]s@%[3]s", pkg.path, dep.path, dep.mod.Version) + } ld.direct[dep.mod.Path] = true } } } } + base.ExitIfErrors() - // Add Go versions, computed during walk. - ld.goVersion = make(map[string]string) - for _, m := range buildList { - v, _ := reqs.(*mvsReqs).versions.Load(m) - ld.goVersion[m.Path], _ = v.(string) - } - - // Mix in direct markings (really, lack of indirect markings) - // from go.mod, unless we scanned the whole module - // and can therefore be sure we know better than go.mod. - if !ld.isALL && modFile != nil { + // If we didn't scan all of the imports from the main module, or didn't use + // imports.AnyTags, then we didn't necessarily load every package that + // contributes “direct” imports — so we can't safely mark existing + // dependencies as indirect-only. + // Conservatively mark those dependencies as direct. + if modFile != nil && (!ld.allPatternIsRoot || !reflect.DeepEqual(ld.Tags, imports.AnyTags())) { for _, r := range modFile.Require { if !r.Indirect { ld.direct[r.Mod.Path] = true } } } + + return ld } -// pkg returns the *loadPkg for path, creating and queuing it if needed. -// If the package should be tested, its test is created but not queued -// (the test is queued after processing pkg). -// If isRoot is true, the pkg is being queued as one of the roots of the work graph. -func (ld *loader) pkg(path string, isRoot bool) *loadPkg { - return ld.pkgCache.Do(path, func() interface{} { - pkg := &loadPkg{ - path: path, +// resolveMissingImports adds module dependencies to the global build list +// in order to resolve missing packages from pkgs. +// +// The newly-resolved packages are added to the addedModuleFor map, and +// resolveMissingImports returns a map from each newly-added module version to +// the first package for which that module was added. +func (ld *loader) resolveMissingImports(addedModuleFor map[string]bool) (modAddedBy map[module.Version]*loadPkg) { + var needPkgs []*loadPkg + for _, pkg := range ld.pkgs { + if pkg.err == nil { + continue } - if ld.testRoots && isRoot || ld.testAll { - test := &loadPkg{ - path: path, - testOf: pkg, - } - pkg.test = test + if pkg.isTest() { + // If we are missing a test, we are also missing its non-test version, and + // we should only add the missing import once. + continue } - if isRoot { - ld.roots = append(ld.roots, pkg) + if !errors.As(pkg.err, new(*ImportMissingError)) { + // Leave other errors for Import or load.Packages to report. + continue } - ld.work.Add(pkg) + + needPkgs = append(needPkgs, pkg) + + pkg := pkg + ld.work.Add(func() { + pkg.mod, pkg.err = queryImport(context.TODO(), pkg.path) + }) + } + <-ld.work.Idle() + + modAddedBy = map[module.Version]*loadPkg{} + for _, pkg := range needPkgs { + if pkg.err != nil { + continue + } + + fmt.Fprintf(os.Stderr, "go: found %s in %s %s\n", pkg.path, pkg.mod.Path, pkg.mod.Version) + if addedModuleFor[pkg.path] { + // TODO(bcmills): This should only be an error if pkg.mod is the same + // version we already tried to add previously. + base.Fatalf("go: %s: looping trying to add package", pkg.stackText()) + } + if modAddedBy[pkg.mod] == nil { + modAddedBy[pkg.mod] = pkg + buildList = append(buildList, pkg.mod) + } + } + + return modAddedBy +} + +// pkg locates the *loadPkg for path, creating and queuing it for loading if +// needed, and updates its state to reflect the given flags. +// +// The imports of the returned *loadPkg will be loaded asynchronously in the +// ld.work queue, and its test (if requested) will also be populated once +// imports have been resolved. When ld.work goes idle, all transitive imports of +// the requested package (and its test, if requested) will have been loaded. +func (ld *loader) pkg(path string, flags loadPkgFlags) *loadPkg { + if flags.has(pkgImportsLoaded) { + panic("internal error: (*loader).pkg called with pkgImportsLoaded flag set") + } + + pkg := ld.pkgCache.Do(path, func() interface{} { + pkg := &loadPkg{ + path: path, + } + ld.applyPkgFlags(pkg, flags) + + ld.work.Add(func() { ld.load(pkg) }) return pkg }).(*loadPkg) + + ld.applyPkgFlags(pkg, flags) + return pkg } -// doPkg processes a package on the work queue. -func (ld *loader) doPkg(item interface{}) { - // TODO: what about replacements? - pkg := item.(*loadPkg) - var imports []string - if pkg.testOf != nil { - pkg.dir = pkg.testOf.dir - pkg.mod = pkg.testOf.mod - imports = pkg.testOf.testImports - } else { - if strings.Contains(pkg.path, "@") { - // Leave for error during load. - return +// applyPkgFlags updates pkg.flags to set the given flags and propagate the +// (transitive) effects of those flags, possibly loading or enqueueing further +// packages as a result. +func (ld *loader) applyPkgFlags(pkg *loadPkg, flags loadPkgFlags) { + if flags == 0 { + return + } + + if flags.has(pkgInAll) && ld.allPatternIsRoot && !pkg.isTest() { + // This package matches a root pattern by virtue of being in "all". + flags |= pkgIsRoot + } + + old := pkg.flags.update(flags) + new := old | flags + if new == old || !new.has(pkgImportsLoaded) { + // We either didn't change the state of pkg, or we don't know anything about + // its dependencies yet. Either way, we can't usefully load its test or + // update its dependencies. + return + } + + if !pkg.isTest() { + // Check whether we should add (or update the flags for) a test for pkg. + // ld.pkgTest is idempotent and extra invocations are inexpensive, + // so it's ok if we call it more than is strictly necessary. + wantTest := false + switch { + case ld.allPatternIsRoot && pkg.mod == Target: + // We are loading the "all" pattern, which includes packages imported by + // tests in the main module. This package is in the main module, so we + // need to identify the imports of its test even if LoadTests is not set. + // + // (We will filter out the extra tests explicitly in computePatternAll.) + wantTest = true + + case ld.allPatternIsRoot && ld.allClosesOverTests && new.has(pkgInAll): + // This variant of the "all" pattern includes imports of tests of every + // package that is itself in "all", and pkg is in "all", so its test is + // also in "all" (as above). + wantTest = true + + case ld.LoadTests && new.has(pkgIsRoot): + // LoadTest explicitly requests tests of “the root packages”. + wantTest = true } - if build.IsLocalImport(pkg.path) || filepath.IsAbs(pkg.path) { - // Leave for error during load. - // (Module mode does not allow local imports.) - return + + if wantTest { + var testFlags loadPkgFlags + if pkg.mod == Target || (ld.allClosesOverTests && new.has(pkgInAll)) { + // Tests of packages in the main module are in "all", in the sense that + // they cause the packages they import to also be in "all". So are tests + // of packages in "all" if "all" closes over test dependencies. + testFlags |= pkgInAll + } + ld.pkgTest(pkg, testFlags) } + } - pkg.mod, pkg.dir, pkg.err = Import(pkg.path) - if pkg.dir == "" { - return + if new.has(pkgInAll) && !old.has(pkgInAll|pkgImportsLoaded) { + // We have just marked pkg with pkgInAll, or we have just loaded its + // imports, or both. Now is the time to propagate pkgInAll to the imports. + for _, dep := range pkg.imports { + ld.applyPkgFlags(dep, pkgInAll) } - var testImports []string + } +} + +// load loads an individual package. +func (ld *loader) load(pkg *loadPkg) { + if strings.Contains(pkg.path, "@") { + // Leave for error during load. + return + } + if build.IsLocalImport(pkg.path) || filepath.IsAbs(pkg.path) { + // Leave for error during load. + // (Module mode does not allow local imports.) + return + } + + if search.IsMetaPackage(pkg.path) { + pkg.err = &invalidImportError{ + importPath: pkg.path, + err: fmt.Errorf("%q is not an importable package; see 'go help packages'", pkg.path), + } + return + } + + pkg.mod, pkg.dir, pkg.err = importFromBuildList(context.TODO(), pkg.path, buildList) + if pkg.dir == "" { + return + } + if pkg.mod == Target { + // Go ahead and mark pkg as in "all". This provides the invariant that a + // package that is *only* imported by other packages in "all" is always + // marked as such before loading its imports. + // + // We don't actually rely on that invariant at the moment, but it may + // improve efficiency somewhat and makes the behavior a bit easier to reason + // about (by reducing churn on the flag bits of dependencies), and costs + // essentially nothing (these atomic flag ops are essentially free compared + // to scanning source code for imports). + ld.applyPkgFlags(pkg, pkgInAll) + } + if ld.AllowPackage != nil { + if err := ld.AllowPackage(context.TODO(), pkg.path, pkg.mod); err != nil { + pkg.err = err + } + } + + pkg.inStd = (search.IsStandardImportPath(pkg.path) && search.InDir(pkg.dir, cfg.GOROOTsrc) != "") + + var imports, testImports []string + + if cfg.BuildContext.Compiler == "gccgo" && pkg.inStd { + // We can't scan standard packages for gccgo. + } else { var err error - imports, testImports, err = scanDir(pkg.dir, ld.tags) + imports, testImports, err = scanDir(pkg.dir, ld.Tags) if err != nil { pkg.err = err return } - if pkg.test != nil { - pkg.testImports = testImports - } } - inStd := (search.IsStandardImportPath(pkg.path) && search.InDir(pkg.dir, cfg.GOROOTsrc) != "") + pkg.imports = make([]*loadPkg, 0, len(imports)) + var importFlags loadPkgFlags + if pkg.flags.has(pkgInAll) { + importFlags = pkgInAll + } for _, path := range imports { - if inStd { + if pkg.inStd { + // Imports from packages in "std" and "cmd" should resolve using + // GOROOT/src/vendor even when "std" is not the main module. path = ld.stdVendor(pkg.path, path) } - pkg.imports = append(pkg.imports, ld.pkg(path, false)) + pkg.imports = append(pkg.imports, ld.pkg(path, importFlags)) } + pkg.testImports = testImports - // Now that pkg.dir, pkg.mod, pkg.testImports are set, we can queue pkg.test. - // TODO: All that's left is creating new imports. Why not just do it now? - if pkg.test != nil { - ld.work.Add(pkg.test) + ld.applyPkgFlags(pkg, pkgImportsLoaded) +} + +// pkgTest locates the test of pkg, creating it if needed, and updates its state +// to reflect the given flags. +// +// pkgTest requires that the imports of pkg have already been loaded (flagged +// with pkgImportsLoaded). +func (ld *loader) pkgTest(pkg *loadPkg, testFlags loadPkgFlags) *loadPkg { + if pkg.isTest() { + panic("pkgTest called on a test package") } + + createdTest := false + pkg.testOnce.Do(func() { + pkg.test = &loadPkg{ + path: pkg.path, + testOf: pkg, + mod: pkg.mod, + dir: pkg.dir, + err: pkg.err, + inStd: pkg.inStd, + } + ld.applyPkgFlags(pkg.test, testFlags) + createdTest = true + }) + + test := pkg.test + if createdTest { + test.imports = make([]*loadPkg, 0, len(pkg.testImports)) + var importFlags loadPkgFlags + if test.flags.has(pkgInAll) { + importFlags = pkgInAll + } + for _, path := range pkg.testImports { + if pkg.inStd { + path = ld.stdVendor(test.path, path) + } + test.imports = append(test.imports, ld.pkg(path, importFlags)) + } + pkg.testImports = nil + ld.applyPkgFlags(test, pkgImportsLoaded) + } else { + ld.applyPkgFlags(test, testFlags) + } + + return test } // stdVendor returns the canonical import path for the package with the given @@ -851,13 +1181,25 @@ func (ld *loader) stdVendor(parentPath, path string) string { } if str.HasPathPrefix(parentPath, "cmd") { - if ld.forceStdVendor || Target.Path != "cmd" { + if Target.Path != "cmd" { vendorPath := pathpkg.Join("cmd", "vendor", path) if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil { return vendorPath } } - } else if ld.forceStdVendor || Target.Path != "std" { + } else if Target.Path != "std" || str.HasPathPrefix(parentPath, "vendor") { + // If we are outside of the 'std' module, resolve imports from within 'std' + // to the vendor directory. + // + // Do the same for importers beginning with the prefix 'vendor/' even if we + // are *inside* of the 'std' module: the 'vendor/' packages that resolve + // globally from GOROOT/src/vendor (and are listed as part of 'go list std') + // are distinct from the real module dependencies, and cannot import + // internal packages from the real module. + // + // (Note that although the 'vendor/' packages match the 'std' *package* + // pattern, they are not part of the std *module*, and do not affect + // 'go mod tidy' and similar module commands when working within std.) vendorPath := pathpkg.Join("vendor", path) if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil { return vendorPath @@ -870,30 +1212,13 @@ func (ld *loader) stdVendor(parentPath, path string) string { // computePatternAll returns the list of packages matching pattern "all", // starting with a list of the import paths for the packages in the main module. -func (ld *loader) computePatternAll(paths []string) []string { - seen := make(map[*loadPkg]bool) - var all []string - var walk func(*loadPkg) - walk = func(pkg *loadPkg) { - if seen[pkg] { - return - } - seen[pkg] = true - if pkg.testOf == nil { +func (ld *loader) computePatternAll() (all []string) { + for _, pkg := range ld.pkgs { + if pkg.flags.has(pkgInAll) && !pkg.isTest() { all = append(all, pkg.path) } - for _, p := range pkg.imports { - walk(p) - } - if p := pkg.test; p != nil { - walk(p) - } - } - for _, path := range paths { - walk(ld.pkg(path, false)) } sort.Strings(all) - return all } @@ -1015,7 +1340,7 @@ func (pkg *loadPkg) why() string { // Why returns the "go mod why" output stanza for the given package, // without the leading # comment. -// The package graph must have been loaded already, usually by LoadALL. +// The package graph must have been loaded already, usually by LoadPackages. // If there is no reason for the package to be in the current build, // Why returns an empty string. func Why(path string) string { diff --git a/cmd/go/_internal_/modload/modfile.go b/cmd/go/_internal_/modload/modfile.go index 257e2c3..40b99aa 100644 --- a/cmd/go/_internal_/modload/modfile.go +++ b/cmd/go/_internal_/modload/modfile.go @@ -5,13 +5,32 @@ package modload import ( + "context" + "errors" + "fmt" + "path/filepath" + "strings" + "sync" + "unicode" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/lockedfile" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/par" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/trace" "golang.org/x/mod/modfile" "golang.org/x/mod/module" + "golang.org/x/mod/semver" ) +// narrowAllVersionV is the Go version (plus leading "v") at which the +// module-module "all" pattern no longer closes over the dependencies of +// tests outside of the main module. +const narrowAllVersionV = "v1.16" +const go116EnableNarrowAll = true + var modFile *modfile.File // A modFileIndex is an index of data corresponding to a modFile @@ -20,9 +39,10 @@ type modFileIndex struct { data []byte dataNeedsFix bool // true if fixVersion applied a change while parsing data module module.Version - goVersion string + goVersionV string // GoVersion with "v" prefix require map[module.Version]requireMeta replace map[module.Version]module.Version + highestReplaced map[string]string // highest replaced version of each module path; empty string for wildcard-only replacements exclude map[module.Version]bool } @@ -33,9 +53,179 @@ type requireMeta struct { indirect bool } -// Allowed reports whether module m is allowed (not excluded) by the main module's go.mod. -func Allowed(m module.Version) bool { - return index == nil || !index.exclude[m] +// CheckAllowed returns an error equivalent to ErrDisallowed if m is excluded by +// the main module's go.mod or retracted by its author. Most version queries use +// this to filter out versions that should not be used. +func CheckAllowed(ctx context.Context, m module.Version) error { + if err := CheckExclusions(ctx, m); err != nil { + return err + } + if err := CheckRetractions(ctx, m); err != nil { + return err + } + return nil +} + +// ErrDisallowed is returned by version predicates passed to Query and similar +// functions to indicate that a version should not be considered. +var ErrDisallowed = errors.New("disallowed module version") + +// CheckExclusions returns an error equivalent to ErrDisallowed if module m is +// excluded by the main module's go.mod file. +func CheckExclusions(ctx context.Context, m module.Version) error { + if index != nil && index.exclude[m] { + return module.VersionError(m, errExcluded) + } + return nil +} + +var errExcluded = &excludedError{} + +type excludedError struct{} + +func (e *excludedError) Error() string { return "excluded by go.mod" } +func (e *excludedError) Is(err error) bool { return err == ErrDisallowed } + +// CheckRetractions returns an error if module m has been retracted by +// its author. +func CheckRetractions(ctx context.Context, m module.Version) error { + if m.Version == "" { + // Main module, standard library, or file replacement module. + // Cannot be retracted. + return nil + } + + // Look up retraction information from the latest available version of + // the module. Cache retraction information so we don't parse the go.mod + // file repeatedly. + type entry struct { + retract []retraction + err error + } + path := m.Path + e := retractCache.Do(path, func() (v interface{}) { + ctx, span := trace.StartSpan(ctx, "checkRetractions "+path) + defer span.Done() + + if repl := Replacement(module.Version{Path: m.Path}); repl.Path != "" { + // All versions of the module were replaced with a local directory. + // Don't load retractions. + return &entry{nil, nil} + } + + // Find the latest version of the module. + // Ignore exclusions from the main module's go.mod. + const ignoreSelected = "" + var allowAll AllowedFunc + rev, err := Query(ctx, path, "latest", ignoreSelected, allowAll) + if err != nil { + return &entry{nil, err} + } + + // Load go.mod for that version. + // If the version is replaced, we'll load retractions from the replacement. + // + // If there's an error loading the go.mod, we'll return it here. + // These errors should generally be ignored by callers of checkRetractions, + // since they happen frequently when we're offline. These errors are not + // equivalent to ErrDisallowed, so they may be distinguished from + // retraction errors. + // + // We load the raw file here: the go.mod file may have a different module + // path that we expect if the module or its repository was renamed. + // We still want to apply retractions to other aliases of the module. + rm := module.Version{Path: path, Version: rev.Version} + if repl := Replacement(rm); repl.Path != "" { + rm = repl + } + summary, err := rawGoModSummary(rm) + if err != nil { + return &entry{nil, err} + } + return &entry{summary.retract, nil} + }).(*entry) + + if err := e.err; err != nil { + // Attribute the error to the version being checked, not the version from + // which the retractions were to be loaded. + var mErr *module.ModuleError + if errors.As(err, &mErr) { + err = mErr.Err + } + return &retractionLoadingError{m: m, err: err} + } + + var rationale []string + isRetracted := false + for _, r := range e.retract { + if semver.Compare(r.Low, m.Version) <= 0 && semver.Compare(m.Version, r.High) <= 0 { + isRetracted = true + if r.Rationale != "" { + rationale = append(rationale, r.Rationale) + } + } + } + if isRetracted { + return module.VersionError(m, &ModuleRetractedError{Rationale: rationale}) + } + return nil +} + +var retractCache par.Cache + +type ModuleRetractedError struct { + Rationale []string +} + +func (e *ModuleRetractedError) Error() string { + msg := "retracted by module author" + if len(e.Rationale) > 0 { + // This is meant to be a short error printed on a terminal, so just + // print the first rationale. + msg += ": " + ShortRetractionRationale(e.Rationale[0]) + } + return msg +} + +func (e *ModuleRetractedError) Is(err error) bool { + return err == ErrDisallowed +} + +type retractionLoadingError struct { + m module.Version + err error +} + +func (e *retractionLoadingError) Error() string { + return fmt.Sprintf("loading module retractions for %v: %v", e.m, e.err) +} + +func (e *retractionLoadingError) Unwrap() error { + return e.err +} + +// ShortRetractionRationale returns a retraction rationale string that is safe +// to print in a terminal. It returns hard-coded strings if the rationale +// is empty, too long, or contains non-printable characters. +func ShortRetractionRationale(rationale string) string { + const maxRationaleBytes = 500 + if i := strings.Index(rationale, "\n"); i >= 0 { + rationale = rationale[:i] + } + rationale = strings.TrimSpace(rationale) + if rationale == "" { + return "retracted by module author" + } + if len(rationale) > maxRationaleBytes { + return "(rationale omitted: too long)" + } + for _, r := range rationale { + if !unicode.IsGraphic(r) && !unicode.IsSpace(r) { + return "(rationale omitted: contains non-printable characters)" + } + } + // NOTE: the go.mod parser rejects invalid UTF-8, so we don't check that here. + return rationale } // Replacement returns the replacement for mod, if any, from go.mod. @@ -66,9 +256,11 @@ func indexModFile(data []byte, modFile *modfile.File, needsFix bool) *modFileInd i.module = modFile.Module.Mod } - i.goVersion = "" + i.goVersionV = "" if modFile.Go != nil { - i.goVersion = modFile.Go.Version + // We're going to use the semver package to compare Go versions, so go ahead + // and add the "v" prefix it expects once instead of every time. + i.goVersionV = "v" + modFile.Go.Version } i.require = make(map[module.Version]requireMeta, len(modFile.Require)) @@ -84,6 +276,14 @@ func indexModFile(data []byte, modFile *modfile.File, needsFix bool) *modFileInd i.replace[r.Old] = r.New } + i.highestReplaced = make(map[string]string) + for _, r := range modFile.Replace { + v, ok := i.highestReplaced[r.Old.Path] + if !ok || semver.Compare(r.Old.Version, v) > 0 { + i.highestReplaced[r.Old.Path] = r.Old.Version + } + } + i.exclude = make(map[module.Version]bool, len(modFile.Exclude)) for _, x := range modFile.Exclude { i.exclude[x.Mod] = true @@ -92,6 +292,23 @@ func indexModFile(data []byte, modFile *modfile.File, needsFix bool) *modFileInd return i } +// allPatternClosesOverTests reports whether the "all" pattern includes +// dependencies of tests outside the main module (as in Go 1.11–1.15). +// (Otherwise — as in Go 1.16+ — the "all" pattern includes only the packages +// transitively *imported by* the packages and tests in the main module.) +func (i *modFileIndex) allPatternClosesOverTests() bool { + if !go116EnableNarrowAll { + return true + } + if i != nil && semver.Compare(i.goVersionV, narrowAllVersionV) < 0 { + // The module explicitly predates the change in "all" for lazy loading, so + // continue to use the older interpretation. (If i == nil, we not in any + // module at all and should use the latest semantics.) + return true + } + return false +} + // modFileIsDirty reports whether the go.mod file differs meaningfully // from what was indexed. // If modFile has been changed (even cosmetically) since it was first read, @@ -114,11 +331,11 @@ func (i *modFileIndex) modFileIsDirty(modFile *modfile.File) bool { } if modFile.Go == nil { - if i.goVersion != "" { + if i.goVersionV != "" { return true } - } else if modFile.Go.Version != i.goVersion { - if i.goVersion == "" && cfg.BuildMod == "readonly" { + } else if "v"+modFile.Go.Version != i.goVersionV { + if i.goVersionV == "" && cfg.BuildMod == "readonly" { // go.mod files did not always require a 'go' version, so do not error out // if one is missing — we may be inside an older module in the module // cache, and should bias toward providing useful behavior. @@ -162,3 +379,213 @@ func (i *modFileIndex) modFileIsDirty(modFile *modfile.File) bool { return false } + +// rawGoVersion records the Go version parsed from each module's go.mod file. +// +// If a module is replaced, the version of the replacement is keyed by the +// replacement module.Version, not the version being replaced. +var rawGoVersion sync.Map // map[module.Version]string + +// A modFileSummary is a summary of a go.mod file for which we do not need to +// retain complete information — for example, the go.mod file of a dependency +// module. +type modFileSummary struct { + module module.Version + goVersionV string // GoVersion with "v" prefix + require []module.Version + retract []retraction +} + +// A retraction consists of a retracted version interval and rationale. +// retraction is like modfile.Retract, but it doesn't point to the syntax tree. +type retraction struct { + modfile.VersionInterval + Rationale string +} + +// goModSummary returns a summary of the go.mod file for module m, +// taking into account any replacements for m, exclusions of its dependencies, +// and/or vendoring. +// +// m must be a version in the module graph, reachable from the Target module. +// In readonly mode, the go.sum file must contain an entry for m's go.mod file +// (or its replacement). goModSummary must not be called for the Target module +// itself, as its requirements may change. Use rawGoModSummary for other +// module versions. +// +// The caller must not modify the returned summary. +func goModSummary(m module.Version) (*modFileSummary, error) { + if m == Target { + panic("internal error: goModSummary called on the Target module") + } + + if cfg.BuildMod == "vendor" { + summary := &modFileSummary{ + module: module.Version{Path: m.Path}, + } + if vendorVersion[m.Path] != m.Version { + // This module is not vendored, so packages cannot be loaded from it and + // it cannot be relevant to the build. + return summary, nil + } + + // For every module other than the target, + // return the full list of modules from modules.txt. + readVendorList() + + // TODO(#36876): Load the "go" version from vendor/modules.txt and store it + // in rawGoVersion with the appropriate key. + + // We don't know what versions the vendored module actually relies on, + // so assume that it requires everything. + summary.require = vendorList + return summary, nil + } + + actual := Replacement(m) + if actual.Path == "" { + actual = m + } + if HasModRoot() && cfg.BuildMod == "readonly" && actual.Version != "" { + key := module.Version{Path: actual.Path, Version: actual.Version + "/go.mod"} + if !modfetch.HaveSum(key) { + suggestion := fmt.Sprintf("; to add it:\n\tgo mod download %s", m.Path) + return nil, module.VersionError(actual, &sumMissingError{suggestion: suggestion}) + } + } + summary, err := rawGoModSummary(actual) + if err != nil { + return nil, err + } + + if actual.Version == "" { + // The actual module is a filesystem-local replacement, for which we have + // unfortunately not enforced any sort of invariants about module lines or + // matching module paths. Anything goes. + // + // TODO(bcmills): Remove this special-case, update tests, and add a + // release note. + } else { + if summary.module.Path == "" { + return nil, module.VersionError(actual, errors.New("parsing go.mod: missing module line")) + } + + // In theory we should only allow mpath to be unequal to m.Path here if the + // version that we fetched lacks an explicit go.mod file: if the go.mod file + // is explicit, then it should match exactly (to ensure that imports of other + // packages within the module are interpreted correctly). Unfortunately, we + // can't determine that information from the module proxy protocol: we'll have + // to leave that validation for when we load actual packages from within the + // module. + if mpath := summary.module.Path; mpath != m.Path && mpath != actual.Path { + return nil, module.VersionError(actual, fmt.Errorf(`parsing go.mod: + module declares its path as: %s + but was required as: %s`, mpath, m.Path)) + } + } + + if index != nil && len(index.exclude) > 0 { + // Drop any requirements on excluded versions. + // Don't modify the cached summary though, since we might need the raw + // summary separately. + haveExcludedReqs := false + for _, r := range summary.require { + if index.exclude[r] { + haveExcludedReqs = true + break + } + } + if haveExcludedReqs { + s := new(modFileSummary) + *s = *summary + s.require = make([]module.Version, 0, len(summary.require)) + for _, r := range summary.require { + if !index.exclude[r] { + s.require = append(s.require, r) + } + } + summary = s + } + } + return summary, nil +} + +// rawGoModSummary returns a new summary of the go.mod file for module m, +// ignoring all replacements that may apply to m and excludes that may apply to +// its dependencies. +// +// rawGoModSummary cannot be used on the Target module. +func rawGoModSummary(m module.Version) (*modFileSummary, error) { + if m == Target { + panic("internal error: rawGoModSummary called on the Target module") + } + + type cached struct { + summary *modFileSummary + err error + } + c := rawGoModSummaryCache.Do(m, func() interface{} { + summary := new(modFileSummary) + var f *modfile.File + if m.Version == "" { + // m is a replacement module with only a file path. + dir := m.Path + if !filepath.IsAbs(dir) { + dir = filepath.Join(ModRoot(), dir) + } + gomod := filepath.Join(dir, "go.mod") + + data, err := lockedfile.Read(gomod) + if err != nil { + return cached{nil, module.VersionError(m, fmt.Errorf("reading %s: %v", base.ShortPath(gomod), err))} + } + f, err = modfile.ParseLax(gomod, data, nil) + if err != nil { + return cached{nil, module.VersionError(m, fmt.Errorf("parsing %s: %v", base.ShortPath(gomod), err))} + } + } else { + if !semver.IsValid(m.Version) { + // Disallow the broader queries supported by fetch.Lookup. + base.Fatalf("go: internal error: %s@%s: unexpected invalid semantic version", m.Path, m.Version) + } + + data, err := modfetch.GoMod(m.Path, m.Version) + if err != nil { + return cached{nil, err} + } + f, err = modfile.ParseLax("go.mod", data, nil) + if err != nil { + return cached{nil, module.VersionError(m, fmt.Errorf("parsing go.mod: %v", err))} + } + } + + if f.Module != nil { + summary.module = f.Module.Mod + } + if f.Go != nil && f.Go.Version != "" { + rawGoVersion.LoadOrStore(m, f.Go.Version) + summary.goVersionV = "v" + f.Go.Version + } + if len(f.Require) > 0 { + summary.require = make([]module.Version, 0, len(f.Require)) + for _, req := range f.Require { + summary.require = append(summary.require, req.Mod) + } + } + if len(f.Retract) > 0 { + summary.retract = make([]retraction, 0, len(f.Retract)) + for _, ret := range f.Retract { + summary.retract = append(summary.retract, retraction{ + VersionInterval: ret.VersionInterval, + Rationale: ret.Rationale, + }) + } + } + + return cached{summary, nil} + }).(cached) + + return c.summary, c.err +} + +var rawGoModSummaryCache par.Cache // module.Version → rawGoModSummary result diff --git a/cmd/go/_internal_/modload/mvs.go b/cmd/go/_internal_/modload/mvs.go index 8899860..17d8dd9 100644 --- a/cmd/go/_internal_/modload/mvs.go +++ b/cmd/go/_internal_/modload/mvs.go @@ -5,21 +5,13 @@ package modload import ( + "context" "errors" - "fmt" "os" - "path/filepath" "sort" - "sync" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/lockedfile" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/mvs" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/par" - "golang.org/x/mod/modfile" "golang.org/x/mod/module" "golang.org/x/mod/semver" ) @@ -27,134 +19,25 @@ import ( // mvsReqs implements mvs.Reqs for module semantic versions, // with any exclusions or replacements applied internally. type mvsReqs struct { - buildList []module.Version - cache par.Cache - versions sync.Map -} - -// Reqs returns the current module requirement graph. -// Future calls to SetBuildList do not affect the operation -// of the returned Reqs. -func Reqs() mvs.Reqs { - r := &mvsReqs{ - buildList: buildList, - } - return r + buildList []module.Version } func (r *mvsReqs) Required(mod module.Version) ([]module.Version, error) { - type cached struct { - list []module.Version - err error - } - - c := r.cache.Do(mod, func() interface{} { - list, err := r.required(mod) - if err != nil { - return cached{nil, err} - } - for i, mv := range list { - if index != nil { - for index.exclude[mv] { - mv1, err := r.next(mv) - if err != nil { - return cached{nil, err} - } - if mv1.Version == "none" { - return cached{nil, fmt.Errorf("%s(%s) depends on excluded %s(%s) with no newer version available", mod.Path, mod.Version, mv.Path, mv.Version)} - } - mv = mv1 - } - } - list[i] = mv - } - - return cached{list, nil} - }).(cached) - - return c.list, c.err -} - -func (r *mvsReqs) modFileToList(f *modfile.File) []module.Version { - list := make([]module.Version, 0, len(f.Require)) - for _, r := range f.Require { - list = append(list, r.Mod) - } - return list -} - -// required returns a unique copy of the requirements of mod. -func (r *mvsReqs) required(mod module.Version) ([]module.Version, error) { if mod == Target { - if modFile != nil && modFile.Go != nil { - r.versions.LoadOrStore(mod, modFile.Go.Version) - } - return append([]module.Version(nil), r.buildList[1:]...), nil - } - - if cfg.BuildMod == "vendor" { - // For every module other than the target, - // return the full list of modules from modules.txt. - readVendorList() - return append([]module.Version(nil), vendorList...), nil - } - - origPath := mod.Path - if repl := Replacement(mod); repl.Path != "" { - if repl.Version == "" { - // TODO: need to slip the new version into the tags list etc. - dir := repl.Path - if !filepath.IsAbs(dir) { - dir = filepath.Join(ModRoot(), dir) - } - gomod := filepath.Join(dir, "go.mod") - data, err := lockedfile.Read(gomod) - if err != nil { - return nil, fmt.Errorf("parsing %s: %v", base.ShortPath(gomod), err) - } - f, err := modfile.ParseLax(gomod, data, nil) - if err != nil { - return nil, fmt.Errorf("parsing %s: %v", base.ShortPath(gomod), err) - } - if f.Go != nil { - r.versions.LoadOrStore(mod, f.Go.Version) - } - return r.modFileToList(f), nil - } - mod = repl + // Use the build list as it existed when r was constructed, not the current + // global build list. + return r.buildList[1:], nil } if mod.Version == "none" { return nil, nil } - if !semver.IsValid(mod.Version) { - // Disallow the broader queries supported by fetch.Lookup. - base.Fatalf("go: internal error: %s@%s: unexpected invalid semantic version", mod.Path, mod.Version) - } - - data, err := modfetch.GoMod(mod.Path, mod.Version) + summary, err := goModSummary(mod) if err != nil { return nil, err } - f, err := modfile.ParseLax("go.mod", data, nil) - if err != nil { - return nil, module.VersionError(mod, fmt.Errorf("parsing go.mod: %v", err)) - } - - if f.Module == nil { - return nil, module.VersionError(mod, errors.New("parsing go.mod: missing module line")) - } - if mpath := f.Module.Mod.Path; mpath != origPath && mpath != mod.Path { - return nil, module.VersionError(mod, fmt.Errorf(`parsing go.mod: - module declares its path as: %s - but was required as: %s`, mpath, origPath)) - } - if f.Go != nil { - r.versions.LoadOrStore(mod, f.Go.Version) - } - - return r.modFileToList(f), nil + return summary.require, nil } // Max returns the maximum of v1 and v2 according to semver.Compare. @@ -164,7 +47,7 @@ func (r *mvsReqs) required(mod module.Version) ([]module.Version, error) { // be chosen over other versions of the same module in the module dependency // graph. func (*mvsReqs) Max(v1, v2 string) string { - if v1 != "" && semver.Compare(v1, v2) == -1 { + if v1 != "" && (v2 == "" || semver.Compare(v1, v2) == -1) { return v2 } return v1 @@ -176,25 +59,50 @@ func (*mvsReqs) Upgrade(m module.Version) (module.Version, error) { return m, nil } -func versions(path string) ([]string, error) { +func versions(ctx context.Context, path string, allowed AllowedFunc) ([]string, error) { // Note: modfetch.Lookup and repo.Versions are cached, // so there's no need for us to add extra caching here. var versions []string err := modfetch.TryProxies(func(proxy string) error { - repo, err := modfetch.Lookup(proxy, path) - if err == nil { - versions, err = repo.Versions("") + repo, err := lookupRepo(proxy, path) + if err != nil { + return err + } + allVersions, err := repo.Versions("") + if err != nil { + return err + } + allowedVersions := make([]string, 0, len(allVersions)) + for _, v := range allVersions { + if err := allowed(ctx, module.Version{Path: path, Version: v}); err == nil { + allowedVersions = append(allowedVersions, v) + } else if !errors.Is(err, ErrDisallowed) { + return err + } } - return err + versions = allowedVersions + return nil }) return versions, err } // Previous returns the tagged version of m.Path immediately prior to // m.Version, or version "none" if no prior version is tagged. +// +// Since the version of Target is not found in the version list, +// it has no previous version. func (*mvsReqs) Previous(m module.Version) (module.Version, error) { - list, err := versions(m.Path) + // TODO(golang.org/issue/38714): thread tracing context through MVS. + + if m == Target { + return module.Version{Path: m.Path, Version: "none"}, nil + } + + list, err := versions(context.TODO(), m.Path, CheckAllowed) if err != nil { + if errors.Is(err, os.ErrNotExist) { + return module.Version{Path: m.Path, Version: "none"}, nil + } return module.Version{}, err } i := sort.Search(len(list), func(i int) bool { return semver.Compare(list[i], m.Version) >= 0 }) @@ -203,57 +111,3 @@ func (*mvsReqs) Previous(m module.Version) (module.Version, error) { } return module.Version{Path: m.Path, Version: "none"}, nil } - -// next returns the next version of m.Path after m.Version. -// It is only used by the exclusion processing in the Required method, -// not called directly by MVS. -func (*mvsReqs) next(m module.Version) (module.Version, error) { - list, err := versions(m.Path) - if err != nil { - return module.Version{}, err - } - i := sort.Search(len(list), func(i int) bool { return semver.Compare(list[i], m.Version) > 0 }) - if i < len(list) { - return module.Version{Path: m.Path, Version: list[i]}, nil - } - return module.Version{Path: m.Path, Version: "none"}, nil -} - -// fetch downloads the given module (or its replacement) -// and returns its location. -// -// The isLocal return value reports whether the replacement, -// if any, is local to the filesystem. -func fetch(mod module.Version) (dir string, isLocal bool, err error) { - if mod == Target { - return ModRoot(), true, nil - } - if r := Replacement(mod); r.Path != "" { - if r.Version == "" { - dir = r.Path - if !filepath.IsAbs(dir) { - dir = filepath.Join(ModRoot(), dir) - } - // Ensure that the replacement directory actually exists: - // dirInModule does not report errors for missing modules, - // so if we don't report the error now, later failures will be - // very mysterious. - if _, err := os.Stat(dir); err != nil { - if os.IsNotExist(err) { - // Semantically the module version itself “exists” — we just don't - // have its source code. Remove the equivalence to os.ErrNotExist, - // and make the message more concise while we're at it. - err = fmt.Errorf("replacement directory %s does not exist", r.Path) - } else { - err = fmt.Errorf("replacement directory %s: %w", r.Path, err) - } - return dir, true, module.VersionError(mod, err) - } - return dir, true, nil - } - mod = r - } - - dir, err = modfetch.Download(mod) - return dir, false, err -} diff --git a/cmd/go/_internal_/modload/query.go b/cmd/go/_internal_/modload/query.go index 7c9cc8b..5946881 100644 --- a/cmd/go/_internal_/modload/query.go +++ b/cmd/go/_internal_/modload/query.go @@ -5,19 +5,24 @@ package modload import ( + "context" "errors" "fmt" + "io/fs" "os" pathpkg "path" "path/filepath" + "sort" "strings" "sync" + "time" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/imports" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/search" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/str" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/trace" "golang.org/x/mod/module" "golang.org/x/mod/semver" @@ -43,27 +48,43 @@ import ( // with non-prereleases preferred over prereleases. // - a repository commit identifier or tag, denoting that commit. // -// current denotes the current version of the module; it may be "" if the -// current version is unknown or should not be considered. If query is +// current denotes the currently-selected version of the module; it may be +// "none" if no version is currently selected, or "" if the currently-selected +// version is unknown or should not be considered. If query is // "upgrade" or "patch", current will be returned if it is a newer // semantic version or a chronologically later pseudo-version than the // version that would otherwise be chosen. This prevents accidental downgrades // from newer pre-release or development versions. // -// If the allowed function is non-nil, Query excludes any versions for which -// allowed returns false. +// The allowed function (which may be nil) is used to filter out unsuitable +// versions (see AllowedFunc documentation for details). If the query refers to +// a specific revision (for example, "master"; see IsRevisionQuery), and the +// revision is disallowed by allowed, Query returns the error. If the query +// does not refer to a specific revision (for example, "latest"), Query +// acts as if versions disallowed by allowed do not exist. // // If path is the path of the main module and the query is "latest", // Query returns Target.Version as the version. -func Query(path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { +func Query(ctx context.Context, path, query, current string, allowed AllowedFunc) (*modfetch.RevInfo, error) { var info *modfetch.RevInfo err := modfetch.TryProxies(func(proxy string) (err error) { - info, err = queryProxy(proxy, path, query, current, allowed) + info, err = queryProxy(ctx, proxy, path, query, current, allowed) return err }) return info, err } +// AllowedFunc is used by Query and other functions to filter out unsuitable +// versions, for example, those listed in exclude directives in the main +// module's go.mod file. +// +// An AllowedFunc returns an error equivalent to ErrDisallowed for an unsuitable +// version. Any other error indicates the function was unable to determine +// whether the version should be allowed, for example, the function was unable +// to fetch or parse a go.mod file containing retractions. Typically, errors +// other than ErrDisallowd may be ignored. +type AllowedFunc func(context.Context, module.Version) error + var errQueryDisabled error = queryDisabledError{} type queryDisabledError struct{} @@ -75,138 +96,56 @@ func (queryDisabledError) Error() string { return fmt.Sprintf("cannot query module due to -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) } -func queryProxy(proxy, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { - if current != "" && !semver.IsValid(current) { +func queryProxy(ctx context.Context, proxy, path, query, current string, allowed AllowedFunc) (*modfetch.RevInfo, error) { + ctx, span := trace.StartSpan(ctx, "modload.queryProxy "+path+" "+query) + defer span.Done() + + if current != "" && current != "none" && !semver.IsValid(current) { return nil, fmt.Errorf("invalid previous version %q", current) } if cfg.BuildMod == "vendor" { return nil, errQueryDisabled } if allowed == nil { - allowed = func(module.Version) bool { return true } + allowed = func(context.Context, module.Version) error { return nil } } - // Parse query to detect parse errors (and possibly handle query) - // before any network I/O. - badVersion := func(v string) (*modfetch.RevInfo, error) { - return nil, fmt.Errorf("invalid semantic version %q in range %q", v, query) - } - matchesMajor := func(v string) bool { - _, pathMajor, ok := module.SplitPathVersion(path) - if !ok { - return false + if path == Target.Path && (query == "upgrade" || query == "patch") { + if err := allowed(ctx, Target); err != nil { + return nil, fmt.Errorf("internal error: main module version is not allowed: %w", err) } - return module.CheckPathMajor(v, pathMajor) == nil + return &modfetch.RevInfo{Version: Target.Version}, nil } - var ( - ok func(module.Version) bool - prefix string - preferOlder bool - mayUseLatest bool - preferIncompatible bool = strings.HasSuffix(current, "+incompatible") - ) - switch { - case query == "latest": - ok = allowed - mayUseLatest = true - - case query == "upgrade": - ok = allowed - mayUseLatest = true - - case query == "patch": - if current == "" { - ok = allowed - mayUseLatest = true - } else { - prefix = semver.MajorMinor(current) - ok = func(m module.Version) bool { - return matchSemverPrefix(prefix, m.Version) && allowed(m) - } - } - - case strings.HasPrefix(query, "<="): - v := query[len("<="):] - if !semver.IsValid(v) { - return badVersion(v) - } - if isSemverPrefix(v) { - // Refuse to say whether <=v1.2 allows v1.2.3 (remember, @v1.2 might mean v1.2.3). - return nil, fmt.Errorf("ambiguous semantic version %q in range %q", v, query) - } - ok = func(m module.Version) bool { - return semver.Compare(m.Version, v) <= 0 && allowed(m) - } - if !matchesMajor(v) { - preferIncompatible = true - } - case strings.HasPrefix(query, "<"): - v := query[len("<"):] - if !semver.IsValid(v) { - return badVersion(v) - } - ok = func(m module.Version) bool { - return semver.Compare(m.Version, v) < 0 && allowed(m) - } - if !matchesMajor(v) { - preferIncompatible = true - } - - case strings.HasPrefix(query, ">="): - v := query[len(">="):] - if !semver.IsValid(v) { - return badVersion(v) - } - ok = func(m module.Version) bool { - return semver.Compare(m.Version, v) >= 0 && allowed(m) - } - preferOlder = true - if !matchesMajor(v) { - preferIncompatible = true - } + if path == "std" || path == "cmd" { + return nil, fmt.Errorf("can't query specific version (%q) of standard-library module %q", query, path) + } - case strings.HasPrefix(query, ">"): - v := query[len(">"):] - if !semver.IsValid(v) { - return badVersion(v) - } - if isSemverPrefix(v) { - // Refuse to say whether >v1.2 allows v1.2.3 (remember, @v1.2 might mean v1.2.3). - return nil, fmt.Errorf("ambiguous semantic version %q in range %q", v, query) - } - ok = func(m module.Version) bool { - return semver.Compare(m.Version, v) > 0 && allowed(m) - } - preferOlder = true - if !matchesMajor(v) { - preferIncompatible = true - } + repo, err := lookupRepo(proxy, path) + if err != nil { + return nil, err + } - case semver.IsValid(query) && isSemverPrefix(query): - ok = func(m module.Version) bool { - return matchSemverPrefix(query, m.Version) && allowed(m) - } - prefix = query + "." - if !matchesMajor(query) { - preferIncompatible = true - } + // Parse query to detect parse errors (and possibly handle query) + // before any network I/O. + qm, err := newQueryMatcher(path, query, current, allowed) + if (err == nil && qm.canStat) || err == errRevQuery { + // Direct lookup of a commit identifier or complete (non-prefix) semantic + // version. - default: - // Direct lookup of semantic version or commit identifier. - // // If the identifier is not a canonical semver tag — including if it's a // semver tag with a +metadata suffix — then modfetch.Stat will populate // info.Version with a suitable pseudo-version. - info, err := modfetch.Stat(proxy, path, query) + info, err := repo.Stat(query) if err != nil { queryErr := err // The full query doesn't correspond to a tag. If it is a semantic version // with a +metadata suffix, see if there is a tag without that suffix: // semantic versioning defines them to be equivalent. - if vers := module.CanonicalVersion(query); vers != "" && vers != query { - info, err = modfetch.Stat(proxy, path, vers) - if !errors.Is(err, os.ErrNotExist) { + canonicalQuery := module.CanonicalVersion(query) + if canonicalQuery != "" && query != canonicalQuery { + info, err = repo.Stat(canonicalQuery) + if err != nil && !errors.Is(err, fs.ErrNotExist) { return info, err } } @@ -214,36 +153,20 @@ func queryProxy(proxy, path, query, current string, allowed func(module.Version) return nil, queryErr } } - if !allowed(module.Version{Path: path, Version: info.Version}) { - return nil, fmt.Errorf("%s@%s excluded", path, info.Version) + if err := allowed(ctx, module.Version{Path: path, Version: info.Version}); errors.Is(err, ErrDisallowed) { + return nil, err } return info, nil - } - - if path == Target.Path { - if query != "latest" { - return nil, fmt.Errorf("can't query specific version (%q) for the main module (%s)", query, path) - } - if !allowed(Target) { - return nil, fmt.Errorf("internal error: main module version is not allowed") - } - return &modfetch.RevInfo{Version: Target.Version}, nil - } - - if str.HasPathPrefix(path, "std") || str.HasPathPrefix(path, "cmd") { - return nil, fmt.Errorf("explicit requirement on standard-library module %s not allowed", path) + } else if err != nil { + return nil, err } // Load versions and execute query. - repo, err := modfetch.Lookup(proxy, path) - if err != nil { - return nil, err - } - versions, err := repo.Versions(prefix) + versions, err := repo.Versions(qm.prefix) if err != nil { return nil, err } - releases, prereleases, err := filterVersions(path, versions, ok, preferIncompatible) + releases, prereleases, err := qm.filterVersions(ctx, versions) if err != nil { return nil, err } @@ -254,11 +177,30 @@ func queryProxy(proxy, path, query, current string, allowed func(module.Version) return nil, err } - // For "upgrade" and "patch", make sure we don't accidentally downgrade - // from a newer prerelease or from a chronologically newer pseudoversion. - if current != "" && (query == "upgrade" || query == "patch") { + if (query == "upgrade" || query == "patch") && modfetch.IsPseudoVersion(current) && !rev.Time.IsZero() { + // Don't allow "upgrade" or "patch" to move from a pseudo-version + // to a chronologically older version or pseudo-version. + // + // If the current version is a pseudo-version from an untagged branch, it + // may be semantically lower than the "latest" release or the latest + // pseudo-version on the main branch. A user on such a version is unlikely + // to intend to “upgrade” to a version that already existed at that point + // in time. + // + // We do this only if the current version is a pseudo-version: if the + // version is tagged, the author of the dependency module has given us + // explicit information about their intended precedence of this version + // relative to other versions, and we shouldn't contradict that + // information. (For example, v1.0.1 might be a backport of a fix already + // incorporated into v1.1.0, in which case v1.0.1 would be chronologically + // newer but v1.1.0 is still an “upgrade”; or v1.0.2 might be a revert of + // an unsuccessful fix in v1.0.1, in which case the v1.0.2 commit may be + // older than the v1.0.1 commit despite the tag itself being newer.) currentTime, err := modfetch.PseudoVersionTime(current) - if semver.Compare(rev.Version, current) < 0 || (err == nil && rev.Time.Before(currentTime)) { + if err == nil && rev.Time.Before(currentTime) { + if err := allowed(ctx, module.Version{Path: path, Version: current}); errors.Is(err, ErrDisallowed) { + return nil, err + } return repo.Stat(current) } } @@ -266,7 +208,7 @@ func queryProxy(proxy, path, query, current string, allowed func(module.Version) return rev, nil } - if preferOlder { + if qm.preferLower { if len(releases) > 0 { return lookup(releases[0]) } @@ -282,22 +224,44 @@ func queryProxy(proxy, path, query, current string, allowed func(module.Version) } } - if mayUseLatest { - // Special case for "latest": if no tags match, use latest commit in repo, - // provided it is not excluded. + if qm.mayUseLatest { latest, err := repo.Latest() if err == nil { - if allowed(module.Version{Path: path, Version: latest.Version}) { + if qm.allowsVersion(ctx, latest.Version) { return lookup(latest.Version) } - } else if !errors.Is(err, os.ErrNotExist) { + } else if !errors.Is(err, fs.ErrNotExist) { + return nil, err + } + } + + if (query == "upgrade" || query == "patch") && current != "" && current != "none" { + // "upgrade" and "patch" may stay on the current version if allowed. + if err := allowed(ctx, module.Version{Path: path, Version: current}); errors.Is(err, ErrDisallowed) { return nil, err } + return lookup(current) } return nil, &NoMatchingVersionError{query: query, current: current} } +// IsRevisionQuery returns true if vers is a version query that may refer to +// a particular version or revision in a repository like "v1.0.0", "master", +// or "0123abcd". IsRevisionQuery returns false if vers is a query that +// chooses from among available versions like "latest" or ">v1.0.0". +func IsRevisionQuery(vers string) bool { + if vers == "latest" || + vers == "upgrade" || + vers == "patch" || + strings.HasPrefix(vers, "<") || + strings.HasPrefix(vers, ">") || + (semver.IsValid(vers) && isSemverPrefix(vers)) { + return false + } + return true +} + // isSemverPrefix reports whether v is a semantic version prefix: v1 or v1.2 (not v1.2.3). // The caller is assumed to have checked that semver.IsValid(v) is true. func isSemverPrefix(v string) bool { @@ -316,25 +280,190 @@ func isSemverPrefix(v string) bool { return true } -// matchSemverPrefix reports whether the shortened semantic version p -// matches the full-width (non-shortened) semantic version v. -func matchSemverPrefix(p, v string) bool { - return len(v) > len(p) && v[len(p)] == '.' && v[:len(p)] == p && semver.Prerelease(v) == "" +type queryMatcher struct { + path string + prefix string + filter func(version string) bool + allowed AllowedFunc + canStat bool // if true, the query can be resolved by repo.Stat + preferLower bool // if true, choose the lowest matching version + mayUseLatest bool + preferIncompatible bool +} + +var errRevQuery = errors.New("query refers to a non-semver revision") + +// newQueryMatcher returns a new queryMatcher that matches the versions +// specified by the given query on the module with the given path. +// +// If the query can only be resolved by statting a non-SemVer revision, +// newQueryMatcher returns errRevQuery. +func newQueryMatcher(path string, query, current string, allowed AllowedFunc) (*queryMatcher, error) { + badVersion := func(v string) (*queryMatcher, error) { + return nil, fmt.Errorf("invalid semantic version %q in range %q", v, query) + } + + matchesMajor := func(v string) bool { + _, pathMajor, ok := module.SplitPathVersion(path) + if !ok { + return false + } + return module.CheckPathMajor(v, pathMajor) == nil + } + + qm := &queryMatcher{ + path: path, + allowed: allowed, + preferIncompatible: strings.HasSuffix(current, "+incompatible"), + } + + switch { + case query == "latest": + qm.mayUseLatest = true + + case query == "upgrade": + if current == "" || current == "none" { + qm.mayUseLatest = true + } else { + qm.mayUseLatest = modfetch.IsPseudoVersion(current) + qm.filter = func(mv string) bool { return semver.Compare(mv, current) >= 0 } + } + + case query == "patch": + if current == "none" { + return nil, &NoPatchBaseError{path} + } + if current == "" { + qm.mayUseLatest = true + } else { + qm.mayUseLatest = modfetch.IsPseudoVersion(current) + qm.prefix = semver.MajorMinor(current) + "." + qm.filter = func(mv string) bool { return semver.Compare(mv, current) >= 0 } + } + + case strings.HasPrefix(query, "<="): + v := query[len("<="):] + if !semver.IsValid(v) { + return badVersion(v) + } + if isSemverPrefix(v) { + // Refuse to say whether <=v1.2 allows v1.2.3 (remember, @v1.2 might mean v1.2.3). + return nil, fmt.Errorf("ambiguous semantic version %q in range %q", v, query) + } + qm.filter = func(mv string) bool { return semver.Compare(mv, v) <= 0 } + if !matchesMajor(v) { + qm.preferIncompatible = true + } + + case strings.HasPrefix(query, "<"): + v := query[len("<"):] + if !semver.IsValid(v) { + return badVersion(v) + } + qm.filter = func(mv string) bool { return semver.Compare(mv, v) < 0 } + if !matchesMajor(v) { + qm.preferIncompatible = true + } + + case strings.HasPrefix(query, ">="): + v := query[len(">="):] + if !semver.IsValid(v) { + return badVersion(v) + } + qm.filter = func(mv string) bool { return semver.Compare(mv, v) >= 0 } + qm.preferLower = true + if !matchesMajor(v) { + qm.preferIncompatible = true + } + + case strings.HasPrefix(query, ">"): + v := query[len(">"):] + if !semver.IsValid(v) { + return badVersion(v) + } + if isSemverPrefix(v) { + // Refuse to say whether >v1.2 allows v1.2.3 (remember, @v1.2 might mean v1.2.3). + return nil, fmt.Errorf("ambiguous semantic version %q in range %q", v, query) + } + qm.filter = func(mv string) bool { return semver.Compare(mv, v) > 0 } + qm.preferLower = true + if !matchesMajor(v) { + qm.preferIncompatible = true + } + + case semver.IsValid(query): + if isSemverPrefix(query) { + qm.prefix = query + "." + // Do not allow the query "v1.2" to match versions lower than "v1.2.0", + // such as prereleases for that version. (https://golang.org/issue/31972) + qm.filter = func(mv string) bool { return semver.Compare(mv, query) >= 0 } + } else { + qm.canStat = true + qm.filter = func(mv string) bool { return semver.Compare(mv, query) == 0 } + qm.prefix = semver.Canonical(query) + } + if !matchesMajor(query) { + qm.preferIncompatible = true + } + + default: + return nil, errRevQuery + } + + return qm, nil +} + +// allowsVersion reports whether version v is allowed by the prefix, filter, and +// AllowedFunc of qm. +func (qm *queryMatcher) allowsVersion(ctx context.Context, v string) bool { + if qm.prefix != "" && !strings.HasPrefix(v, qm.prefix) { + return false + } + if qm.filter != nil && !qm.filter(v) { + return false + } + if qm.allowed != nil { + if err := qm.allowed(ctx, module.Version{Path: qm.path, Version: v}); errors.Is(err, ErrDisallowed) { + return false + } + } + return true } // filterVersions classifies versions into releases and pre-releases, filtering // out: -// 1. versions that do not satisfy the 'ok' predicate, and +// 1. versions that do not satisfy the 'allowed' predicate, and // 2. "+incompatible" versions, if a compatible one satisfies the predicate // and the incompatible version is not preferred. -func filterVersions(path string, versions []string, ok func(module.Version) bool, preferIncompatible bool) (releases, prereleases []string, err error) { +// +// If the allowed predicate returns an error not equivalent to ErrDisallowed, +// filterVersions returns that error. +func (qm *queryMatcher) filterVersions(ctx context.Context, versions []string) (releases, prereleases []string, err error) { + needIncompatible := qm.preferIncompatible + var lastCompatible string for _, v := range versions { - if !ok(module.Version{Path: path, Version: v}) { + if !qm.allowsVersion(ctx, v) { continue } - if !preferIncompatible { + if !needIncompatible { + // We're not yet sure whether we need to include +incomptaible versions. + // Keep track of the last compatible version we've seen, and use the + // presence (or absence) of a go.mod file in that version to decide: a + // go.mod file implies that the module author is supporting modules at a + // compatible version (and we should ignore +incompatible versions unless + // requested explicitly), while a lack of go.mod file implies the + // potential for legacy (pre-modules) versioning without semantic import + // paths (and thus *with* +incompatible versions). + // + // This isn't strictly accurate if the latest compatible version has been + // replaced by a local file path, because we do not allow file-path + // replacements without a go.mod file: the user would have needed to add + // one. However, replacing the last compatible version while + // simultaneously expecting to upgrade implicitly to a +incompatible + // version seems like an extreme enough corner case to ignore for now. + if !strings.HasSuffix(v, "+incompatible") { lastCompatible = v } else if lastCompatible != "" { @@ -342,19 +471,22 @@ func filterVersions(path string, versions []string, ok func(module.Version) bool // ignore any version with a higher (+incompatible) major version. (See // https://golang.org/issue/34165.) Note that we even prefer a // compatible pre-release over an incompatible release. - - ok, err := versionHasGoMod(module.Version{Path: path, Version: lastCompatible}) + ok, err := versionHasGoMod(ctx, module.Version{Path: qm.path, Version: lastCompatible}) if err != nil { return nil, nil, err } if ok { + // The last compatible version has a go.mod file, so that's the + // highest version we're willing to consider. Don't bother even + // looking at higher versions, because they're all +incompatible from + // here onward. break } // No acceptable compatible release has a go.mod file, so the versioning // for the module might not be module-aware, and we should respect // legacy major-version tags. - preferIncompatible = true + needIncompatible = true } } @@ -374,34 +506,42 @@ type QueryResult struct { Packages []string } -// QueryPackage looks up the module(s) containing path at a revision matching -// query. The results are sorted by module path length in descending order. -// -// If the package is in the main module, QueryPackage considers only the main -// module and only the version "latest", without checking for other possible -// modules. -func QueryPackage(path, query string, allowed func(module.Version) bool) ([]QueryResult, error) { - m := search.NewMatch(path) - if m.IsLocal() || !m.IsLiteral() { - return nil, fmt.Errorf("pattern %s is not an importable package", path) +// QueryPackages is like QueryPattern, but requires that the pattern match at +// least one package and omits the non-package result (if any). +func QueryPackages(ctx context.Context, pattern, query string, current func(string) string, allowed AllowedFunc) ([]QueryResult, error) { + pkgMods, modOnly, err := QueryPattern(ctx, pattern, query, current, allowed) + + if len(pkgMods) == 0 && err == nil { + return nil, &PackageNotInModuleError{ + Mod: modOnly.Mod, + Replacement: Replacement(modOnly.Mod), + Query: query, + Pattern: pattern, + } } - return QueryPattern(path, query, allowed) + + return pkgMods, err } // QueryPattern looks up the module(s) containing at least one package matching // the given pattern at the given version. The results are sorted by module path -// length in descending order. +// length in descending order. If any proxy provides a non-empty set of candidate +// modules, no further proxies are tried. // -// QueryPattern queries modules with package paths up to the first "..." -// in the pattern. For the pattern "example.com/a/b.../c", QueryPattern would -// consider prefixes of "example.com/a". If multiple modules have versions -// that match the query and packages that match the pattern, QueryPattern -// picks the one with the longest module path. +// For wildcard patterns, QueryPattern looks in modules with package paths up to +// the first "..." in the pattern. For the pattern "example.com/a/b.../c", +// QueryPattern would consider prefixes of "example.com/a". // // If any matching package is in the main module, QueryPattern considers only // the main module and only the version "latest", without checking for other // possible modules. -func QueryPattern(pattern, query string, allowed func(module.Version) bool) ([]QueryResult, error) { +// +// QueryPattern always returns at least one QueryResult (which may be only +// modOnly) or a non-nil error. +func QueryPattern(ctx context.Context, pattern, query string, current func(string) string, allowed AllowedFunc) (pkgMods []QueryResult, modOnly *QueryResult, err error) { + ctx, span := trace.StartSpan(ctx, "modload.QueryPattern "+pattern+" "+query) + defer span.Done() + base := pattern firstError := func(m *search.Match) error { @@ -412,12 +552,16 @@ func QueryPattern(pattern, query string, allowed func(module.Version) bool) ([]Q } var match func(mod module.Version, root string, isLocal bool) *search.Match + matchPattern := search.MatchPattern(pattern) if i := strings.Index(pattern, "..."); i >= 0 { base = pathpkg.Dir(pattern[:i+3]) + if base == "." { + return nil, nil, &WildcardInFirstElementError{Pattern: pattern, Query: query} + } match = func(mod module.Version, root string, isLocal bool) *search.Match { m := search.NewMatch(pattern) - matchPackages(m, imports.AnyTags(), omitStd, []module.Version{mod}) + matchPackages(ctx, m, imports.AnyTags(), omitStd, []module.Version{mod}) return m } } else { @@ -436,23 +580,41 @@ func QueryPattern(pattern, query string, allowed func(module.Version) bool) ([]Q } } + var queryMatchesMainModule bool if HasModRoot() { m := match(Target, modRoot, true) if len(m.Pkgs) > 0 { - if query != "latest" { - return nil, fmt.Errorf("can't query specific version for package %s in the main module (%s)", pattern, Target.Path) + if query != "upgrade" && query != "patch" { + return nil, nil, &QueryMatchesPackagesInMainModuleError{ + Pattern: pattern, + Query: query, + Packages: m.Pkgs, + } } - if !allowed(Target) { - return nil, fmt.Errorf("internal error: package %s is in the main module (%s), but version is not allowed", pattern, Target.Path) + if err := allowed(ctx, Target); err != nil { + return nil, nil, fmt.Errorf("internal error: package %s is in the main module (%s), but version is not allowed: %w", pattern, Target.Path, err) } return []QueryResult{{ Mod: Target, Rev: &modfetch.RevInfo{Version: Target.Version}, Packages: m.Pkgs, - }}, nil + }}, nil, nil } if err := firstError(m); err != nil { - return nil, err + return nil, nil, err + } + + if matchPattern(Target.Path) { + queryMatchesMainModule = true + } + + if (query == "upgrade" || query == "patch") && queryMatchesMainModule { + if err := allowed(ctx, Target); err == nil { + modOnly = &QueryResult{ + Mod: Target, + Rev: &modfetch.RevInfo{Version: Target.Version}, + } + } } } @@ -461,29 +623,42 @@ func QueryPattern(pattern, query string, allowed func(module.Version) bool) ([]Q candidateModules = modulePrefixesExcludingTarget(base) ) if len(candidateModules) == 0 { - return nil, &PackageNotInModuleError{ - Mod: Target, - Query: query, - Pattern: pattern, + if modOnly != nil { + return nil, modOnly, nil + } else if queryMatchesMainModule { + return nil, nil, &QueryMatchesMainModuleError{ + Pattern: pattern, + Query: query, + } + } else { + return nil, nil, &PackageNotInModuleError{ + Mod: Target, + Query: query, + Pattern: pattern, + } } } - err := modfetch.TryProxies(func(proxy string) error { - queryModule := func(path string) (r QueryResult, err error) { - current := findCurrentVersion(path) + err = modfetch.TryProxies(func(proxy string) error { + queryModule := func(ctx context.Context, path string) (r QueryResult, err error) { + ctx, span := trace.StartSpan(ctx, "modload.QueryPattern.queryModule ["+proxy+"] "+path) + defer span.Done() + + pathCurrent := current(path) r.Mod.Path = path - r.Rev, err = queryProxy(proxy, path, query, current, allowed) + r.Rev, err = queryProxy(ctx, proxy, path, query, pathCurrent, allowed) if err != nil { return r, err } r.Mod.Version = r.Rev.Version - root, isLocal, err := fetch(r.Mod) + needSum := true + root, isLocal, err := fetch(ctx, r.Mod, needSum) if err != nil { return r, err } m := match(r.Mod, root, isLocal) r.Packages = m.Pkgs - if len(r.Packages) == 0 { + if len(r.Packages) == 0 && !matchPattern(path) { if err := firstError(m); err != nil { return r, err } @@ -497,12 +672,25 @@ func QueryPattern(pattern, query string, allowed func(module.Version) bool) ([]Q return r, nil } - var err error - results, err = queryPrefixModules(candidateModules, queryModule) + allResults, err := queryPrefixModules(ctx, candidateModules, queryModule) + results = allResults[:0] + for _, r := range allResults { + if len(r.Packages) == 0 { + modOnly = &r + } else { + results = append(results, r) + } + } return err }) - return results, err + if queryMatchesMainModule && len(results) == 0 && modOnly == nil && errors.Is(err, fs.ErrNotExist) { + return nil, nil, &QueryMatchesMainModuleError{ + Pattern: pattern, + Query: query, + } + } + return results[:len(results):len(results)], modOnly, err } // modulePrefixesExcludingTarget returns all prefixes of path that may plausibly @@ -528,21 +716,10 @@ func modulePrefixesExcludingTarget(path string) []string { return prefixes } -func findCurrentVersion(path string) string { - for _, m := range buildList { - if m.Path == path { - return m.Version - } - } - return "" -} +func queryPrefixModules(ctx context.Context, candidateModules []string, queryModule func(ctx context.Context, path string) (QueryResult, error)) (found []QueryResult, err error) { + ctx, span := trace.StartSpan(ctx, "modload.queryPrefixModules") + defer span.Done() -type prefixResult struct { - QueryResult - err error -} - -func queryPrefixModules(candidateModules []string, queryModule func(path string) (QueryResult, error)) (found []QueryResult, err error) { // If the path we're attempting is not in the module cache and we don't have a // fetch result cached either, we'll end up making a (potentially slow) // request to the proxy or (often even slower) the origin server. @@ -555,8 +732,9 @@ func queryPrefixModules(candidateModules []string, queryModule func(path string) var wg sync.WaitGroup wg.Add(len(candidateModules)) for i, p := range candidateModules { + ctx := trace.StartGoroutine(ctx) go func(p string, r *result) { - r.QueryResult, r.err = queryModule(p) + r.QueryResult, r.err = queryModule(ctx, p) wg.Done() }(p, &results[i]) } @@ -568,6 +746,7 @@ func queryPrefixModules(candidateModules []string, queryModule func(path string) var ( noPackage *PackageNotInModuleError noVersion *NoMatchingVersionError + noPatchBase *NoPatchBaseError notExistErr error ) for _, r := range results { @@ -584,8 +763,12 @@ func queryPrefixModules(candidateModules []string, queryModule func(path string) if noVersion == nil { noVersion = rErr } + case *NoPatchBaseError: + if noPatchBase == nil { + noPatchBase = rErr + } default: - if errors.Is(rErr, os.ErrNotExist) { + if errors.Is(rErr, fs.ErrNotExist) { if notExistErr == nil { notExistErr = rErr } @@ -615,6 +798,8 @@ func queryPrefixModules(candidateModules []string, queryModule func(path string) err = noPackage case noVersion != nil: err = noVersion + case noPatchBase != nil: + err = noPatchBase case notExistErr != nil: err = notExistErr default: @@ -628,7 +813,7 @@ func queryPrefixModules(candidateModules []string, queryModule func(path string) // A NoMatchingVersionError indicates that Query found a module at the requested // path, but not at any versions satisfying the query string and allow-function. // -// NOTE: NoMatchingVersionError MUST NOT implement Is(os.ErrNotExist). +// NOTE: NoMatchingVersionError MUST NOT implement Is(fs.ErrNotExist). // // If the module came from a proxy, that proxy had to return a successful status // code for the versions it knows about, and thus did not have the opportunity @@ -639,17 +824,39 @@ type NoMatchingVersionError struct { func (e *NoMatchingVersionError) Error() string { currentSuffix := "" - if (e.query == "upgrade" || e.query == "patch") && e.current != "" { + if (e.query == "upgrade" || e.query == "patch") && e.current != "" && e.current != "none" { currentSuffix = fmt.Sprintf(" (current version is %s)", e.current) } return fmt.Sprintf("no matching versions for query %q", e.query) + currentSuffix } +// A NoPatchBaseError indicates that Query was called with the query "patch" +// but with a current version of "" or "none". +type NoPatchBaseError struct { + path string +} + +func (e *NoPatchBaseError) Error() string { + return fmt.Sprintf(`can't query version "patch" of module %s: no existing version is required`, e.path) +} + +// A WildcardInFirstElementError indicates that a pattern passed to QueryPattern +// had a wildcard in its first path element, and therefore had no pattern-prefix +// modules to search in. +type WildcardInFirstElementError struct { + Pattern string + Query string +} + +func (e *WildcardInFirstElementError) Error() string { + return fmt.Sprintf("no modules to query for %s@%s because first path element contains a wildcard", e.Pattern, e.Query) +} + // A PackageNotInModuleError indicates that QueryPattern found a candidate // module at the requested version, but that module did not contain any packages // matching the requested pattern. // -// NOTE: PackageNotInModuleError MUST NOT implement Is(os.ErrNotExist). +// NOTE: PackageNotInModuleError MUST NOT implement Is(fs.ErrNotExist). // // If the module came from a proxy, that proxy had to return a successful status // code for the versions it knows about, and thus did not have the opportunity @@ -698,8 +905,9 @@ func (e *PackageNotInModuleError) ImportPath() string { } // ModuleHasRootPackage returns whether module m contains a package m.Path. -func ModuleHasRootPackage(m module.Version) (bool, error) { - root, isLocal, err := fetch(m) +func ModuleHasRootPackage(ctx context.Context, m module.Version) (bool, error) { + needSum := false + root, isLocal, err := fetch(ctx, m, needSum) if err != nil { return false, err } @@ -707,11 +915,197 @@ func ModuleHasRootPackage(m module.Version) (bool, error) { return ok, err } -func versionHasGoMod(m module.Version) (bool, error) { - root, _, err := fetch(m) +func versionHasGoMod(ctx context.Context, m module.Version) (bool, error) { + needSum := false + root, _, err := fetch(ctx, m, needSum) if err != nil { return false, err } fi, err := os.Stat(filepath.Join(root, "go.mod")) return err == nil && !fi.IsDir(), nil } + +// A versionRepo is a subset of modfetch.Repo that can report information about +// available versions, but cannot fetch specific source files. +type versionRepo interface { + ModulePath() string + Versions(prefix string) ([]string, error) + Stat(rev string) (*modfetch.RevInfo, error) + Latest() (*modfetch.RevInfo, error) +} + +var _ versionRepo = modfetch.Repo(nil) + +func lookupRepo(proxy, path string) (repo versionRepo, err error) { + err = module.CheckPath(path) + if err == nil { + repo = modfetch.Lookup(proxy, path) + } else { + repo = emptyRepo{path: path, err: err} + } + + if index == nil { + return repo, err + } + if _, ok := index.highestReplaced[path]; !ok { + return repo, err + } + + return &replacementRepo{repo: repo}, nil +} + +// An emptyRepo is a versionRepo that contains no versions. +type emptyRepo struct { + path string + err error +} + +var _ versionRepo = emptyRepo{} + +func (er emptyRepo) ModulePath() string { return er.path } +func (er emptyRepo) Versions(prefix string) ([]string, error) { return nil, nil } +func (er emptyRepo) Stat(rev string) (*modfetch.RevInfo, error) { return nil, er.err } +func (er emptyRepo) Latest() (*modfetch.RevInfo, error) { return nil, er.err } + +// A replacementRepo augments a versionRepo to include the replacement versions +// (if any) found in the main module's go.mod file. +// +// A replacementRepo suppresses "not found" errors for otherwise-nonexistent +// modules, so a replacementRepo should only be constructed for a module that +// actually has one or more valid replacements. +type replacementRepo struct { + repo versionRepo +} + +var _ versionRepo = (*replacementRepo)(nil) + +func (rr *replacementRepo) ModulePath() string { return rr.repo.ModulePath() } + +// Versions returns the versions from rr.repo augmented with any matching +// replacement versions. +func (rr *replacementRepo) Versions(prefix string) ([]string, error) { + repoVersions, err := rr.repo.Versions(prefix) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + + versions := repoVersions + if index != nil && len(index.replace) > 0 { + path := rr.ModulePath() + for m, _ := range index.replace { + if m.Path == path && strings.HasPrefix(m.Version, prefix) && m.Version != "" && !modfetch.IsPseudoVersion(m.Version) { + versions = append(versions, m.Version) + } + } + } + + if len(versions) == len(repoVersions) { // No replacement versions added. + return versions, nil + } + + sort.Slice(versions, func(i, j int) bool { + return semver.Compare(versions[i], versions[j]) < 0 + }) + str.Uniq(&versions) + return versions, nil +} + +func (rr *replacementRepo) Stat(rev string) (*modfetch.RevInfo, error) { + info, err := rr.repo.Stat(rev) + if err == nil || index == nil || len(index.replace) == 0 { + return info, err + } + + v := module.CanonicalVersion(rev) + if v != rev { + // The replacements in the go.mod file list only canonical semantic versions, + // so a non-canonical version can't possibly have a replacement. + return info, err + } + + path := rr.ModulePath() + _, pathMajor, ok := module.SplitPathVersion(path) + if ok && pathMajor == "" { + if err := module.CheckPathMajor(v, pathMajor); err != nil && semver.Build(v) == "" { + v += "+incompatible" + } + } + + if r := Replacement(module.Version{Path: path, Version: v}); r.Path == "" { + return info, err + } + return rr.replacementStat(v) +} + +func (rr *replacementRepo) Latest() (*modfetch.RevInfo, error) { + info, err := rr.repo.Latest() + + if index != nil { + path := rr.ModulePath() + if v, ok := index.highestReplaced[path]; ok { + if v == "" { + // The only replacement is a wildcard that doesn't specify a version, so + // synthesize a pseudo-version with an appropriate major version and a + // timestamp below any real timestamp. That way, if the main module is + // used from within some other module, the user will be able to upgrade + // the requirement to any real version they choose. + if _, pathMajor, ok := module.SplitPathVersion(path); ok && len(pathMajor) > 0 { + v = modfetch.PseudoVersion(pathMajor[1:], "", time.Time{}, "000000000000") + } else { + v = modfetch.PseudoVersion("v0", "", time.Time{}, "000000000000") + } + } + + if err != nil || semver.Compare(v, info.Version) > 0 { + return rr.replacementStat(v) + } + } + } + + return info, err +} + +func (rr *replacementRepo) replacementStat(v string) (*modfetch.RevInfo, error) { + rev := &modfetch.RevInfo{Version: v} + if modfetch.IsPseudoVersion(v) { + rev.Time, _ = modfetch.PseudoVersionTime(v) + rev.Short, _ = modfetch.PseudoVersionRev(v) + } + return rev, nil +} + +// A QueryMatchesMainModuleError indicates that a query requests +// a version of the main module that cannot be satisfied. +// (The main module's version cannot be changed.) +type QueryMatchesMainModuleError struct { + Pattern string + Query string +} + +func (e *QueryMatchesMainModuleError) Error() string { + if e.Pattern == Target.Path { + return fmt.Sprintf("can't request version %q of the main module (%s)", e.Query, e.Pattern) + } + + return fmt.Sprintf("can't request version %q of pattern %q that includes the main module (%s)", e.Query, e.Pattern, Target.Path) +} + +// A QueryMatchesPackagesInMainModuleError indicates that a query cannot be +// satisfied because it matches one or more packages found in the main module. +type QueryMatchesPackagesInMainModuleError struct { + Pattern string + Query string + Packages []string +} + +func (e *QueryMatchesPackagesInMainModuleError) Error() string { + if len(e.Packages) > 1 { + return fmt.Sprintf("pattern %s matches %d packages in the main module, so can't request version %s", e.Pattern, len(e.Packages), e.Query) + } + + if search.IsMetaPackage(e.Pattern) || strings.Contains(e.Pattern, "...") { + return fmt.Sprintf("pattern %s matches package %s in the main module, so can't request version %s", e.Pattern, e.Packages[0], e.Query) + } + + return fmt.Sprintf("package %s is in the main module, so can't request version %s", e.Packages[0], e.Query) +} diff --git a/cmd/go/_internal_/modload/search.go b/cmd/go/_internal_/modload/search.go index d4eee74..5dace6a 100644 --- a/cmd/go/_internal_/modload/search.go +++ b/cmd/go/_internal_/modload/search.go @@ -5,12 +5,15 @@ package modload import ( + "context" "fmt" + "io/fs" "os" "path/filepath" "strings" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/fsys" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/imports" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/search" @@ -27,7 +30,7 @@ const ( // matchPackages is like m.MatchPackages, but uses a local variable (rather than // a global) for tags, can include or exclude packages in the standard library, // and is restricted to the given list of modules. -func matchPackages(m *search.Match, tags map[string]bool, filter stdFilter, modules []module.Version) { +func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, filter stdFilter, modules []module.Version) { m.Pkgs = []string{} isMatch := func(string) bool { return true } @@ -52,7 +55,7 @@ func matchPackages(m *search.Match, tags map[string]bool, filter stdFilter, modu walkPkgs := func(root, importPathRoot string, prune pruning) { root = filepath.Clean(root) - err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { + err := fsys.Walk(root, func(path string, fi fs.FileInfo, err error) error { if err != nil { m.AddError(err) return nil @@ -83,8 +86,8 @@ func matchPackages(m *search.Match, tags map[string]bool, filter stdFilter, modu } if !fi.IsDir() { - if fi.Mode()&os.ModeSymlink != 0 && want { - if target, err := os.Stat(path); err == nil && target.IsDir() { + if fi.Mode()&fs.ModeSymlink != 0 && want { + if target, err := fsys.Stat(path); err == nil && target.IsDir() { fmt.Fprintf(os.Stderr, "warning: ignoring symlink %s\n", path) } } @@ -153,7 +156,8 @@ func matchPackages(m *search.Match, tags map[string]bool, filter stdFilter, modu isLocal = true } else { var err error - root, isLocal, err = fetch(mod) + const needSum = true + root, isLocal, err = fetch(ctx, mod, needSum) if err != nil { m.AddError(err) continue @@ -170,3 +174,46 @@ func matchPackages(m *search.Match, tags map[string]bool, filter stdFilter, modu return } + +// MatchInModule identifies the packages matching the given pattern within the +// given module version, which does not need to be in the build list or module +// requirement graph. +// +// If m is the zero module.Version, MatchInModule matches the pattern +// against the standard library (std and cmd) in GOROOT/src. +func MatchInModule(ctx context.Context, pattern string, m module.Version, tags map[string]bool) *search.Match { + match := search.NewMatch(pattern) + if m == (module.Version{}) { + matchPackages(ctx, match, tags, includeStd, nil) + } + + LoadModFile(ctx) + + if !match.IsLiteral() { + matchPackages(ctx, match, tags, omitStd, []module.Version{m}) + return match + } + + const needSum = true + root, isLocal, err := fetch(ctx, m, needSum) + if err != nil { + match.Errs = []error{err} + return match + } + + dir, haveGoFiles, err := dirInModule(pattern, m.Path, root, isLocal) + if err != nil { + match.Errs = []error{err} + return match + } + if haveGoFiles { + if _, _, err := scanDir(dir, tags); err != imports.ErrNoGo { + // ErrNoGo indicates that the directory is not actually a Go package, + // perhaps due to the tags in use. Any other non-nil error indicates a + // problem with one or more of the Go source files, but such an error does + // not stop the package from existing, so it has no impact on matching. + match.Pkgs = []string{pattern} + } + } + return match +} diff --git a/cmd/go/_internal_/modload/stat_unix.go b/cmd/go/_internal_/modload/stat_unix.go index ea3b801..f49278e 100644 --- a/cmd/go/_internal_/modload/stat_unix.go +++ b/cmd/go/_internal_/modload/stat_unix.go @@ -2,11 +2,12 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux netbsd openbsd solaris +// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris package modload import ( + "io/fs" "os" "syscall" ) @@ -17,7 +18,7 @@ import ( // Although the root user on most Unix systems can write to files even without // permission, hasWritePerm reports false if no appropriate permission bit is // set even if the current user is root. -func hasWritePerm(path string, fi os.FileInfo) bool { +func hasWritePerm(path string, fi fs.FileInfo) bool { if os.Getuid() == 0 { // The root user can access any file, but we still want to default to // read-only mode if the go.mod file is marked as globally non-writable. diff --git a/cmd/go/_internal_/modload/vendor.go b/cmd/go/_internal_/modload/vendor.go index db3735c..faa8788 100644 --- a/cmd/go/_internal_/modload/vendor.go +++ b/cmd/go/_internal_/modload/vendor.go @@ -7,7 +7,7 @@ package modload import ( "errors" "fmt" - "io/ioutil" + "io/fs" "os" "path/filepath" "strings" @@ -40,9 +40,9 @@ func readVendorList() { vendorPkgModule = make(map[string]module.Version) vendorVersion = make(map[string]string) vendorMeta = make(map[module.Version]vendorMetadata) - data, err := ioutil.ReadFile(filepath.Join(ModRoot(), "vendor/modules.txt")) + data, err := os.ReadFile(filepath.Join(ModRoot(), "vendor/modules.txt")) if err != nil { - if !errors.Is(err, os.ErrNotExist) { + if !errors.Is(err, fs.ErrNotExist) { base.Fatalf("go: %s", err) } return @@ -133,7 +133,7 @@ func checkVendorConsistency() { readVendorList() pre114 := false - if modFile.Go == nil || semver.Compare("v"+modFile.Go.Version, "v1.14") < 0 { + if semver.Compare(index.goVersionV, "v1.14") < 0 { // Go versions before 1.14 did not include enough information in // vendor/modules.txt to check for consistency. // If we know that we're on an earlier version, relax the consistency check. @@ -150,6 +150,8 @@ func checkVendorConsistency() { } } + // Iterate over the Require directives in their original (not indexed) order + // so that the errors match the original file. for _, r := range modFile.Require { if !vendorMeta[r.Mod].Explicit { if pre114 { @@ -212,6 +214,6 @@ func checkVendorConsistency() { } if vendErrors.Len() > 0 { - base.Fatalf("go: inconsistent vendoring in %s:%s\n\nrun 'go mod vendor' to sync, or use -mod=mod or -mod=readonly to ignore the vendor directory", modRoot, vendErrors) + base.Fatalf("go: inconsistent vendoring in %s:%s\n\n\tTo ignore the vendor directory, use -mod=readonly or -mod=mod.\n\tTo sync the vendor directory, run:\n\t\tgo mod vendor", modRoot, vendErrors) } } diff --git a/cmd/go/_internal_/mvs/errors.go b/cmd/go/_internal_/mvs/errors.go new file mode 100644 index 0000000..8d9f969 --- /dev/null +++ b/cmd/go/_internal_/mvs/errors.go @@ -0,0 +1,101 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mvs + +import ( + "fmt" + "strings" + + "golang.org/x/mod/module" +) + +// BuildListError decorates an error that occurred gathering requirements +// while constructing a build list. BuildListError prints the chain +// of requirements to the module where the error occurred. +type BuildListError struct { + Err error + stack []buildListErrorElem +} + +type buildListErrorElem struct { + m module.Version + + // nextReason is the reason this module depends on the next module in the + // stack. Typically either "requires", or "updating to". + nextReason string +} + +// NewBuildListError returns a new BuildListError wrapping an error that +// occurred at a module found along the given path of requirements and/or +// upgrades, which must be non-empty. +// +// The isUpgrade function reports whether a path step is due to an upgrade. +// A nil isUpgrade function indicates that none of the path steps are due to upgrades. +func NewBuildListError(err error, path []module.Version, isUpgrade func(from, to module.Version) bool) *BuildListError { + stack := make([]buildListErrorElem, 0, len(path)) + for len(path) > 1 { + reason := "requires" + if isUpgrade != nil && isUpgrade(path[0], path[1]) { + reason = "updating to" + } + stack = append(stack, buildListErrorElem{ + m: path[0], + nextReason: reason, + }) + path = path[1:] + } + stack = append(stack, buildListErrorElem{m: path[0]}) + + return &BuildListError{ + Err: err, + stack: stack, + } +} + +// Module returns the module where the error occurred. If the module stack +// is empty, this returns a zero value. +func (e *BuildListError) Module() module.Version { + if len(e.stack) == 0 { + return module.Version{} + } + return e.stack[len(e.stack)-1].m +} + +func (e *BuildListError) Error() string { + b := &strings.Builder{} + stack := e.stack + + // Don't print modules at the beginning of the chain without a + // version. These always seem to be the main module or a + // synthetic module ("target@"). + for len(stack) > 0 && stack[0].m.Version == "" { + stack = stack[1:] + } + + if len(stack) == 0 { + b.WriteString(e.Err.Error()) + } else { + for _, elem := range stack[:len(stack)-1] { + fmt.Fprintf(b, "%s %s\n\t", elem.m, elem.nextReason) + } + // Ensure that the final module path and version are included as part of the + // error message. + m := stack[len(stack)-1].m + if mErr, ok := e.Err.(*module.ModuleError); ok { + actual := module.Version{Path: mErr.Path, Version: mErr.Version} + if v, ok := mErr.Err.(*module.InvalidVersionError); ok { + actual.Version = v.Version + } + if actual == m { + fmt.Fprintf(b, "%v", e.Err) + } else { + fmt.Fprintf(b, "%s (replaced by %s): %v", m, actual, mErr.Err) + } + } else { + fmt.Fprintf(b, "%v", module.VersionError(m, e.Err)) + } + } + return b.String() +} diff --git a/cmd/go/_internal_/mvs/mvs.go b/cmd/go/_internal_/mvs/mvs.go index 59905aa..52710d8 100644 --- a/cmd/go/_internal_/mvs/mvs.go +++ b/cmd/go/_internal_/mvs/mvs.go @@ -9,7 +9,6 @@ package mvs import ( "fmt" "sort" - "strings" "sync" "sync/atomic" @@ -61,59 +60,6 @@ type Reqs interface { Previous(m module.Version) (module.Version, error) } -// BuildListError decorates an error that occurred gathering requirements -// while constructing a build list. BuildListError prints the chain -// of requirements to the module where the error occurred. -type BuildListError struct { - Err error - stack []buildListErrorElem -} - -type buildListErrorElem struct { - m module.Version - - // nextReason is the reason this module depends on the next module in the - // stack. Typically either "requires", or "upgraded to". - nextReason string -} - -// Module returns the module where the error occurred. If the module stack -// is empty, this returns a zero value. -func (e *BuildListError) Module() module.Version { - if len(e.stack) == 0 { - return module.Version{} - } - return e.stack[0].m -} - -func (e *BuildListError) Error() string { - b := &strings.Builder{} - stack := e.stack - - // Don't print modules at the beginning of the chain without a - // version. These always seem to be the main module or a - // synthetic module ("target@"). - for len(stack) > 0 && stack[len(stack)-1].m.Version == "" { - stack = stack[:len(stack)-1] - } - - for i := len(stack) - 1; i >= 1; i-- { - fmt.Fprintf(b, "%s@%s %s\n\t", stack[i].m.Path, stack[i].m.Version, stack[i].nextReason) - } - if len(stack) == 0 { - b.WriteString(e.Err.Error()) - } else { - // Ensure that the final module path and version are included as part of the - // error message. - if _, ok := e.Err.(*module.ModuleError); ok { - fmt.Fprintf(b, "%v", e.Err) - } else { - fmt.Fprintf(b, "%v", module.VersionError(stack[0].m, e.Err)) - } - } - return b.String() -} - // BuildList returns the build list for the target module. // // target is the root vertex of a module requirement graph. For cmd/go, this is @@ -162,19 +108,23 @@ func buildList(target module.Version, reqs Reqs, upgrade func(module.Version) (m node := &modGraphNode{m: m} mu.Lock() modGraph[m] = node - if v, ok := min[m.Path]; !ok || reqs.Max(v, m.Version) != v { - min[m.Path] = m.Version + if m.Version != "none" { + if v, ok := min[m.Path]; !ok || reqs.Max(v, m.Version) != v { + min[m.Path] = m.Version + } } mu.Unlock() - required, err := reqs.Required(m) - if err != nil { - setErr(node, err) - return - } - node.required = required - for _, r := range node.required { - work.Add(r) + if m.Version != "none" { + required, err := reqs.Required(m) + if err != nil { + setErr(node, err) + return + } + node.required = required + for _, r := range node.required { + work.Add(r) + } } if upgrade != nil { @@ -202,18 +152,30 @@ func buildList(target module.Version, reqs Reqs, upgrade func(module.Version) (m q = q[1:] if node.err != nil { - err := &BuildListError{ - Err: node.err, - stack: []buildListErrorElem{{m: node.m}}, - } + pathUpgrade := map[module.Version]module.Version{} + + // Construct the error path reversed (from the error to the main module), + // then reverse it to obtain the usual order (from the main module to + // the error). + errPath := []module.Version{node.m} for n, prev := neededBy[node], node; n != nil; n, prev = neededBy[n], n { - reason := "requires" if n.upgrade == prev.m { - reason = "updating to" + pathUpgrade[n.m] = prev.m } - err.stack = append(err.stack, buildListErrorElem{m: n.m, nextReason: reason}) + errPath = append(errPath, n.m) } - return nil, err + i, j := 0, len(errPath)-1 + for i < j { + errPath[i], errPath[j] = errPath[j], errPath[i] + i++ + j-- + } + + isUpgrade := func(from, to module.Version) bool { + return pathUpgrade[from] == to + } + + return nil, NewBuildListError(node.err, errPath, isUpgrade) } neighbors := node.required @@ -250,6 +212,9 @@ func buildList(target module.Version, reqs Reqs, upgrade func(module.Version) (m n := modGraph[module.Version{Path: path, Version: vers}] required := n.required for _, r := range required { + if r.Version == "none" { + continue + } v := min[r.Path] if r.Path != target.Path && reqs.Max(v, r.Version) != v { panic(fmt.Sprintf("mistake: version %q does not satisfy requirement %+v", v, r)) // TODO: Don't panic. @@ -370,16 +335,36 @@ func Upgrade(target module.Version, reqs Reqs, upgrade ...module.Version) ([]mod if err != nil { return nil, err } - // TODO: Maybe if an error is given, - // rerun with BuildList(upgrade[0], reqs) etc - // to find which ones are the buggy ones. + + pathInList := make(map[string]bool, len(list)) + for _, m := range list { + pathInList[m.Path] = true + } list = append([]module.Version(nil), list...) - list = append(list, upgrade...) - return BuildList(target, &override{target, list, reqs}) + + upgradeTo := make(map[string]string, len(upgrade)) + for _, u := range upgrade { + if !pathInList[u.Path] { + list = append(list, module.Version{Path: u.Path, Version: "none"}) + } + if prev, dup := upgradeTo[u.Path]; dup { + upgradeTo[u.Path] = reqs.Max(prev, u.Version) + } else { + upgradeTo[u.Path] = u.Version + } + } + + return buildList(target, &override{target, list, reqs}, func(m module.Version) (module.Version, error) { + if v, ok := upgradeTo[m.Path]; ok { + return module.Version{Path: m.Path, Version: v}, nil + } + return m, nil + }) } // Downgrade returns a build list for the target module -// in which the given additional modules are downgraded. +// in which the given additional modules are downgraded, +// potentially overriding the requirements of the target. // // The versions to be downgraded may be unreachable from reqs.Latest and // reqs.Previous, but the methods of reqs must otherwise handle such versions diff --git a/cmd/go/_internal_/par/queue.go b/cmd/go/_internal_/par/queue.go new file mode 100644 index 0000000..e0d049c --- /dev/null +++ b/cmd/go/_internal_/par/queue.go @@ -0,0 +1,88 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package par + +import "fmt" + +// Queue manages a set of work items to be executed in parallel. The number of +// active work items is limited, and excess items are queued sequentially. +type Queue struct { + maxActive int + st chan queueState +} + +type queueState struct { + active int // number of goroutines processing work; always nonzero when len(backlog) > 0 + backlog []func() + idle chan struct{} // if non-nil, closed when active becomes 0 +} + +// NewQueue returns a Queue that executes up to maxActive items in parallel. +// +// maxActive must be positive. +func NewQueue(maxActive int) *Queue { + if maxActive < 1 { + panic(fmt.Sprintf("par.NewQueue called with nonpositive limit (%d)", maxActive)) + } + + q := &Queue{ + maxActive: maxActive, + st: make(chan queueState, 1), + } + q.st <- queueState{} + return q +} + +// Add adds f as a work item in the queue. +// +// Add returns immediately, but the queue will be marked as non-idle until after +// f (and any subsequently-added work) has completed. +func (q *Queue) Add(f func()) { + st := <-q.st + if st.active == q.maxActive { + st.backlog = append(st.backlog, f) + q.st <- st + return + } + if st.active == 0 { + // Mark q as non-idle. + st.idle = nil + } + st.active++ + q.st <- st + + go func() { + for { + f() + + st := <-q.st + if len(st.backlog) == 0 { + if st.active--; st.active == 0 && st.idle != nil { + close(st.idle) + } + q.st <- st + return + } + f, st.backlog = st.backlog[0], st.backlog[1:] + q.st <- st + } + }() +} + +// Idle returns a channel that will be closed when q has no (active or enqueued) +// work outstanding. +func (q *Queue) Idle() <-chan struct{} { + st := <-q.st + defer func() { q.st <- st }() + + if st.idle == nil { + st.idle = make(chan struct{}) + if st.active == 0 { + close(st.idle) + } + } + + return st.idle +} diff --git a/cmd/go/_internal_/renameio/renameio.go b/cmd/go/_internal_/renameio/renameio.go index 0c0a66b..1761fee 100644 --- a/cmd/go/_internal_/renameio/renameio.go +++ b/cmd/go/_internal_/renameio/renameio.go @@ -8,6 +8,7 @@ package renameio import ( "bytes" "io" + "io/fs" "math/rand" "os" "path/filepath" @@ -24,18 +25,18 @@ func Pattern(filename string) string { return filepath.Join(filepath.Dir(filename), filepath.Base(filename)+patternSuffix) } -// WriteFile is like ioutil.WriteFile, but first writes data to an arbitrary +// WriteFile is like os.WriteFile, but first writes data to an arbitrary // file in the same directory as filename, then renames it atomically to the // final name. // // That ensures that the final location, if it exists, is always a complete file. -func WriteFile(filename string, data []byte, perm os.FileMode) (err error) { +func WriteFile(filename string, data []byte, perm fs.FileMode) (err error) { return WriteToFile(filename, bytes.NewReader(data), perm) } // WriteToFile is a variant of WriteFile that accepts the data as an io.Reader // instead of a slice. -func WriteToFile(filename string, data io.Reader, perm os.FileMode) (err error) { +func WriteToFile(filename string, data io.Reader, perm fs.FileMode) (err error) { f, err := tempFile(filepath.Dir(filename), filepath.Base(filename), perm) if err != nil { return err @@ -66,7 +67,7 @@ func WriteToFile(filename string, data io.Reader, perm os.FileMode) (err error) return robustio.Rename(f.Name(), filename) } -// ReadFile is like ioutil.ReadFile, but on Windows retries spurious errors that +// ReadFile is like os.ReadFile, but on Windows retries spurious errors that // may occur if the file is concurrently replaced. // // Errors are classified heuristically and retries are bounded, so even this @@ -80,7 +81,7 @@ func ReadFile(filename string) ([]byte, error) { } // tempFile creates a new temporary file with given permission bits. -func tempFile(dir, prefix string, perm os.FileMode) (f *os.File, err error) { +func tempFile(dir, prefix string, perm fs.FileMode) (f *os.File, err error) { for i := 0; i < 10000; i++ { name := filepath.Join(dir, prefix+strconv.Itoa(rand.Intn(1000000000))+patternSuffix) f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, perm) diff --git a/cmd/go/_internal_/robustio/robustio.go b/cmd/go/_internal_/robustio/robustio.go index 76e47ad..ce3dbbd 100644 --- a/cmd/go/_internal_/robustio/robustio.go +++ b/cmd/go/_internal_/robustio/robustio.go @@ -22,7 +22,7 @@ func Rename(oldpath, newpath string) error { return rename(oldpath, newpath) } -// ReadFile is like ioutil.ReadFile, but on Windows retries errors that may +// ReadFile is like os.ReadFile, but on Windows retries errors that may // occur if the file is concurrently replaced. // // (See golang.org/issue/31247 and golang.org/issue/32188.) diff --git a/cmd/go/_internal_/robustio/robustio_flaky.go b/cmd/go/_internal_/robustio/robustio_flaky.go index 3f0d75e..ed65645 100644 --- a/cmd/go/_internal_/robustio/robustio_flaky.go +++ b/cmd/go/_internal_/robustio/robustio_flaky.go @@ -8,7 +8,6 @@ package robustio import ( "errors" - "io/ioutil" "math/rand" "os" "syscall" @@ -70,11 +69,11 @@ func rename(oldpath, newpath string) (err error) { }) } -// readFile is like ioutil.ReadFile, but retries ephemeral errors. +// readFile is like os.ReadFile, but retries ephemeral errors. func readFile(filename string) ([]byte, error) { var b []byte err := retry(func() (err error, mayRetry bool) { - b, err = ioutil.ReadFile(filename) + b, err = os.ReadFile(filename) // Unlike in rename, we do not retry errFileNotFound here: it can occur // as a spurious error, but the file may also genuinely not exist, so the diff --git a/cmd/go/_internal_/search/search.go b/cmd/go/_internal_/search/search.go index e1faa90..61f1ab7 100644 --- a/cmd/go/_internal_/search/search.go +++ b/cmd/go/_internal_/search/search.go @@ -7,8 +7,10 @@ package search import ( "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/fsys" "fmt" "go/build" + "io/fs" "os" "path" "path/filepath" @@ -127,7 +129,7 @@ func (m *Match) MatchPackages() { if m.pattern == "cmd" { root += "cmd" + string(filepath.Separator) } - err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { + err := fsys.Walk(root, func(path string, fi fs.FileInfo, err error) error { if err != nil { return err // Likely a permission error, which could interfere with matching. } @@ -153,8 +155,8 @@ func (m *Match) MatchPackages() { } if !fi.IsDir() { - if fi.Mode()&os.ModeSymlink != 0 && want { - if target, err := os.Stat(path); err == nil && target.IsDir() { + if fi.Mode()&fs.ModeSymlink != 0 && want { + if target, err := fsys.Stat(path); err == nil && target.IsDir() { fmt.Fprintf(os.Stderr, "warning: ignoring symlink %s\n", path) } } @@ -263,7 +265,7 @@ func (m *Match) MatchDirs() { } } - err := filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error { + err := fsys.Walk(dir, func(path string, fi fs.FileInfo, err error) error { if err != nil { return err // Likely a permission error, which could interfere with matching. } @@ -272,7 +274,7 @@ func (m *Match) MatchDirs() { } top := false if path == dir { - // filepath.Walk starts at dir and recurses. For the recursive case, + // Walk starts at dir and recurses. For the recursive case, // the path is the result of filepath.Join, which calls filepath.Clean. // The initial case is not Cleaned, though, so we do this explicitly. // @@ -293,7 +295,7 @@ func (m *Match) MatchDirs() { if !top && cfg.ModulesEnabled { // Ignore other modules found in subdirectories. - if fi, err := os.Stat(filepath.Join(path, "go.mod")); err == nil && !fi.IsDir() { + if fi, err := fsys.Stat(filepath.Join(path, "go.mod")); err == nil && !fi.IsDir() { return filepath.SkipDir } } diff --git a/cmd/go/_internal_/str/path.go b/cmd/go/_internal_/str/path.go index 95d91a3..51ab2af 100644 --- a/cmd/go/_internal_/str/path.go +++ b/cmd/go/_internal_/str/path.go @@ -5,7 +5,6 @@ package str import ( - "path" "path/filepath" "strings" ) @@ -50,47 +49,3 @@ func HasFilePathPrefix(s, prefix string) bool { return s[len(prefix)] == filepath.Separator && s[:len(prefix)] == prefix } } - -// GlobsMatchPath reports whether any path prefix of target -// matches one of the glob patterns (as defined by path.Match) -// in the comma-separated globs list. -// It ignores any empty or malformed patterns in the list. -func GlobsMatchPath(globs, target string) bool { - for globs != "" { - // Extract next non-empty glob in comma-separated list. - var glob string - if i := strings.Index(globs, ","); i >= 0 { - glob, globs = globs[:i], globs[i+1:] - } else { - glob, globs = globs, "" - } - if glob == "" { - continue - } - - // A glob with N+1 path elements (N slashes) needs to be matched - // against the first N+1 path elements of target, - // which end just before the N+1'th slash. - n := strings.Count(glob, "/") - prefix := target - // Walk target, counting slashes, truncating at the N+1'th slash. - for i := 0; i < len(target); i++ { - if target[i] == '/' { - if n == 0 { - prefix = target[:i] - break - } - n-- - } - } - if n > 0 { - // Not enough prefix elements. - continue - } - matched, _ := path.Match(glob, prefix) - if matched { - return true - } - } - return false -} diff --git a/cmd/go/_internal_/str/str.go b/cmd/go/_internal_/str/str.go index 0413ed8..9106ebf 100644 --- a/cmd/go/_internal_/str/str.go +++ b/cmd/go/_internal_/str/str.go @@ -96,6 +96,20 @@ func Contains(x []string, s string) bool { return false } +// Uniq removes consecutive duplicate strings from ss. +func Uniq(ss *[]string) { + if len(*ss) <= 1 { + return + } + uniq := (*ss)[:1] + for _, s := range *ss { + if s != uniq[len(uniq)-1] { + uniq = append(uniq, s) + } + } + *ss = uniq +} + func isSpaceByte(c byte) bool { return c == ' ' || c == '\t' || c == '\n' || c == '\r' } diff --git a/cmd/go/_internal_/trace/trace.go b/cmd/go/_internal_/trace/trace.go new file mode 100644 index 0000000..e9f1a89 --- /dev/null +++ b/cmd/go/_internal_/trace/trace.go @@ -0,0 +1,206 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package trace + +import ( + "github.com/dependabot/gomodules-extracted/cmd/_internal_/traceviewer" + "context" + "encoding/json" + "errors" + "os" + "strings" + "sync/atomic" + "time" +) + +// Constants used in event fields. +// See https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU +// for more details. +const ( + phaseDurationBegin = "B" + phaseDurationEnd = "E" + phaseFlowStart = "s" + phaseFlowEnd = "f" + + bindEnclosingSlice = "e" +) + +var traceStarted int32 + +func getTraceContext(ctx context.Context) (traceContext, bool) { + if atomic.LoadInt32(&traceStarted) == 0 { + return traceContext{}, false + } + v := ctx.Value(traceKey{}) + if v == nil { + return traceContext{}, false + } + return v.(traceContext), true +} + +// StartSpan starts a trace event with the given name. The Span ends when its Done method is called. +func StartSpan(ctx context.Context, name string) (context.Context, *Span) { + tc, ok := getTraceContext(ctx) + if !ok { + return ctx, nil + } + childSpan := &Span{t: tc.t, name: name, tid: tc.tid, start: time.Now()} + tc.t.writeEvent(&traceviewer.Event{ + Name: childSpan.name, + Time: float64(childSpan.start.UnixNano()) / float64(time.Microsecond), + TID: childSpan.tid, + Phase: phaseDurationBegin, + }) + ctx = context.WithValue(ctx, traceKey{}, traceContext{tc.t, tc.tid}) + return ctx, childSpan +} + +// StartGoroutine associates the context with a new Thread ID. The Chrome trace viewer associates each +// trace event with a thread, and doesn't expect events with the same thread id to happen at the +// same time. +func StartGoroutine(ctx context.Context) context.Context { + tc, ok := getTraceContext(ctx) + if !ok { + return ctx + } + return context.WithValue(ctx, traceKey{}, traceContext{tc.t, tc.t.getNextTID()}) +} + +// Flow marks a flow indicating that the 'to' span depends on the 'from' span. +// Flow should be called while the 'to' span is in progress. +func Flow(ctx context.Context, from *Span, to *Span) { + tc, ok := getTraceContext(ctx) + if !ok || from == nil || to == nil { + return + } + + id := tc.t.getNextFlowID() + tc.t.writeEvent(&traceviewer.Event{ + Name: from.name + " -> " + to.name, + Category: "flow", + ID: id, + Time: float64(from.end.UnixNano()) / float64(time.Microsecond), + Phase: phaseFlowStart, + TID: from.tid, + }) + tc.t.writeEvent(&traceviewer.Event{ + Name: from.name + " -> " + to.name, + Category: "flow", // TODO(matloob): Add Category to Flow? + ID: id, + Time: float64(to.start.UnixNano()) / float64(time.Microsecond), + Phase: phaseFlowEnd, + TID: to.tid, + BindPoint: bindEnclosingSlice, + }) +} + +type Span struct { + t *tracer + + name string + tid uint64 + start time.Time + end time.Time +} + +func (s *Span) Done() { + if s == nil { + return + } + s.end = time.Now() + s.t.writeEvent(&traceviewer.Event{ + Name: s.name, + Time: float64(s.end.UnixNano()) / float64(time.Microsecond), + TID: s.tid, + Phase: phaseDurationEnd, + }) +} + +type tracer struct { + file chan traceFile // 1-buffered + + nextTID uint64 + nextFlowID uint64 +} + +func (t *tracer) writeEvent(ev *traceviewer.Event) error { + f := <-t.file + defer func() { t.file <- f }() + var err error + if f.entries == 0 { + _, err = f.sb.WriteString("[\n") + } else { + _, err = f.sb.WriteString(",") + } + f.entries++ + if err != nil { + return nil + } + + if err := f.enc.Encode(ev); err != nil { + return err + } + + // Write event string to output file. + _, err = f.f.WriteString(f.sb.String()) + f.sb.Reset() + return err +} + +func (t *tracer) Close() error { + f := <-t.file + defer func() { t.file <- f }() + + _, firstErr := f.f.WriteString("]") + if err := f.f.Close(); firstErr == nil { + firstErr = err + } + return firstErr +} + +func (t *tracer) getNextTID() uint64 { + return atomic.AddUint64(&t.nextTID, 1) +} + +func (t *tracer) getNextFlowID() uint64 { + return atomic.AddUint64(&t.nextFlowID, 1) +} + +// traceKey is the context key for tracing information. It is unexported to prevent collisions with context keys defined in +// other packages. +type traceKey struct{} + +type traceContext struct { + t *tracer + tid uint64 +} + +// Start starts a trace which writes to the given file. +func Start(ctx context.Context, file string) (context.Context, func() error, error) { + atomic.StoreInt32(&traceStarted, 1) + if file == "" { + return nil, nil, errors.New("no trace file supplied") + } + f, err := os.Create(file) + if err != nil { + return nil, nil, err + } + t := &tracer{file: make(chan traceFile, 1)} + sb := new(strings.Builder) + t.file <- traceFile{ + f: f, + sb: sb, + enc: json.NewEncoder(sb), + } + ctx = context.WithValue(ctx, traceKey{}, traceContext{t: t}) + return ctx, t.Close, nil +} + +type traceFile struct { + f *os.File + sb *strings.Builder + enc *json.Encoder + entries int64 +} diff --git a/cmd/go/_internal_/vcs/discovery.go b/cmd/go/_internal_/vcs/discovery.go new file mode 100644 index 0000000..6b87955 --- /dev/null +++ b/cmd/go/_internal_/vcs/discovery.go @@ -0,0 +1,97 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package vcs + +import ( + "encoding/xml" + "fmt" + "io" + "strings" +) + +// charsetReader returns a reader that converts from the given charset to UTF-8. +// Currently it only supports UTF-8 and ASCII. Otherwise, it returns a meaningful +// error which is printed by go get, so the user can find why the package +// wasn't downloaded if the encoding is not supported. Note that, in +// order to reduce potential errors, ASCII is treated as UTF-8 (i.e. characters +// greater than 0x7f are not rejected). +func charsetReader(charset string, input io.Reader) (io.Reader, error) { + switch strings.ToLower(charset) { + case "utf-8", "ascii": + return input, nil + default: + return nil, fmt.Errorf("can't decode XML document using charset %q", charset) + } +} + +// parseMetaGoImports returns meta imports from the HTML in r. +// Parsing ends at the end of the section or the beginning of the . +func parseMetaGoImports(r io.Reader, mod ModuleMode) ([]metaImport, error) { + d := xml.NewDecoder(r) + d.CharsetReader = charsetReader + d.Strict = false + var imports []metaImport + for { + t, err := d.RawToken() + if err != nil { + if err != io.EOF && len(imports) == 0 { + return nil, err + } + break + } + if e, ok := t.(xml.StartElement); ok && strings.EqualFold(e.Name.Local, "body") { + break + } + if e, ok := t.(xml.EndElement); ok && strings.EqualFold(e.Name.Local, "head") { + break + } + e, ok := t.(xml.StartElement) + if !ok || !strings.EqualFold(e.Name.Local, "meta") { + continue + } + if attrValue(e.Attr, "name") != "go-import" { + continue + } + if f := strings.Fields(attrValue(e.Attr, "content")); len(f) == 3 { + imports = append(imports, metaImport{ + Prefix: f[0], + VCS: f[1], + RepoRoot: f[2], + }) + } + } + + // Extract mod entries if we are paying attention to them. + var list []metaImport + var have map[string]bool + if mod == PreferMod { + have = make(map[string]bool) + for _, m := range imports { + if m.VCS == "mod" { + have[m.Prefix] = true + list = append(list, m) + } + } + } + + // Append non-mod entries, ignoring those superseded by a mod entry. + for _, m := range imports { + if m.VCS != "mod" && !have[m.Prefix] { + list = append(list, m) + } + } + return list, nil +} + +// attrValue returns the attribute value for the case-insensitive key +// `name', or the empty string if nothing is found. +func attrValue(attrs []xml.Attr, name string) string { + for _, a := range attrs { + if strings.EqualFold(a.Name.Local, name) { + return a.Value + } + } + return "" +} diff --git a/cmd/go/_internal_/vcs/vcs.go b/cmd/go/_internal_/vcs/vcs.go new file mode 100644 index 0000000..0ff70a6 --- /dev/null +++ b/cmd/go/_internal_/vcs/vcs.go @@ -0,0 +1,1363 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package vcs + +import ( + "encoding/json" + "errors" + "fmt" + exec "github.com/dependabot/gomodules-extracted/_internal_/execabs" + "github.com/dependabot/gomodules-extracted/_internal_/lazyregexp" + "github.com/dependabot/gomodules-extracted/_internal_/singleflight" + "io/fs" + "log" + urlpkg "net/url" + "os" + "path/filepath" + "regexp" + "strings" + "sync" + + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/base" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/cfg" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/search" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/str" + "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/web" + + "golang.org/x/mod/module" +) + +// A vcsCmd describes how to use a version control system +// like Mercurial, Git, or Subversion. +type Cmd struct { + Name string + Cmd string // name of binary to invoke command + + CreateCmd []string // commands to download a fresh copy of a repository + DownloadCmd []string // commands to download updates into an existing repository + + TagCmd []tagCmd // commands to list tags + TagLookupCmd []tagCmd // commands to lookup tags before running tagSyncCmd + TagSyncCmd []string // commands to sync to specific tag + TagSyncDefault []string // commands to sync to default tag + + Scheme []string + PingCmd string + + RemoteRepo func(v *Cmd, rootDir string) (remoteRepo string, err error) + ResolveRepo func(v *Cmd, rootDir, remoteRepo string) (realRepo string, err error) +} + +var defaultSecureScheme = map[string]bool{ + "https": true, + "git+ssh": true, + "bzr+ssh": true, + "svn+ssh": true, + "ssh": true, +} + +func (v *Cmd) IsSecure(repo string) bool { + u, err := urlpkg.Parse(repo) + if err != nil { + // If repo is not a URL, it's not secure. + return false + } + return v.isSecureScheme(u.Scheme) +} + +func (v *Cmd) isSecureScheme(scheme string) bool { + switch v.Cmd { + case "git": + // GIT_ALLOW_PROTOCOL is an environment variable defined by Git. It is a + // colon-separated list of schemes that are allowed to be used with git + // fetch/clone. Any scheme not mentioned will be considered insecure. + if allow := os.Getenv("GIT_ALLOW_PROTOCOL"); allow != "" { + for _, s := range strings.Split(allow, ":") { + if s == scheme { + return true + } + } + return false + } + } + return defaultSecureScheme[scheme] +} + +// A tagCmd describes a command to list available tags +// that can be passed to tagSyncCmd. +type tagCmd struct { + cmd string // command to list tags + pattern string // regexp to extract tags from list +} + +// vcsList lists the known version control systems +var vcsList = []*Cmd{ + vcsHg, + vcsGit, + vcsSvn, + vcsBzr, + vcsFossil, +} + +// vcsMod is a stub for the "mod" scheme. It's returned by +// repoRootForImportPathDynamic, but is otherwise not treated as a VCS command. +var vcsMod = &Cmd{Name: "mod"} + +// vcsByCmd returns the version control system for the given +// command name (hg, git, svn, bzr). +func vcsByCmd(cmd string) *Cmd { + for _, vcs := range vcsList { + if vcs.Cmd == cmd { + return vcs + } + } + return nil +} + +// vcsHg describes how to use Mercurial. +var vcsHg = &Cmd{ + Name: "Mercurial", + Cmd: "hg", + + CreateCmd: []string{"clone -U -- {repo} {dir}"}, + DownloadCmd: []string{"pull"}, + + // We allow both tag and branch names as 'tags' + // for selecting a version. This lets people have + // a go.release.r60 branch and a go1 branch + // and make changes in both, without constantly + // editing .hgtags. + TagCmd: []tagCmd{ + {"tags", `^(\S+)`}, + {"branches", `^(\S+)`}, + }, + TagSyncCmd: []string{"update -r {tag}"}, + TagSyncDefault: []string{"update default"}, + + Scheme: []string{"https", "http", "ssh"}, + PingCmd: "identify -- {scheme}://{repo}", + RemoteRepo: hgRemoteRepo, +} + +func hgRemoteRepo(vcsHg *Cmd, rootDir string) (remoteRepo string, err error) { + out, err := vcsHg.runOutput(rootDir, "paths default") + if err != nil { + return "", err + } + return strings.TrimSpace(string(out)), nil +} + +// vcsGit describes how to use Git. +var vcsGit = &Cmd{ + Name: "Git", + Cmd: "git", + + CreateCmd: []string{"clone -- {repo} {dir}", "-go-internal-cd {dir} submodule update --init --recursive"}, + DownloadCmd: []string{"pull --ff-only", "submodule update --init --recursive"}, + + TagCmd: []tagCmd{ + // tags/xxx matches a git tag named xxx + // origin/xxx matches a git branch named xxx on the default remote repository + {"show-ref", `(?:tags|origin)/(\S+)$`}, + }, + TagLookupCmd: []tagCmd{ + {"show-ref tags/{tag} origin/{tag}", `((?:tags|origin)/\S+)$`}, + }, + TagSyncCmd: []string{"checkout {tag}", "submodule update --init --recursive"}, + // both createCmd and downloadCmd update the working dir. + // No need to do more here. We used to 'checkout master' + // but that doesn't work if the default branch is not named master. + // DO NOT add 'checkout master' here. + // See golang.org/issue/9032. + TagSyncDefault: []string{"submodule update --init --recursive"}, + + Scheme: []string{"git", "https", "http", "git+ssh", "ssh"}, + + // Leave out the '--' separator in the ls-remote command: git 2.7.4 does not + // support such a separator for that command, and this use should be safe + // without it because the {scheme} value comes from the predefined list above. + // See golang.org/issue/33836. + PingCmd: "ls-remote {scheme}://{repo}", + + RemoteRepo: gitRemoteRepo, +} + +// scpSyntaxRe matches the SCP-like addresses used by Git to access +// repositories by SSH. +var scpSyntaxRe = lazyregexp.New(`^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.*)$`) + +func gitRemoteRepo(vcsGit *Cmd, rootDir string) (remoteRepo string, err error) { + cmd := "config remote.origin.url" + errParse := errors.New("unable to parse output of git " + cmd) + errRemoteOriginNotFound := errors.New("remote origin not found") + outb, err := vcsGit.run1(rootDir, cmd, nil, false) + if err != nil { + // if it doesn't output any message, it means the config argument is correct, + // but the config value itself doesn't exist + if outb != nil && len(outb) == 0 { + return "", errRemoteOriginNotFound + } + return "", err + } + out := strings.TrimSpace(string(outb)) + + var repoURL *urlpkg.URL + if m := scpSyntaxRe.FindStringSubmatch(out); m != nil { + // Match SCP-like syntax and convert it to a URL. + // Eg, "git@github.com:user/repo" becomes + // "ssh://git@github.com/user/repo". + repoURL = &urlpkg.URL{ + Scheme: "ssh", + User: urlpkg.User(m[1]), + Host: m[2], + Path: m[3], + } + } else { + repoURL, err = urlpkg.Parse(out) + if err != nil { + return "", err + } + } + + // Iterate over insecure schemes too, because this function simply + // reports the state of the repo. If we can't see insecure schemes then + // we can't report the actual repo URL. + for _, s := range vcsGit.Scheme { + if repoURL.Scheme == s { + return repoURL.String(), nil + } + } + return "", errParse +} + +// vcsBzr describes how to use Bazaar. +var vcsBzr = &Cmd{ + Name: "Bazaar", + Cmd: "bzr", + + CreateCmd: []string{"branch -- {repo} {dir}"}, + + // Without --overwrite bzr will not pull tags that changed. + // Replace by --overwrite-tags after http://pad.lv/681792 goes in. + DownloadCmd: []string{"pull --overwrite"}, + + TagCmd: []tagCmd{{"tags", `^(\S+)`}}, + TagSyncCmd: []string{"update -r {tag}"}, + TagSyncDefault: []string{"update -r revno:-1"}, + + Scheme: []string{"https", "http", "bzr", "bzr+ssh"}, + PingCmd: "info -- {scheme}://{repo}", + RemoteRepo: bzrRemoteRepo, + ResolveRepo: bzrResolveRepo, +} + +func bzrRemoteRepo(vcsBzr *Cmd, rootDir string) (remoteRepo string, err error) { + outb, err := vcsBzr.runOutput(rootDir, "config parent_location") + if err != nil { + return "", err + } + return strings.TrimSpace(string(outb)), nil +} + +func bzrResolveRepo(vcsBzr *Cmd, rootDir, remoteRepo string) (realRepo string, err error) { + outb, err := vcsBzr.runOutput(rootDir, "info "+remoteRepo) + if err != nil { + return "", err + } + out := string(outb) + + // Expect: + // ... + // (branch root|repository branch): + // ... + + found := false + for _, prefix := range []string{"\n branch root: ", "\n repository branch: "} { + i := strings.Index(out, prefix) + if i >= 0 { + out = out[i+len(prefix):] + found = true + break + } + } + if !found { + return "", fmt.Errorf("unable to parse output of bzr info") + } + + i := strings.Index(out, "\n") + if i < 0 { + return "", fmt.Errorf("unable to parse output of bzr info") + } + out = out[:i] + return strings.TrimSpace(out), nil +} + +// vcsSvn describes how to use Subversion. +var vcsSvn = &Cmd{ + Name: "Subversion", + Cmd: "svn", + + CreateCmd: []string{"checkout -- {repo} {dir}"}, + DownloadCmd: []string{"update"}, + + // There is no tag command in subversion. + // The branch information is all in the path names. + + Scheme: []string{"https", "http", "svn", "svn+ssh"}, + PingCmd: "info -- {scheme}://{repo}", + RemoteRepo: svnRemoteRepo, +} + +func svnRemoteRepo(vcsSvn *Cmd, rootDir string) (remoteRepo string, err error) { + outb, err := vcsSvn.runOutput(rootDir, "info") + if err != nil { + return "", err + } + out := string(outb) + + // Expect: + // + // ... + // URL: + // ... + // + // Note that we're not using the Repository Root line, + // because svn allows checking out subtrees. + // The URL will be the URL of the subtree (what we used with 'svn co') + // while the Repository Root may be a much higher parent. + i := strings.Index(out, "\nURL: ") + if i < 0 { + return "", fmt.Errorf("unable to parse output of svn info") + } + out = out[i+len("\nURL: "):] + i = strings.Index(out, "\n") + if i < 0 { + return "", fmt.Errorf("unable to parse output of svn info") + } + out = out[:i] + return strings.TrimSpace(out), nil +} + +// fossilRepoName is the name go get associates with a fossil repository. In the +// real world the file can be named anything. +const fossilRepoName = ".fossil" + +// vcsFossil describes how to use Fossil (fossil-scm.org) +var vcsFossil = &Cmd{ + Name: "Fossil", + Cmd: "fossil", + + CreateCmd: []string{"-go-internal-mkdir {dir} clone -- {repo} " + filepath.Join("{dir}", fossilRepoName), "-go-internal-cd {dir} open .fossil"}, + DownloadCmd: []string{"up"}, + + TagCmd: []tagCmd{{"tag ls", `(.*)`}}, + TagSyncCmd: []string{"up tag:{tag}"}, + TagSyncDefault: []string{"up trunk"}, + + Scheme: []string{"https", "http"}, + RemoteRepo: fossilRemoteRepo, +} + +func fossilRemoteRepo(vcsFossil *Cmd, rootDir string) (remoteRepo string, err error) { + out, err := vcsFossil.runOutput(rootDir, "remote-url") + if err != nil { + return "", err + } + return strings.TrimSpace(string(out)), nil +} + +func (v *Cmd) String() string { + return v.Name +} + +// run runs the command line cmd in the given directory. +// keyval is a list of key, value pairs. run expands +// instances of {key} in cmd into value, but only after +// splitting cmd into individual arguments. +// If an error occurs, run prints the command line and the +// command's combined stdout+stderr to standard error. +// Otherwise run discards the command's output. +func (v *Cmd) run(dir string, cmd string, keyval ...string) error { + _, err := v.run1(dir, cmd, keyval, true) + return err +} + +// runVerboseOnly is like run but only generates error output to standard error in verbose mode. +func (v *Cmd) runVerboseOnly(dir string, cmd string, keyval ...string) error { + _, err := v.run1(dir, cmd, keyval, false) + return err +} + +// runOutput is like run but returns the output of the command. +func (v *Cmd) runOutput(dir string, cmd string, keyval ...string) ([]byte, error) { + return v.run1(dir, cmd, keyval, true) +} + +// run1 is the generalized implementation of run and runOutput. +func (v *Cmd) run1(dir string, cmdline string, keyval []string, verbose bool) ([]byte, error) { + m := make(map[string]string) + for i := 0; i < len(keyval); i += 2 { + m[keyval[i]] = keyval[i+1] + } + args := strings.Fields(cmdline) + for i, arg := range args { + args[i] = expand(m, arg) + } + + if len(args) >= 2 && args[0] == "-go-internal-mkdir" { + var err error + if filepath.IsAbs(args[1]) { + err = os.Mkdir(args[1], fs.ModePerm) + } else { + err = os.Mkdir(filepath.Join(dir, args[1]), fs.ModePerm) + } + if err != nil { + return nil, err + } + args = args[2:] + } + + if len(args) >= 2 && args[0] == "-go-internal-cd" { + if filepath.IsAbs(args[1]) { + dir = args[1] + } else { + dir = filepath.Join(dir, args[1]) + } + args = args[2:] + } + + _, err := exec.LookPath(v.Cmd) + if err != nil { + fmt.Fprintf(os.Stderr, + "go: missing %s command. See https://golang.org/s/gogetcmd\n", + v.Name) + return nil, err + } + + cmd := exec.Command(v.Cmd, args...) + cmd.Dir = dir + cmd.Env = base.AppendPWD(os.Environ(), cmd.Dir) + if cfg.BuildX { + fmt.Fprintf(os.Stderr, "cd %s\n", dir) + fmt.Fprintf(os.Stderr, "%s %s\n", v.Cmd, strings.Join(args, " ")) + } + out, err := cmd.Output() + if err != nil { + if verbose || cfg.BuildV { + fmt.Fprintf(os.Stderr, "# cd %s; %s %s\n", dir, v.Cmd, strings.Join(args, " ")) + if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 { + os.Stderr.Write(ee.Stderr) + } else { + fmt.Fprintf(os.Stderr, err.Error()) + } + } + } + return out, err +} + +// Ping pings to determine scheme to use. +func (v *Cmd) Ping(scheme, repo string) error { + return v.runVerboseOnly(".", v.PingCmd, "scheme", scheme, "repo", repo) +} + +// Create creates a new copy of repo in dir. +// The parent of dir must exist; dir must not. +func (v *Cmd) Create(dir, repo string) error { + for _, cmd := range v.CreateCmd { + if err := v.run(".", cmd, "dir", dir, "repo", repo); err != nil { + return err + } + } + return nil +} + +// Download downloads any new changes for the repo in dir. +func (v *Cmd) Download(dir string) error { + for _, cmd := range v.DownloadCmd { + if err := v.run(dir, cmd); err != nil { + return err + } + } + return nil +} + +// Tags returns the list of available tags for the repo in dir. +func (v *Cmd) Tags(dir string) ([]string, error) { + var tags []string + for _, tc := range v.TagCmd { + out, err := v.runOutput(dir, tc.cmd) + if err != nil { + return nil, err + } + re := regexp.MustCompile(`(?m-s)` + tc.pattern) + for _, m := range re.FindAllStringSubmatch(string(out), -1) { + tags = append(tags, m[1]) + } + } + return tags, nil +} + +// tagSync syncs the repo in dir to the named tag, +// which either is a tag returned by tags or is v.tagDefault. +func (v *Cmd) TagSync(dir, tag string) error { + if v.TagSyncCmd == nil { + return nil + } + if tag != "" { + for _, tc := range v.TagLookupCmd { + out, err := v.runOutput(dir, tc.cmd, "tag", tag) + if err != nil { + return err + } + re := regexp.MustCompile(`(?m-s)` + tc.pattern) + m := re.FindStringSubmatch(string(out)) + if len(m) > 1 { + tag = m[1] + break + } + } + } + + if tag == "" && v.TagSyncDefault != nil { + for _, cmd := range v.TagSyncDefault { + if err := v.run(dir, cmd); err != nil { + return err + } + } + return nil + } + + for _, cmd := range v.TagSyncCmd { + if err := v.run(dir, cmd, "tag", tag); err != nil { + return err + } + } + return nil +} + +// A vcsPath describes how to convert an import path into a +// version control system and repository name. +type vcsPath struct { + pathPrefix string // prefix this description applies to + regexp *lazyregexp.Regexp // compiled pattern for import path + repo string // repository to use (expand with match of re) + vcs string // version control system to use (expand with match of re) + check func(match map[string]string) error // additional checks + schemelessRepo bool // if true, the repo pattern lacks a scheme +} + +// FromDir inspects dir and its parents to determine the +// version control system and code repository to use. +// On return, root is the import path +// corresponding to the root of the repository. +func FromDir(dir, srcRoot string) (vcs *Cmd, root string, err error) { + // Clean and double-check that dir is in (a subdirectory of) srcRoot. + dir = filepath.Clean(dir) + srcRoot = filepath.Clean(srcRoot) + if len(dir) <= len(srcRoot) || dir[len(srcRoot)] != filepath.Separator { + return nil, "", fmt.Errorf("directory %q is outside source root %q", dir, srcRoot) + } + + var vcsRet *Cmd + var rootRet string + + origDir := dir + for len(dir) > len(srcRoot) { + for _, vcs := range vcsList { + if _, err := os.Stat(filepath.Join(dir, "."+vcs.Cmd)); err == nil { + root := filepath.ToSlash(dir[len(srcRoot)+1:]) + // Record first VCS we find, but keep looking, + // to detect mistakes like one kind of VCS inside another. + if vcsRet == nil { + vcsRet = vcs + rootRet = root + continue + } + // Allow .git inside .git, which can arise due to submodules. + if vcsRet == vcs && vcs.Cmd == "git" { + continue + } + // Otherwise, we have one VCS inside a different VCS. + return nil, "", fmt.Errorf("directory %q uses %s, but parent %q uses %s", + filepath.Join(srcRoot, rootRet), vcsRet.Cmd, filepath.Join(srcRoot, root), vcs.Cmd) + } + } + + // Move to parent. + ndir := filepath.Dir(dir) + if len(ndir) >= len(dir) { + // Shouldn't happen, but just in case, stop. + break + } + dir = ndir + } + + if vcsRet != nil { + if err := checkGOVCS(vcsRet, rootRet); err != nil { + return nil, "", err + } + return vcsRet, rootRet, nil + } + + return nil, "", fmt.Errorf("directory %q is not using a known version control system", origDir) +} + +// A govcsRule is a single GOVCS rule like private:hg|svn. +type govcsRule struct { + pattern string + allowed []string +} + +// A govcsConfig is a full GOVCS configuration. +type govcsConfig []govcsRule + +func parseGOVCS(s string) (govcsConfig, error) { + s = strings.TrimSpace(s) + if s == "" { + return nil, nil + } + var cfg govcsConfig + have := make(map[string]string) + for _, item := range strings.Split(s, ",") { + item = strings.TrimSpace(item) + if item == "" { + return nil, fmt.Errorf("empty entry in GOVCS") + } + i := strings.Index(item, ":") + if i < 0 { + return nil, fmt.Errorf("malformed entry in GOVCS (missing colon): %q", item) + } + pattern, list := strings.TrimSpace(item[:i]), strings.TrimSpace(item[i+1:]) + if pattern == "" { + return nil, fmt.Errorf("empty pattern in GOVCS: %q", item) + } + if list == "" { + return nil, fmt.Errorf("empty VCS list in GOVCS: %q", item) + } + if search.IsRelativePath(pattern) { + return nil, fmt.Errorf("relative pattern not allowed in GOVCS: %q", pattern) + } + if old := have[pattern]; old != "" { + return nil, fmt.Errorf("unreachable pattern in GOVCS: %q after %q", item, old) + } + have[pattern] = item + allowed := strings.Split(list, "|") + for i, a := range allowed { + a = strings.TrimSpace(a) + if a == "" { + return nil, fmt.Errorf("empty VCS name in GOVCS: %q", item) + } + allowed[i] = a + } + cfg = append(cfg, govcsRule{pattern, allowed}) + } + return cfg, nil +} + +func (c *govcsConfig) allow(path string, private bool, vcs string) bool { + for _, rule := range *c { + match := false + switch rule.pattern { + case "private": + match = private + case "public": + match = !private + default: + // Note: rule.pattern is known to be comma-free, + // so MatchPrefixPatterns is only matching a single pattern for us. + match = module.MatchPrefixPatterns(rule.pattern, path) + } + if !match { + continue + } + for _, allow := range rule.allowed { + if allow == vcs || allow == "all" { + return true + } + } + return false + } + + // By default, nothing is allowed. + return false +} + +var ( + govcs govcsConfig + govcsErr error + govcsOnce sync.Once +) + +// defaultGOVCS is the default setting for GOVCS. +// Setting GOVCS adds entries ahead of these but does not remove them. +// (They are appended to the parsed GOVCS setting.) +// +// The rationale behind allowing only Git and Mercurial is that +// these two systems have had the most attention to issues +// of being run as clients of untrusted servers. In contrast, +// Bazaar, Fossil, and Subversion have primarily been used +// in trusted, authenticated environments and are not as well +// scrutinized as attack surfaces. +// +// See golang.org/issue/41730 for details. +var defaultGOVCS = govcsConfig{ + {"private", []string{"all"}}, + {"public", []string{"git", "hg"}}, +} + +func checkGOVCS(vcs *Cmd, root string) error { + if vcs == vcsMod { + // Direct module (proxy protocol) fetches don't + // involve an external version control system + // and are always allowed. + return nil + } + + govcsOnce.Do(func() { + govcs, govcsErr = parseGOVCS(os.Getenv("GOVCS")) + govcs = append(govcs, defaultGOVCS...) + }) + if govcsErr != nil { + return govcsErr + } + + private := module.MatchPrefixPatterns(cfg.GOPRIVATE, root) + if !govcs.allow(root, private, vcs.Cmd) { + what := "public" + if private { + what = "private" + } + return fmt.Errorf("GOVCS disallows using %s for %s %s; see 'go help vcs'", vcs.Cmd, what, root) + } + + return nil +} + +// CheckNested checks for an incorrectly-nested VCS-inside-VCS +// situation for dir, checking parents up until srcRoot. +func CheckNested(vcs *Cmd, dir, srcRoot string) error { + if len(dir) <= len(srcRoot) || dir[len(srcRoot)] != filepath.Separator { + return fmt.Errorf("directory %q is outside source root %q", dir, srcRoot) + } + + otherDir := dir + for len(otherDir) > len(srcRoot) { + for _, otherVCS := range vcsList { + if _, err := os.Stat(filepath.Join(otherDir, "."+otherVCS.Cmd)); err == nil { + // Allow expected vcs in original dir. + if otherDir == dir && otherVCS == vcs { + continue + } + // Allow .git inside .git, which can arise due to submodules. + if otherVCS == vcs && vcs.Cmd == "git" { + continue + } + // Otherwise, we have one VCS inside a different VCS. + return fmt.Errorf("directory %q uses %s, but parent %q uses %s", dir, vcs.Cmd, otherDir, otherVCS.Cmd) + } + } + // Move to parent. + newDir := filepath.Dir(otherDir) + if len(newDir) >= len(otherDir) { + // Shouldn't happen, but just in case, stop. + break + } + otherDir = newDir + } + + return nil +} + +// RepoRoot describes the repository root for a tree of source code. +type RepoRoot struct { + Repo string // repository URL, including scheme + Root string // import path corresponding to root of repo + IsCustom bool // defined by served tags (as opposed to hard-coded pattern) + VCS *Cmd +} + +func httpPrefix(s string) string { + for _, prefix := range [...]string{"http:", "https:"} { + if strings.HasPrefix(s, prefix) { + return prefix + } + } + return "" +} + +// ModuleMode specifies whether to prefer modules when looking up code sources. +type ModuleMode int + +const ( + IgnoreMod ModuleMode = iota + PreferMod +) + +// RepoRootForImportPath analyzes importPath to determine the +// version control system, and code repository to use. +func RepoRootForImportPath(importPath string, mod ModuleMode, security web.SecurityMode) (*RepoRoot, error) { + rr, err := repoRootFromVCSPaths(importPath, security, vcsPaths) + if err == errUnknownSite { + rr, err = repoRootForImportDynamic(importPath, mod, security) + if err != nil { + err = importErrorf(importPath, "unrecognized import path %q: %v", importPath, err) + } + } + if err != nil { + rr1, err1 := repoRootFromVCSPaths(importPath, security, vcsPathsAfterDynamic) + if err1 == nil { + rr = rr1 + err = nil + } + } + + // Should have been taken care of above, but make sure. + if err == nil && strings.Contains(importPath, "...") && strings.Contains(rr.Root, "...") { + // Do not allow wildcards in the repo root. + rr = nil + err = importErrorf(importPath, "cannot expand ... in %q", importPath) + } + return rr, err +} + +var errUnknownSite = errors.New("dynamic lookup required to find mapping") + +// repoRootFromVCSPaths attempts to map importPath to a repoRoot +// using the mappings defined in vcsPaths. +func repoRootFromVCSPaths(importPath string, security web.SecurityMode, vcsPaths []*vcsPath) (*RepoRoot, error) { + if str.HasPathPrefix(importPath, "example.net") { + // TODO(rsc): This should not be necessary, but it's required to keep + // tests like ../../testdata/script/mod_get_extra.txt from using the network. + // That script has everything it needs in the replacement set, but it is still + // doing network calls. + return nil, fmt.Errorf("no modules on example.net") + } + if importPath == "rsc.io" { + // This special case allows tests like ../../testdata/script/govcs.txt + // to avoid making any network calls. The module lookup for a path + // like rsc.io/nonexist.svn/foo needs to not make a network call for + // a lookup on rsc.io. + return nil, fmt.Errorf("rsc.io is not a module") + } + // A common error is to use https://packagepath because that's what + // hg and git require. Diagnose this helpfully. + if prefix := httpPrefix(importPath); prefix != "" { + // The importPath has been cleaned, so has only one slash. The pattern + // ignores the slashes; the error message puts them back on the RHS at least. + return nil, fmt.Errorf("%q not allowed in import path", prefix+"//") + } + for _, srv := range vcsPaths { + if !str.HasPathPrefix(importPath, srv.pathPrefix) { + continue + } + m := srv.regexp.FindStringSubmatch(importPath) + if m == nil { + if srv.pathPrefix != "" { + return nil, importErrorf(importPath, "invalid %s import path %q", srv.pathPrefix, importPath) + } + continue + } + + // Build map of named subexpression matches for expand. + match := map[string]string{ + "prefix": srv.pathPrefix + "/", + "import": importPath, + } + for i, name := range srv.regexp.SubexpNames() { + if name != "" && match[name] == "" { + match[name] = m[i] + } + } + if srv.vcs != "" { + match["vcs"] = expand(match, srv.vcs) + } + if srv.repo != "" { + match["repo"] = expand(match, srv.repo) + } + if srv.check != nil { + if err := srv.check(match); err != nil { + return nil, err + } + } + vcs := vcsByCmd(match["vcs"]) + if vcs == nil { + return nil, fmt.Errorf("unknown version control system %q", match["vcs"]) + } + if err := checkGOVCS(vcs, match["root"]); err != nil { + return nil, err + } + var repoURL string + if !srv.schemelessRepo { + repoURL = match["repo"] + } else { + scheme := vcs.Scheme[0] // default to first scheme + repo := match["repo"] + if vcs.PingCmd != "" { + // If we know how to test schemes, scan to find one. + for _, s := range vcs.Scheme { + if security == web.SecureOnly && !vcs.isSecureScheme(s) { + continue + } + if vcs.Ping(s, repo) == nil { + scheme = s + break + } + } + } + repoURL = scheme + "://" + repo + } + rr := &RepoRoot{ + Repo: repoURL, + Root: match["root"], + VCS: vcs, + } + return rr, nil + } + return nil, errUnknownSite +} + +// urlForImportPath returns a partially-populated URL for the given Go import path. +// +// The URL leaves the Scheme field blank so that web.Get will try any scheme +// allowed by the selected security mode. +func urlForImportPath(importPath string) (*urlpkg.URL, error) { + slash := strings.Index(importPath, "/") + if slash < 0 { + slash = len(importPath) + } + host, path := importPath[:slash], importPath[slash:] + if !strings.Contains(host, ".") { + return nil, errors.New("import path does not begin with hostname") + } + if len(path) == 0 { + path = "/" + } + return &urlpkg.URL{Host: host, Path: path, RawQuery: "go-get=1"}, nil +} + +// repoRootForImportDynamic finds a *RepoRoot for a custom domain that's not +// statically known by repoRootForImportPathStatic. +// +// This handles custom import paths like "name.tld/pkg/foo" or just "name.tld". +func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.SecurityMode) (*RepoRoot, error) { + url, err := urlForImportPath(importPath) + if err != nil { + return nil, err + } + resp, err := web.Get(security, url) + if err != nil { + msg := "https fetch: %v" + if security == web.Insecure { + msg = "http/" + msg + } + return nil, fmt.Errorf(msg, err) + } + body := resp.Body + defer body.Close() + imports, err := parseMetaGoImports(body, mod) + if len(imports) == 0 { + if respErr := resp.Err(); respErr != nil { + // If the server's status was not OK, prefer to report that instead of + // an XML parse error. + return nil, respErr + } + } + if err != nil { + return nil, fmt.Errorf("parsing %s: %v", importPath, err) + } + // Find the matched meta import. + mmi, err := matchGoImport(imports, importPath) + if err != nil { + if _, ok := err.(ImportMismatchError); !ok { + return nil, fmt.Errorf("parse %s: %v", url, err) + } + return nil, fmt.Errorf("parse %s: no go-import meta tags (%s)", resp.URL, err) + } + if cfg.BuildV { + log.Printf("get %q: found meta tag %#v at %s", importPath, mmi, url) + } + // If the import was "uni.edu/bob/project", which said the + // prefix was "uni.edu" and the RepoRoot was "evilroot.com", + // make sure we don't trust Bob and check out evilroot.com to + // "uni.edu" yet (possibly overwriting/preempting another + // non-evil student). Instead, first verify the root and see + // if it matches Bob's claim. + if mmi.Prefix != importPath { + if cfg.BuildV { + log.Printf("get %q: verifying non-authoritative meta tag", importPath) + } + var imports []metaImport + url, imports, err = metaImportsForPrefix(mmi.Prefix, mod, security) + if err != nil { + return nil, err + } + metaImport2, err := matchGoImport(imports, importPath) + if err != nil || mmi != metaImport2 { + return nil, fmt.Errorf("%s and %s disagree about go-import for %s", resp.URL, url, mmi.Prefix) + } + } + + if err := validateRepoRoot(mmi.RepoRoot); err != nil { + return nil, fmt.Errorf("%s: invalid repo root %q: %v", resp.URL, mmi.RepoRoot, err) + } + var vcs *Cmd + if mmi.VCS == "mod" { + vcs = vcsMod + } else { + vcs = vcsByCmd(mmi.VCS) + if vcs == nil { + return nil, fmt.Errorf("%s: unknown vcs %q", resp.URL, mmi.VCS) + } + } + + if err := checkGOVCS(vcs, mmi.Prefix); err != nil { + return nil, err + } + + rr := &RepoRoot{ + Repo: mmi.RepoRoot, + Root: mmi.Prefix, + IsCustom: true, + VCS: vcs, + } + return rr, nil +} + +// validateRepoRoot returns an error if repoRoot does not seem to be +// a valid URL with scheme. +func validateRepoRoot(repoRoot string) error { + url, err := urlpkg.Parse(repoRoot) + if err != nil { + return err + } + if url.Scheme == "" { + return errors.New("no scheme") + } + if url.Scheme == "file" { + return errors.New("file scheme disallowed") + } + return nil +} + +var fetchGroup singleflight.Group +var ( + fetchCacheMu sync.Mutex + fetchCache = map[string]fetchResult{} // key is metaImportsForPrefix's importPrefix +) + +// metaImportsForPrefix takes a package's root import path as declared in a tag +// and returns its HTML discovery URL and the parsed metaImport lines +// found on the page. +// +// The importPath is of the form "golang.org/x/tools". +// It is an error if no imports are found. +// url will still be valid if err != nil. +// The returned url will be of the form "https://golang.org/x/tools?go-get=1" +func metaImportsForPrefix(importPrefix string, mod ModuleMode, security web.SecurityMode) (*urlpkg.URL, []metaImport, error) { + setCache := func(res fetchResult) (fetchResult, error) { + fetchCacheMu.Lock() + defer fetchCacheMu.Unlock() + fetchCache[importPrefix] = res + return res, nil + } + + resi, _, _ := fetchGroup.Do(importPrefix, func() (resi interface{}, err error) { + fetchCacheMu.Lock() + if res, ok := fetchCache[importPrefix]; ok { + fetchCacheMu.Unlock() + return res, nil + } + fetchCacheMu.Unlock() + + url, err := urlForImportPath(importPrefix) + if err != nil { + return setCache(fetchResult{err: err}) + } + resp, err := web.Get(security, url) + if err != nil { + return setCache(fetchResult{url: url, err: fmt.Errorf("fetching %s: %v", importPrefix, err)}) + } + body := resp.Body + defer body.Close() + imports, err := parseMetaGoImports(body, mod) + if len(imports) == 0 { + if respErr := resp.Err(); respErr != nil { + // If the server's status was not OK, prefer to report that instead of + // an XML parse error. + return setCache(fetchResult{url: url, err: respErr}) + } + } + if err != nil { + return setCache(fetchResult{url: url, err: fmt.Errorf("parsing %s: %v", resp.URL, err)}) + } + if len(imports) == 0 { + err = fmt.Errorf("fetching %s: no go-import meta tag found in %s", importPrefix, resp.URL) + } + return setCache(fetchResult{url: url, imports: imports, err: err}) + }) + res := resi.(fetchResult) + return res.url, res.imports, res.err +} + +type fetchResult struct { + url *urlpkg.URL + imports []metaImport + err error +} + +// metaImport represents the parsed tags from HTML files. +type metaImport struct { + Prefix, VCS, RepoRoot string +} + +// A ImportMismatchError is returned where metaImport/s are present +// but none match our import path. +type ImportMismatchError struct { + importPath string + mismatches []string // the meta imports that were discarded for not matching our importPath +} + +func (m ImportMismatchError) Error() string { + formattedStrings := make([]string, len(m.mismatches)) + for i, pre := range m.mismatches { + formattedStrings[i] = fmt.Sprintf("meta tag %s did not match import path %s", pre, m.importPath) + } + return strings.Join(formattedStrings, ", ") +} + +// matchGoImport returns the metaImport from imports matching importPath. +// An error is returned if there are multiple matches. +// An ImportMismatchError is returned if none match. +func matchGoImport(imports []metaImport, importPath string) (metaImport, error) { + match := -1 + + errImportMismatch := ImportMismatchError{importPath: importPath} + for i, im := range imports { + if !str.HasPathPrefix(importPath, im.Prefix) { + errImportMismatch.mismatches = append(errImportMismatch.mismatches, im.Prefix) + continue + } + + if match >= 0 { + if imports[match].VCS == "mod" && im.VCS != "mod" { + // All the mod entries precede all the non-mod entries. + // We have a mod entry and don't care about the rest, + // matching or not. + break + } + return metaImport{}, fmt.Errorf("multiple meta tags match import path %q", importPath) + } + match = i + } + + if match == -1 { + return metaImport{}, errImportMismatch + } + return imports[match], nil +} + +// expand rewrites s to replace {k} with match[k] for each key k in match. +func expand(match map[string]string, s string) string { + // We want to replace each match exactly once, and the result of expansion + // must not depend on the iteration order through the map. + // A strings.Replacer has exactly the properties we're looking for. + oldNew := make([]string, 0, 2*len(match)) + for k, v := range match { + oldNew = append(oldNew, "{"+k+"}", v) + } + return strings.NewReplacer(oldNew...).Replace(s) +} + +// vcsPaths defines the meaning of import paths referring to +// commonly-used VCS hosting sites (github.com/user/dir) +// and import paths referring to a fully-qualified importPath +// containing a VCS type (foo.com/repo.git/dir) +var vcsPaths = []*vcsPath{ + // Github + { + pathPrefix: "github.com", + regexp: lazyregexp.New(`^(?Pgithub\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`), + vcs: "git", + repo: "https://{root}", + check: noVCSSuffix, + }, + + // Bitbucket + { + pathPrefix: "bitbucket.org", + regexp: lazyregexp.New(`^(?Pbitbucket\.org/(?P[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`), + repo: "https://{root}", + check: bitbucketVCS, + }, + + // IBM DevOps Services (JazzHub) + { + pathPrefix: "hub.jazz.net/git", + regexp: lazyregexp.New(`^(?Phub\.jazz\.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`), + vcs: "git", + repo: "https://{root}", + check: noVCSSuffix, + }, + + // Git at Apache + { + pathPrefix: "git.apache.org", + regexp: lazyregexp.New(`^(?Pgit\.apache\.org/[a-z0-9_.\-]+\.git)(/[A-Za-z0-9_.\-]+)*$`), + vcs: "git", + repo: "https://{root}", + }, + + // Git at OpenStack + { + pathPrefix: "git.openstack.org", + regexp: lazyregexp.New(`^(?Pgit\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/[A-Za-z0-9_.\-]+)*$`), + vcs: "git", + repo: "https://{root}", + }, + + // chiselapp.com for fossil + { + pathPrefix: "chiselapp.com", + regexp: lazyregexp.New(`^(?Pchiselapp\.com/user/[A-Za-z0-9]+/repository/[A-Za-z0-9_.\-]+)$`), + vcs: "fossil", + repo: "https://{root}", + }, + + // General syntax for any server. + // Must be last. + { + regexp: lazyregexp.New(`(?P(?P([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\-]+)+?)\.(?Pbzr|fossil|git|hg|svn))(/~?[A-Za-z0-9_.\-]+)*$`), + schemelessRepo: true, + }, +} + +// vcsPathsAfterDynamic gives additional vcsPaths entries +// to try after the dynamic HTML check. +// This gives those sites a chance to introduce tags +// as part of a graceful transition away from the hard-coded logic. +var vcsPathsAfterDynamic = []*vcsPath{ + // Launchpad. See golang.org/issue/11436. + { + pathPrefix: "launchpad.net", + regexp: lazyregexp.New(`^(?Plaunchpad\.net/((?P[A-Za-z0-9_.\-]+)(?P/[A-Za-z0-9_.\-]+)?|~[A-Za-z0-9_.\-]+/(\+junk|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`), + vcs: "bzr", + repo: "https://{root}", + check: launchpadVCS, + }, +} + +// noVCSSuffix checks that the repository name does not +// end in .foo for any version control system foo. +// The usual culprit is ".git". +func noVCSSuffix(match map[string]string) error { + repo := match["repo"] + for _, vcs := range vcsList { + if strings.HasSuffix(repo, "."+vcs.Cmd) { + return fmt.Errorf("invalid version control suffix in %s path", match["prefix"]) + } + } + return nil +} + +// bitbucketVCS determines the version control system for a +// Bitbucket repository, by using the Bitbucket API. +func bitbucketVCS(match map[string]string) error { + if err := noVCSSuffix(match); err != nil { + return err + } + + var resp struct { + SCM string `json:"scm"` + } + url := &urlpkg.URL{ + Scheme: "https", + Host: "api.bitbucket.org", + Path: expand(match, "/2.0/repositories/{bitname}"), + RawQuery: "fields=scm", + } + data, err := web.GetBytes(url) + if err != nil { + if httpErr, ok := err.(*web.HTTPError); ok && httpErr.StatusCode == 403 { + // this may be a private repository. If so, attempt to determine which + // VCS it uses. See issue 5375. + root := match["root"] + for _, vcs := range []string{"git", "hg"} { + if vcsByCmd(vcs).Ping("https", root) == nil { + resp.SCM = vcs + break + } + } + } + + if resp.SCM == "" { + return err + } + } else { + if err := json.Unmarshal(data, &resp); err != nil { + return fmt.Errorf("decoding %s: %v", url, err) + } + } + + if vcsByCmd(resp.SCM) != nil { + match["vcs"] = resp.SCM + if resp.SCM == "git" { + match["repo"] += ".git" + } + return nil + } + + return fmt.Errorf("unable to detect version control system for bitbucket.org/ path") +} + +// launchpadVCS solves the ambiguity for "lp.net/project/foo". In this case, +// "foo" could be a series name registered in Launchpad with its own branch, +// and it could also be the name of a directory within the main project +// branch one level up. +func launchpadVCS(match map[string]string) error { + if match["project"] == "" || match["series"] == "" { + return nil + } + url := &urlpkg.URL{ + Scheme: "https", + Host: "code.launchpad.net", + Path: expand(match, "/{project}{series}/.bzr/branch-format"), + } + _, err := web.GetBytes(url) + if err != nil { + match["root"] = expand(match, "launchpad.net/{project}") + match["repo"] = expand(match, "https://{root}") + } + return nil +} + +// importError is a copy of load.importError, made to avoid a dependency cycle +// on cmd/go/internal/load. It just needs to satisfy load.ImportPathError. +type importError struct { + importPath string + err error +} + +func importErrorf(path, format string, args ...interface{}) error { + err := &importError{importPath: path, err: fmt.Errorf(format, args...)} + if errStr := err.Error(); !strings.Contains(errStr, path) { + panic(fmt.Sprintf("path %q not in error %q", path, errStr)) + } + return err +} + +func (e *importError) Error() string { + return e.err.Error() +} + +func (e *importError) Unwrap() error { + // Don't return e.err directly, since we're only wrapping an error if %w + // was passed to ImportErrorf. + return errors.Unwrap(e.err) +} + +func (e *importError) ImportPath() string { + return e.importPath +} diff --git a/cmd/go/_internal_/web/api.go b/cmd/go/_internal_/web/api.go index f6c9aae..6910b99 100644 --- a/cmd/go/_internal_/web/api.go +++ b/cmd/go/_internal_/web/api.go @@ -13,9 +13,8 @@ import ( "bytes" "fmt" "io" - "io/ioutil" + "io/fs" "net/url" - "os" "strings" "unicode" "unicode/utf8" @@ -56,7 +55,7 @@ func (e *HTTPError) Error() string { } if err := e.Err; err != nil { - if pErr, ok := e.Err.(*os.PathError); ok && strings.HasSuffix(e.URL, pErr.Path) { + if pErr, ok := e.Err.(*fs.PathError); ok && strings.HasSuffix(e.URL, pErr.Path) { // Remove the redundant copy of the path. err = pErr.Err } @@ -67,7 +66,7 @@ func (e *HTTPError) Error() string { } func (e *HTTPError) Is(target error) bool { - return target == os.ErrNotExist && (e.StatusCode == 404 || e.StatusCode == 410) + return target == fs.ErrNotExist && (e.StatusCode == 404 || e.StatusCode == 410) } func (e *HTTPError) Unwrap() error { @@ -87,7 +86,7 @@ func GetBytes(u *url.URL) ([]byte, error) { if err := resp.Err(); err != nil { return nil, err } - b, err := ioutil.ReadAll(resp.Body) + b, err := io.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("reading %s: %v", u.Redacted(), err) } @@ -130,7 +129,7 @@ func (r *Response) formatErrorDetail() string { } // Ensure that r.errorDetail has been populated. - _, _ = io.Copy(ioutil.Discard, r.Body) + _, _ = io.Copy(io.Discard, r.Body) s := r.errorDetail.buf.String() if !utf8.ValidString(s) { diff --git a/cmd/go/_internal_/web/http.go b/cmd/go/_internal_/web/http.go index 0f5525b..5760da8 100644 --- a/cmd/go/_internal_/web/http.go +++ b/cmd/go/_internal_/web/http.go @@ -80,6 +80,13 @@ func get(security SecurityMode, url *urlpkg.URL) (*Response, error) { return res, nil } + if url.Host == "localhost.localdev" { + return nil, fmt.Errorf("no such host localhost.localdev") + } + if os.Getenv("TESTGONETWORK") == "panic" && !strings.HasPrefix(url.Host, "127.0.0.1") && !strings.HasPrefix(url.Host, "0.0.0.0") { + panic("use of network: " + url.String()) + } + fetch := func(url *urlpkg.URL) (*urlpkg.URL, *http.Response, error) { // Note: The -v build flag does not mean "print logging information", // despite its historical misuse for this in GOPATH-based go get. From 45a865fbc04a05c8927a6e37a8d3ac11422f60ea Mon Sep 17 00:00:00 2001 From: Jeff Widman Date: Tue, 27 Apr 2021 22:30:36 -0700 Subject: [PATCH 2/2] Update go.mod to go 1.16 --- extract/go.mod | 2 +- extract/go.sum | 2 -- go.mod | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/extract/go.mod b/extract/go.mod index dca9cb0..2c5af73 100644 --- a/extract/go.mod +++ b/extract/go.mod @@ -1,5 +1,5 @@ module github.com/dependabot/gomodules-extracted/extract -go 1.15 +go 1.16 require github.com/hmarr/pkgextract v0.1.0 diff --git a/extract/go.sum b/extract/go.sum index 772ddbd..cdbecec 100644 --- a/extract/go.sum +++ b/extract/go.sum @@ -1,4 +1,2 @@ -github.com/hmarr/pkgextract v0.0.0-20181020214423-fb37ca687b30 h1:yv5QpAklti/aXzXutayHrR0H5q/scACFjhapn5ghONo= -github.com/hmarr/pkgextract v0.0.0-20181020214423-fb37ca687b30/go.mod h1:6iRWK5o4q/zaYE+7CCrJw+orU8YTOQk+6ceIsR/i+4c= github.com/hmarr/pkgextract v0.1.0 h1:h6foxcQmFOSWvzAN5TwVoU33zjT8KZISSX/Godj7ZzM= github.com/hmarr/pkgextract v0.1.0/go.mod h1:6iRWK5o4q/zaYE+7CCrJw+orU8YTOQk+6ceIsR/i+4c= diff --git a/go.mod b/go.mod index e819ba2..6d92b24 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/dependabot/gomodules-extracted -go 1.15 +go 1.16