diff --git a/gofmt.go b/gofmt.go index 4d29520..0fe3fdd 100644 --- a/gofmt.go +++ b/gofmt.go @@ -23,6 +23,7 @@ import ( "runtime" "runtime/pprof" "strings" + "sync" "golang.org/x/sync/semaphore" exec "golang.org/x/sys/execabs" @@ -288,15 +289,10 @@ func processFile(filename string, info fs.FileInfo, in io.Reader, r *reporter, e // If either -lang or -modpath aren't set, fetch them from go.mod. if *langVersion == "" || *modulePath == "" { - out, err := exec.Command("go", "mod", "edit", "-json").Output() - if err == nil && len(out) > 0 { - var mod struct { - Go string - Module struct { - Path string - } - } - _ = json.Unmarshal(out, &mod) + dir := filepath.Dir(filename) + mod, ok := moduleCacheByDir.Load(dir) + if ok && mod != nil { + mod := mod.(*module) if *langVersion == "" { *langVersion = mod.Go } @@ -532,7 +528,49 @@ func gofmtMain(s *sequencer) { } } +type module struct { + Go string + Module struct { + Path string + } +} + +func loadModuleInfo(dir string) interface{} { + // Spawning "go mod edit" will open files by design, + // such as the named pipe to obtain stdout. + // TODO(mvdan): if we run into "too many open files" errors again in the + // future, we probably need to turn fdSem into a weighted semaphore so this + // operation can acquire a weight larger than 1. + fdSem <- true + out, err := exec.Command("go", "mod", "edit", "-json").Output() + defer func() { <-fdSem }() + if err != nil || len(out) == 0 { + return nil + } + mod := new(module) + if err := json.Unmarshal(out, mod); err != nil { + return nil + } + if *langVersion == "" { + *langVersion = mod.Go + } + if *modulePath == "" { + *modulePath = mod.Module.Path + } + return mod +} + +// Written to by fileWeight, read from fileWeight and processFile. +// A present but nil value means that loading the module info failed. +// Note that we don't require the keys to be absolute directories, +// so duplicates are possible. The same can happen with symlinks. +var moduleCacheByDir sync.Map // map[dirString]*module + func fileWeight(path string, info fs.FileInfo) int64 { + dir := filepath.Dir(path) + if _, ok := moduleCacheByDir.Load(dir); !ok { + moduleCacheByDir.Store(dir, loadModuleInfo(dir)) + } if info == nil { return exclusive } diff --git a/ulimit_unix_test.go b/ulimit_unix_test.go new file mode 100644 index 0000000..9d67c74 --- /dev/null +++ b/ulimit_unix_test.go @@ -0,0 +1,88 @@ +// Copyright (c) 2019, Daniel Martí +// See LICENSE for licensing information + +// TODO: replace with the unix build tag once we require Go 1.19 or later +//go:build linux + +package main + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strconv" + "syscall" + "testing" + + qt "github.com/frankban/quicktest" + exec "golang.org/x/sys/execabs" +) + +func init() { + // Here rather than in TestMain, to reuse the unix build tag. + if limit := os.Getenv("TEST_WITH_FILE_LIMIT"); limit != "" { + n, err := strconv.ParseUint(limit, 10, 64) + if err != nil { + panic(err) + } + rlimit := syscall.Rlimit{Cur: n, Max: n} + if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rlimit); err != nil { + panic(err) + } + os.Exit(main1()) + } +} + +func TestWithLowOpenFileLimit(t *testing.T) { + // Safe to run in parallel, as we only change the limit for child processes. + t.Parallel() + + tempDir := t.TempDir() + testBinary, err := os.Executable() + qt.Assert(t, err, qt.IsNil) + + const ( + // Enough directories to run into the ulimit. + // Enough number of files in total to run into the ulimit. + numberDirs = 500 + numberFilesPerDir = 20 + numberFilesTotal = numberDirs * numberFilesPerDir + ) + t.Logf("writing %d tiny Go files", numberFilesTotal) + var allGoFiles []string + for i := 0; i < numberDirs; i++ { + // Prefix "p", so the package name is a valid identifier. + dirName := fmt.Sprintf("p%03d", i) + dirPath := filepath.Join(tempDir, dirName) + err := os.MkdirAll(dirPath, 0o777) + qt.Assert(t, err, qt.IsNil) + for j := 0; j < numberFilesPerDir; j++ { + filePath := filepath.Join(dirPath, fmt.Sprintf("%03d.go", j)) + err := os.WriteFile(filePath, + // Extra newlines so that "-l" prints all paths. + []byte(fmt.Sprintf("package %s\n\n\n", dirName)), + 0o666) + qt.Assert(t, err, qt.IsNil) + allGoFiles = append(allGoFiles, filePath) + } + } + if len(allGoFiles) != numberFilesTotal { + panic("allGoFiles doesn't have the expected number of files?") + } + runGofmt := func(paths ...string) { + t.Logf("running with %d paths", len(paths)) + cmd := exec.Command(testBinary, append([]string{"-l"}, paths...)...) + // 256 is a relatively common low limit, e.g. on Mac. + cmd.Env = append(os.Environ(), "TEST_WITH_FILE_LIMIT=256") + out, err := cmd.Output() + var stderr []byte + if err, _ := err.(*exec.ExitError); err != nil { + stderr = err.Stderr + } + qt.Assert(t, err, qt.IsNil, qt.Commentf("stderr:\n%s", stderr)) + qt.Assert(t, bytes.Count(out, []byte("\n")), qt.Equals, len(allGoFiles)) + } + runGofmt(tempDir) + runGofmt(allGoFiles...) +}