From 5821053f7f5ad1434f01cd26d24a840c3447c6c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Sat, 29 Apr 2023 15:11:04 +0100 Subject: [PATCH] cache: update to Go tip as of April 2023 As of commit 0fd6ae548f550bdbee4a434285ff052fb9dc7417. Besides rewriting import paths, we swapped base.Fatalf with log.Fatalf, and replaced cfg.Getenv with os.Getenv, adding a note about the difference in behavior. The old code already had this limitation. We hadn't updated this package since it was first copied in 2018, so quite a few changes have taken place. Of note, it now supports mmap; leave that out for now, to keep this commit simple and to leave adding the mmap package for another patch. A minor API change is that Trim now returns an error. While technically a breaking change, the vast majority of users will be simply calling the API without expecting a result, and that will continue to work like it did before. Checking for errors on trim is useful, which is why upstream added it. Finally, the cache now uses lockedfile, which we already copied over. --- cache/cache.go | 179 ++++++++++++++++++++++++++++--------- cache/cache_test.go | 94 +++++++------------ cache/default.go | 86 +++++++++--------- cache/default_unix_test.go | 68 -------------- cache/hash.go | 18 +++- cache/hash_test.go | 3 +- 6 files changed, 231 insertions(+), 217 deletions(-) delete mode 100644 cache/default_unix_test.go diff --git a/cache/cache.go b/cache/cache.go index e6fe3d30..93c90535 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -12,12 +12,14 @@ import ( "errors" "fmt" "io" - "io/ioutil" + "io/fs" "os" "path/filepath" "strconv" "strings" "time" + + "github.com/rogpeppe/go-internal/lockedfile" ) // An ActionID is a cache action key, the hash of a complete description of a @@ -31,7 +33,6 @@ type OutputID [HashSize]byte // A Cache is a package cache, backed by a file system directory tree. type Cache struct { dir string - log *os.File now func() time.Time } @@ -52,21 +53,16 @@ func Open(dir string) (*Cache, error) { return nil, err } if !info.IsDir() { - return nil, &os.PathError{Op: "open", Path: dir, Err: fmt.Errorf("not a directory")} + return nil, &fs.PathError{Op: "open", Path: dir, Err: fmt.Errorf("not a directory")} } for i := 0; i < 256; i++ { name := filepath.Join(dir, fmt.Sprintf("%02x", i)) - if err := os.MkdirAll(name, 0o777); err != nil { + if err := os.MkdirAll(name, 0777); err != nil { return nil, err } } - f, err := os.OpenFile(filepath.Join(dir, "log.txt"), os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0o666) - if err != nil { - return nil, err - } c := &Cache{ dir: dir, - log: f, now: time.Now, } return c, nil @@ -77,7 +73,22 @@ func (c *Cache) fileName(id [HashSize]byte, key string) string { return filepath.Join(c.dir, fmt.Sprintf("%02x", id[0]), fmt.Sprintf("%x", id)+"-"+key) } -var errMissing = errors.New("cache entry not found") +// An entryNotFoundError indicates that a cache entry was not found, with an +// optional underlying reason. +type entryNotFoundError struct { + Err error +} + +func (e *entryNotFoundError) Error() string { + if e.Err == nil { + return "cache entry not found" + } + return fmt.Sprintf("cache entry not found: %v", e.Err) +} + +func (e *entryNotFoundError) Unwrap() error { + return e.Err +} const ( // action entry file is "v1 \n" @@ -96,6 +107,8 @@ const ( // GODEBUG=gocacheverify=1. var verify = false +var errVerifyMode = errors.New("gocacheverify=1") + // DebugTest is set when GODEBUG=gocachetest=1 is in the environment. var DebugTest = false @@ -124,7 +137,7 @@ func initEnv() { // saved file for that output ID is still available. func (c *Cache) Get(id ActionID) (Entry, error) { if verify { - return Entry{}, errMissing + return Entry{}, &entryNotFoundError{Err: errVerifyMode} } return c.get(id) } @@ -137,52 +150,62 @@ type Entry struct { // get is Get but does not respect verify mode, so that Put can use it. func (c *Cache) get(id ActionID) (Entry, error) { - missing := func() (Entry, error) { - fmt.Fprintf(c.log, "%d miss %x\n", c.now().Unix(), id) - return Entry{}, errMissing + missing := func(reason error) (Entry, error) { + return Entry{}, &entryNotFoundError{Err: reason} } f, err := os.Open(c.fileName(id, "a")) if err != nil { - return missing() + return missing(err) } defer f.Close() entry := make([]byte, entrySize+1) // +1 to detect whether f is too long - if n, err := io.ReadFull(f, entry); n != entrySize || err != io.ErrUnexpectedEOF { - return missing() + if n, err := io.ReadFull(f, entry); n > entrySize { + return missing(errors.New("too long")) + } else if err != io.ErrUnexpectedEOF { + if err == io.EOF { + return missing(errors.New("file is empty")) + } + return missing(err) + } else if n < entrySize { + return missing(errors.New("entry file incomplete")) } if entry[0] != 'v' || entry[1] != '1' || entry[2] != ' ' || entry[3+hexSize] != ' ' || entry[3+hexSize+1+hexSize] != ' ' || entry[3+hexSize+1+hexSize+1+20] != ' ' || entry[entrySize-1] != '\n' { - return missing() + return missing(errors.New("invalid header")) } eid, entry := entry[3:3+hexSize], entry[3+hexSize:] eout, entry := entry[1:1+hexSize], entry[1+hexSize:] esize, entry := entry[1:1+20], entry[1+20:] etime, entry := entry[1:1+20], entry[1+20:] var buf [HashSize]byte - if _, err := hex.Decode(buf[:], eid); err != nil || buf != id { - return missing() + if _, err := hex.Decode(buf[:], eid); err != nil { + return missing(fmt.Errorf("decoding ID: %v", err)) + } else if buf != id { + return missing(errors.New("mismatched ID")) } if _, err := hex.Decode(buf[:], eout); err != nil { - return missing() + return missing(fmt.Errorf("decoding output ID: %v", err)) } i := 0 for i < len(esize) && esize[i] == ' ' { i++ } size, err := strconv.ParseInt(string(esize[i:]), 10, 64) - if err != nil || size < 0 { - return missing() + if err != nil { + return missing(fmt.Errorf("parsing size: %v", err)) + } else if size < 0 { + return missing(errors.New("negative size")) } i = 0 for i < len(etime) && etime[i] == ' ' { i++ } tm, err := strconv.ParseInt(string(etime[i:]), 10, 64) - if err != nil || size < 0 { - return missing() + if err != nil { + return missing(fmt.Errorf("parsing timestamp: %v", err)) + } else if tm < 0 { + return missing(errors.New("negative timestamp")) } - fmt.Fprintf(c.log, "%d get %x\n", c.now().Unix(), id) - c.used(c.fileName(id, "a")) return Entry{buf, size, time.Unix(0, tm)}, nil @@ -197,8 +220,11 @@ func (c *Cache) GetFile(id ActionID) (file string, entry Entry, err error) { } file = c.OutputFile(entry.OutputID) info, err := os.Stat(file) - if err != nil || info.Size() != entry.Size { - return "", Entry{}, errMissing + if err != nil { + return "", Entry{}, &entryNotFoundError{Err: err} + } + if info.Size() != entry.Size { + return "", Entry{}, &entryNotFoundError{Err: errors.New("file incomplete")} } return file, entry, nil } @@ -211,13 +237,35 @@ func (c *Cache) GetBytes(id ActionID) ([]byte, Entry, error) { if err != nil { return nil, entry, err } - data, _ := ioutil.ReadFile(c.OutputFile(entry.OutputID)) + data, _ := os.ReadFile(c.OutputFile(entry.OutputID)) if sha256.Sum256(data) != entry.OutputID { - return nil, entry, errMissing + return nil, entry, &entryNotFoundError{Err: errors.New("bad checksum")} } return data, entry, nil } +/* +TODO: consider copying cmd/go/internal/mmap over for this method + +// GetMmap looks up the action ID in the cache and returns +// the corresponding output bytes. +// GetMmap should only be used for data that can be expected to fit in memory. +func (c *Cache) GetMmap(id ActionID) ([]byte, Entry, error) { + entry, err := c.Get(id) + if err != nil { + return nil, entry, err + } + md, err := mmap.Mmap(c.OutputFile(entry.OutputID)) + if err != nil { + return nil, Entry{}, err + } + if int64(len(md.Data)) != entry.Size { + return nil, Entry{}, &entryNotFoundError{Err: errors.New("file incomplete")} + } + return md.Data, entry, nil +} +*/ + // OutputFile returns the name of the cache file storing output with the given OutputID. func (c *Cache) OutputFile(out OutputID) string { file := c.fileName(out, "d") @@ -261,16 +309,23 @@ func (c *Cache) used(file string) { } // Trim removes old cache entries that are likely not to be reused. -func (c *Cache) Trim() { +func (c *Cache) Trim() error { now := c.now() // We maintain in dir/trim.txt the time of the last completed cache trim. // If the cache has been trimmed recently enough, do nothing. // This is the common case. - data, _ := ioutil.ReadFile(filepath.Join(c.dir, "trim.txt")) - t, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64) - if err == nil && now.Sub(time.Unix(t, 0)) < trimInterval { - return + // If the trim file is corrupt, detected if the file can't be parsed, or the + // trim time is too far in the future, attempt the trim anyway. It's possible that + // the cache was full when the corruption happened. Attempting a trim on + // an empty cache is cheap, so there wouldn't be a big performance hit in that case. + if data, err := lockedfile.Read(filepath.Join(c.dir, "trim.txt")); err == nil { + if t, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64); err == nil { + lastTrim := time.Unix(t, 0) + if d := now.Sub(lastTrim); d < trimInterval && d > -mtimeInterval { + return nil + } + } } // Trim each of the 256 subdirectories. @@ -282,7 +337,15 @@ func (c *Cache) Trim() { c.trimSubdir(subdir, cutoff) } - ioutil.WriteFile(filepath.Join(c.dir, "trim.txt"), []byte(fmt.Sprintf("%d", now.Unix())), 0o666) + // Ignore errors from here: if we don't write the complete timestamp, the + // cache will appear older than it is, and we'll trim it again next time. + var b bytes.Buffer + fmt.Fprintf(&b, "%d", now.Unix()) + if err := lockedfile.Write(filepath.Join(c.dir, "trim.txt"), &b, 0666); err != nil { + return err + } + + return nil } // trimSubdir trims a single cache subdirectory. @@ -326,7 +389,7 @@ func (c *Cache) putIndexEntry(id ActionID, out OutputID, size int64, allowVerify // in verify mode we are double-checking that the cache entries // are entirely reproducible. As just noted, this may be unrealistic // in some cases but the check is also useful for shaking out real bugs. - entry := []byte(fmt.Sprintf("v1 %x %x %20d %20d\n", id, out, size, time.Now().UnixNano())) + entry := fmt.Sprintf("v1 %x %x %20d %20d\n", id, out, size, time.Now().UnixNano()) if verify && allowVerify { old, err := c.get(id) if err == nil && (old.OutputID != out || old.Size != size) { @@ -336,13 +399,35 @@ func (c *Cache) putIndexEntry(id ActionID, out OutputID, size int64, allowVerify } } file := c.fileName(id, "a") - if err := ioutil.WriteFile(file, entry, 0o666); err != nil { + + // Copy file to cache directory. + mode := os.O_WRONLY | os.O_CREATE + f, err := os.OpenFile(file, mode, 0666) + if err != nil { + return err + } + _, err = f.WriteString(entry) + if err == nil { + // Truncate the file only *after* writing it. + // (This should be a no-op, but truncate just in case of previous corruption.) + // + // This differs from os.WriteFile, which truncates to 0 *before* writing + // via os.O_TRUNC. Truncating only after writing ensures that a second write + // of the same content to the same file is idempotent, and does not — even + // temporarily! — undo the effect of the first write. + err = f.Truncate(int64(len(entry))) + } + if closeErr := f.Close(); err == nil { + err = closeErr + } + if err != nil { + // TODO(bcmills): This Remove potentially races with another go command writing to file. + // Can we eliminate it? os.Remove(file) return err } os.Chtimes(file, c.now(), c.now()) // mainly for tests - fmt.Fprintf(c.log, "%d put %x %x %d\n", c.now().Unix(), id, out, size) return nil } @@ -413,7 +498,7 @@ func (c *Cache) copyFile(file io.ReadSeeker, out OutputID, size int64) error { if err == nil && info.Size() > size { // shouldn't happen but fix in case mode |= os.O_TRUNC } - f, err := os.OpenFile(name, mode, 0o666) + f, err := os.OpenFile(name, mode, 0666) if err != nil { return err } @@ -471,3 +556,15 @@ func (c *Cache) copyFile(file io.ReadSeeker, out OutputID, size int64) error { return nil } + +// FuzzDir returns a subdirectory within the cache for storing fuzzing data. +// The subdirectory may not exist. +// +// This directory is managed by the internal/fuzz package. Files in this +// directory aren't removed by the 'go clean -cache' command or by Trim. +// They may be removed with 'go clean -fuzzcache'. +// +// TODO(#48526): make Trim remove unused files from this directory. +func (c *Cache) FuzzDir() string { + return filepath.Join(c.dir, "fuzz") +} diff --git a/cache/cache_test.go b/cache/cache_test.go index e5d1adbc..5ff84c2b 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -8,7 +8,6 @@ import ( "bytes" "encoding/binary" "fmt" - "io/ioutil" "os" "path/filepath" "testing" @@ -20,7 +19,7 @@ func init() { } func TestBasic(t *testing.T) { - dir, err := ioutil.TempDir("", "cachetest-") + dir, err := os.MkdirTemp("", "cachetest-") if err != nil { t.Fatal(err) } @@ -31,7 +30,7 @@ func TestBasic(t *testing.T) { } cdir := filepath.Join(dir, "c1") - if err := os.Mkdir(cdir, 0o777); err != nil { + if err := os.Mkdir(cdir, 0777); err != nil { t.Fatal(err) } @@ -65,7 +64,7 @@ func TestBasic(t *testing.T) { } func TestGrowth(t *testing.T) { - dir, err := ioutil.TempDir("", "cachetest-") + dir, err := os.MkdirTemp("", "cachetest-") if err != nil { t.Fatal(err) } @@ -78,7 +77,7 @@ func TestGrowth(t *testing.T) { n := 10000 if testing.Short() { - n = 1000 + n = 10 } for i := 0; i < n; i++ { @@ -118,7 +117,7 @@ func TestVerifyPanic(t *testing.T) { t.Fatal("initEnv did not set verify") } - dir, err := ioutil.TempDir("", "cachetest-") + dir, err := os.MkdirTemp("", "cachetest-") if err != nil { t.Fatal(err) } @@ -144,55 +143,6 @@ func TestVerifyPanic(t *testing.T) { t.Fatal("mismatched Put did not panic in verify mode") } -func TestCacheLog(t *testing.T) { - dir, err := ioutil.TempDir("", "cachetest-") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - - c, err := Open(dir) - if err != nil { - t.Fatalf("Open: %v", err) - } - c.now = func() time.Time { return time.Unix(1e9, 0) } - - id := ActionID(dummyID(1)) - c.Get(id) - c.PutBytes(id, []byte("abc")) - c.Get(id) - - c, err = Open(dir) - if err != nil { - t.Fatalf("Open #2: %v", err) - } - c.now = func() time.Time { return time.Unix(1e9+1, 0) } - c.Get(id) - - id2 := ActionID(dummyID(2)) - c.Get(id2) - c.PutBytes(id2, []byte("abc")) - c.Get(id2) - c.Get(id) - - data, err := ioutil.ReadFile(filepath.Join(dir, "log.txt")) - if err != nil { - t.Fatal(err) - } - want := `1000000000 miss 0100000000000000000000000000000000000000000000000000000000000000 -1000000000 put 0100000000000000000000000000000000000000000000000000000000000000 ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad 3 -1000000000 get 0100000000000000000000000000000000000000000000000000000000000000 -1000000001 get 0100000000000000000000000000000000000000000000000000000000000000 -1000000001 miss 0200000000000000000000000000000000000000000000000000000000000000 -1000000001 put 0200000000000000000000000000000000000000000000000000000000000000 ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad 3 -1000000001 get 0200000000000000000000000000000000000000000000000000000000000000 -1000000001 get 0100000000000000000000000000000000000000000000000000000000000000 -` - if string(data) != want { - t.Fatalf("log:\n%s\nwant:\n%s", string(data), want) - } -} - func dummyID(x int) [HashSize]byte { var out [HashSize]byte binary.LittleEndian.PutUint64(out[:], uint64(x)) @@ -200,7 +150,7 @@ func dummyID(x int) [HashSize]byte { } func TestCacheTrim(t *testing.T) { - dir, err := ioutil.TempDir("", "cachetest-") + dir, err := os.MkdirTemp("", "cachetest-") if err != nil { t.Fatal(err) } @@ -251,12 +201,18 @@ func TestCacheTrim(t *testing.T) { checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime2) // Trim should leave everything alone: it's all too new. - c.Trim() + if err := c.Trim(); err != nil { + // if testenv.SyscallIsNotSupported(err) { + if true { + t.Skipf("skipping: Trim is unsupported (%v)", err) + } + t.Fatal(err) + } if _, err := c.Get(id); err != nil { t.Fatal(err) } c.OutputFile(entry.OutputID) - data, err := ioutil.ReadFile(filepath.Join(dir, "trim.txt")) + data, err := os.ReadFile(filepath.Join(dir, "trim.txt")) if err != nil { t.Fatal(err) } @@ -264,12 +220,14 @@ func TestCacheTrim(t *testing.T) { // Trim less than a day later should not do any work at all. now = start + 80000 - c.Trim() + if err := c.Trim(); err != nil { + t.Fatal(err) + } if _, err := c.Get(id); err != nil { t.Fatal(err) } c.OutputFile(entry.OutputID) - data2, err := ioutil.ReadFile(filepath.Join(dir, "trim.txt")) + data2, err := os.ReadFile(filepath.Join(dir, "trim.txt")) if err != nil { t.Fatal(err) } @@ -284,7 +242,9 @@ func TestCacheTrim(t *testing.T) { // and we haven't looked at it since, so 5 days later it should be gone. now += 5 * 86400 checkTime(fmt.Sprintf("%x-a", dummyID(2)), start) - c.Trim() + if err := c.Trim(); err != nil { + t.Fatal(err) + } if _, err := c.Get(id); err != nil { t.Fatal(err) } @@ -298,7 +258,9 @@ func TestCacheTrim(t *testing.T) { // Check that another 5 days later it is still not gone, // but check by using checkTime, which doesn't bring mtime forward. now += 5 * 86400 - c.Trim() + if err := c.Trim(); err != nil { + t.Fatal(err) + } checkTime(fmt.Sprintf("%x-a", id), mtime3) checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime3) @@ -306,13 +268,17 @@ func TestCacheTrim(t *testing.T) { // Even though the entry for id is now old enough to be trimmed, // it gets a reprieve until the time comes for a new Trim scan. now += 86400 / 2 - c.Trim() + if err := c.Trim(); err != nil { + t.Fatal(err) + } checkTime(fmt.Sprintf("%x-a", id), mtime3) checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime3) // Another half a day later, Trim should actually run, and it should remove id. now += 86400/2 + 1 - c.Trim() + if err := c.Trim(); err != nil { + t.Fatal(err) + } if _, err := c.Get(dummyID(1)); err == nil { t.Fatal("Trim did not remove dummyID(1)") } diff --git a/cache/default.go b/cache/default.go index d641931e..a20b33cd 100644 --- a/cache/default.go +++ b/cache/default.go @@ -6,13 +6,14 @@ package cache import ( "fmt" - "io/ioutil" + "log" "os" "path/filepath" "sync" ) -// Default returns the default cache to use, or nil if no cache should be used. +// Default returns the default cache to use. +// It never returns nil. func Default() *Cache { defaultOnce.Do(initDefaultCache) return defaultCache @@ -28,68 +29,71 @@ var ( // README as a courtesy to explain where it came from. const cacheREADME = `This directory holds cached build artifacts from the Go build system. Run "go clean -cache" if the directory is getting too large. +Run "go clean -fuzzcache" to delete the fuzz cache. See golang.org to learn more about Go. ` // initDefaultCache does the work of finding the default cache // the first time Default is called. func initDefaultCache() { - dir, showWarnings := defaultDir() + dir := DefaultDir() if dir == "off" { - return - } - if err := os.MkdirAll(dir, 0o777); err != nil { - if showWarnings { - fmt.Fprintf(os.Stderr, "go: disabling cache (%s) due to initialization failure: %s\n", dir, err) + if defaultDirErr != nil { + log.Fatalf("build cache is required, but could not be located: %v", defaultDirErr) } - return + log.Fatalf("build cache is disabled by GOCACHE=off, but required as of Go 1.12") + } + if err := os.MkdirAll(dir, 0777); err != nil { + log.Fatalf("failed to initialize build cache at %s: %s\n", dir, err) } if _, err := os.Stat(filepath.Join(dir, "README")); err != nil { // Best effort. - ioutil.WriteFile(filepath.Join(dir, "README"), []byte(cacheREADME), 0o666) + os.WriteFile(filepath.Join(dir, "README"), []byte(cacheREADME), 0666) } c, err := Open(dir) if err != nil { - if showWarnings { - fmt.Fprintf(os.Stderr, "go: disabling cache (%s) due to initialization failure: %s\n", dir, err) - } - return + log.Fatalf("failed to initialize build cache at %s: %s\n", dir, err) } defaultCache = c } +var ( + defaultDirOnce sync.Once + defaultDir string + defaultDirErr error +) + // DefaultDir returns the effective GOCACHE setting. // It returns "off" if the cache is disabled. func DefaultDir() string { - dir, _ := defaultDir() - return dir -} + // Save the result of the first call to DefaultDir for later use in + // initDefaultCache. cmd/go/main.go explicitly sets GOCACHE so that + // subprocesses will inherit it, but that means initDefaultCache can't + // otherwise distinguish between an explicit "off" and a UserCacheDir error. -// defaultDir returns the effective GOCACHE setting. -// It returns "off" if the cache is disabled. -// The second return value reports whether warnings should -// be shown if the cache fails to initialize. -func defaultDir() (string, bool) { - dir := os.Getenv("GOCACHE") - if dir != "" { - return dir, true - } + defaultDirOnce.Do(func() { + // NOTE: changed from upstream's cfg.Getenv, so it will ignore "go env -w". + // Consider calling "go env" or copying the cfg package instead. + defaultDir = os.Getenv("GOCACHE") + if filepath.IsAbs(defaultDir) || defaultDir == "off" { + return + } + if defaultDir != "" { + defaultDir = "off" + defaultDirErr = fmt.Errorf("GOCACHE is not an absolute path") + return + } - // Compute default location. - dir, err := os.UserCacheDir() - if err != nil { - return "off", true - } - dir = filepath.Join(dir, "go-build") + // Compute default location. + dir, err := os.UserCacheDir() + if err != nil { + defaultDir = "off" + defaultDirErr = fmt.Errorf("GOCACHE is not defined and %v", err) + return + } + defaultDir = filepath.Join(dir, "go-build") + }) - // Do this after filepath.Join, so that the path has been cleaned. - showWarnings := true - switch dir { - case "/.cache/go-build": - // probably docker run with -u flag - // https://golang.org/issue/26280 - showWarnings = false - } - return dir, showWarnings + return defaultDir } diff --git a/cache/default_unix_test.go b/cache/default_unix_test.go deleted file mode 100644 index ddc95a5e..00000000 --- a/cache/default_unix_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !windows && !darwin && !plan9 -// +build !windows,!darwin,!plan9 - -package cache - -import ( - "os" - "strings" - "testing" -) - -func TestDefaultDir(t *testing.T) { - goCacheDir := "/tmp/test-go-cache" - xdgCacheDir := "/tmp/test-xdg-cache" - homeDir := "/tmp/test-home" - - // undo env changes when finished - defer func(GOCACHE, XDG_CACHE_HOME, HOME string) { - os.Setenv("GOCACHE", GOCACHE) - os.Setenv("XDG_CACHE_HOME", XDG_CACHE_HOME) - os.Setenv("HOME", HOME) - }(os.Getenv("GOCACHE"), os.Getenv("XDG_CACHE_HOME"), os.Getenv("HOME")) - - os.Setenv("GOCACHE", goCacheDir) - os.Setenv("XDG_CACHE_HOME", xdgCacheDir) - os.Setenv("HOME", homeDir) - - dir, showWarnings := defaultDir() - if dir != goCacheDir { - t.Errorf("Cache DefaultDir %q should be $GOCACHE %q", dir, goCacheDir) - } - if !showWarnings { - t.Error("Warnings should be shown when $GOCACHE is set") - } - - os.Unsetenv("GOCACHE") - dir, showWarnings = defaultDir() - if !strings.HasPrefix(dir, xdgCacheDir+"/") { - t.Errorf("Cache DefaultDir %q should be under $XDG_CACHE_HOME %q when $GOCACHE is unset", dir, xdgCacheDir) - } - if !showWarnings { - t.Error("Warnings should be shown when $XDG_CACHE_HOME is set") - } - - os.Unsetenv("XDG_CACHE_HOME") - dir, showWarnings = defaultDir() - if !strings.HasPrefix(dir, homeDir+"/.cache/") { - t.Errorf("Cache DefaultDir %q should be under $HOME/.cache %q when $GOCACHE and $XDG_CACHE_HOME are unset", dir, homeDir+"/.cache") - } - if !showWarnings { - t.Error("Warnings should be shown when $HOME is not /") - } - - os.Unsetenv("HOME") - if dir, _ := defaultDir(); dir != "off" { - t.Error("Cache not disabled when $GOCACHE, $XDG_CACHE_HOME, and $HOME are unset") - } - - os.Setenv("HOME", "/") - if _, showWarnings := defaultDir(); showWarnings { - // https://golang.org/issue/26280 - t.Error("Cache initialization warnings should be squelched when $GOCACHE and $XDG_CACHE_HOME are unset and $HOME is /") - } -} diff --git a/cache/hash.go b/cache/hash.go index e4bb2a34..4f79c315 100644 --- a/cache/hash.go +++ b/cache/hash.go @@ -12,6 +12,7 @@ import ( "io" "os" "runtime" + "strings" "sync" ) @@ -36,7 +37,22 @@ type Hash struct { // of other versions. This salt will result in additional ActionID files // in the cache, but not additional copies of the large output files, // which are still addressed by unsalted SHA256. -var hashSalt = []byte(runtime.Version()) +// +// We strip any GOEXPERIMENTs the go tool was built with from this +// version string on the assumption that they shouldn't affect go tool +// execution. This allows bootstrapping to converge faster: dist builds +// go_bootstrap without any experiments, so by stripping experiments +// go_bootstrap and the final go binary will use the same salt. +var hashSalt = []byte(stripExperiment(runtime.Version())) + +// stripExperiment strips any GOEXPERIMENT configuration from the Go +// version string. +func stripExperiment(version string) string { + if i := strings.Index(version, " X:"); i >= 0 { + return version[:i] + } + return version +} // Subkey returns an action ID corresponding to mixing a parent // action ID with a string description of the subkey. diff --git a/cache/hash_test.go b/cache/hash_test.go index 3bf71430..a0356771 100644 --- a/cache/hash_test.go +++ b/cache/hash_test.go @@ -6,7 +6,6 @@ package cache import ( "fmt" - "io/ioutil" "os" "testing" ) @@ -28,7 +27,7 @@ func TestHash(t *testing.T) { } func TestHashFile(t *testing.T) { - f, err := ioutil.TempFile("", "cmd-go-test-") + f, err := os.CreateTemp("", "cmd-go-test-") if err != nil { t.Fatal(err) }