From 4348caf720ef46e508bfced4c291ed6cde1ee27f Mon Sep 17 00:00:00 2001 From: Asra Ali Date: Mon, 23 May 2022 11:34:25 -0500 Subject: [PATCH 01/11] fix: fix fetching updated targets from TUF root Signed-off-by: Asra Ali add comment Signed-off-by: Asra Ali update Signed-off-by: Asra Ali update Signed-off-by: Asra Ali possible fix windows Signed-off-by: Asra Ali lint Signed-off-by: Asra Ali fix windows maybe Signed-off-by: Asra Ali fix close Signed-off-by: Asra Ali --- Makefile | 2 +- .../cli/fulcio/fulcioroots/fulcioroots.go | 2 +- pkg/cosign/tuf/client.go | 311 +++++++++++++----- pkg/cosign/tuf/client_test.go | 117 ++++++- 4 files changed, 340 insertions(+), 92 deletions(-) diff --git a/Makefile b/Makefile index e7e4fc91e74..a47074f75ca 100644 --- a/Makefile +++ b/Makefile @@ -118,7 +118,7 @@ cross: golangci-lint: rm -f $(GOLANGCI_LINT_BIN) || : set -e ;\ - GOBIN=$(GOLANGCI_LINT_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.43.0 ;\ + GOBIN=$(GOLANGCI_LINT_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.46.0 ;\ lint: golangci-lint ## Run golangci-lint linter $(GOLANGCI_LINT_BIN) run -n diff --git a/cmd/cosign/cli/fulcio/fulcioroots/fulcioroots.go b/cmd/cosign/cli/fulcio/fulcioroots/fulcioroots.go index c77378a9b3a..cba25ffe376 100644 --- a/cmd/cosign/cli/fulcio/fulcioroots/fulcioroots.go +++ b/cmd/cosign/cli/fulcio/fulcioroots/fulcioroots.go @@ -121,7 +121,7 @@ func initRoots() (*x509.CertPool, *x509.CertPool, error) { // call is made to update the root. targets, err := tufClient.GetTargetsByMeta(tuf.Fulcio, []string{fulcioTargetStr, fulcioV1TargetStr}) if err != nil { - return nil, nil, errors.New("error getting targets") + return nil, nil, fmt.Errorf("error getting targets: %w", err) } if len(targets) == 0 { return nil, nil, errors.New("none of the Fulcio roots have been found") diff --git a/pkg/cosign/tuf/client.go b/pkg/cosign/tuf/client.go index 9994d981c4c..cb3abddb180 100644 --- a/pkg/cosign/tuf/client.go +++ b/pkg/cosign/tuf/client.go @@ -20,18 +20,20 @@ import ( "context" "embed" "encoding/json" + "errors" "fmt" "io" + "io/fs" "io/ioutil" "net/url" "os" - "path" "path/filepath" "runtime" "strconv" "strings" "time" + "github.com/secure-systems-lab/go-securesystemslib/cjson" "github.com/theupdateframework/go-tuf/client" tuf_leveldbstore "github.com/theupdateframework/go-tuf/client/leveldbstore" "github.com/theupdateframework/go-tuf/data" @@ -44,6 +46,10 @@ const ( SigstoreNoCache = "SIGSTORE_NO_CACHE" ) +var GetRemoteRoot = func() string { + return DefaultRemoteRoot +} + type TUF struct { client *client.Client targets targetImpl @@ -186,7 +192,11 @@ func getRoot(meta map[string]json.RawMessage) (json.RawMessage, error) { return trustedRoot, nil } // On first initialize, there will be no root in the TUF DB, so read from embedded. - trustedRoot, err := embeddedRootRepo.ReadFile(path.Join("repository", "root.json")) + rd, ok := GetEmbedded().(fs.ReadFileFS) + if !ok { + return nil, errors.New("fs.ReadFileFS unimplemented for embedded repo") + } + trustedRoot, err := rd.ReadFile(filepath.FromSlash(filepath.Join("repository", "root.json"))) if err != nil { return nil, err } @@ -222,20 +232,18 @@ func initializeTUF(ctx context.Context, embed bool, mirror string, root []byte, embedded: embed, } + t.targets = newFileImpl() + // Lazily init the local store in case we start with an embedded. + var initLocal = newLocalStore var err error if t.embedded { - t.local, err = embeddedLocalStore() - if err != nil { - return nil, err - } - t.targets = newEmbeddedImpl() + t.targets = wrapEmbedded(t.targets) + t.local = wrapEmbeddedLocal(initLocal) } else { - tufDB := filepath.Join(rootCacheDir(), "tuf.db") - t.local, err = localStore(tufDB) + t.local, err = initLocal() if err != nil { return nil, err } - t.targets = newFileImpl() } t.remote, err = remoteFromMirror(ctx, t.mirror) @@ -283,7 +291,7 @@ func initializeTUF(ctx context.Context, embed bool, mirror string, root []byte, func NewFromEnv(ctx context.Context) (*TUF, error) { // Get local and mirror from env - tufDB := filepath.Join(rootCacheDir(), "tuf.db") + tufDB := filepath.FromSlash(filepath.Join(rootCacheDir(), "tuf.db")) var embed bool // Check for the current local. @@ -296,12 +304,12 @@ func NewFromEnv(ctx context.Context) (*TUF, error) { // Some other error, bail return nil, statErr default: - // There is a root! Happy path. + // There is a local root! Happy path. embed = false } // Check for the current remote mirror. - mirror := DefaultRemoteRoot + mirror := GetRemoteRoot() b, err := os.ReadFile(cachedRemote(rootCacheDir())) if err == nil { remoteInfo := remoteCache{} @@ -340,7 +348,6 @@ func (t *TUF) GetTarget(name string) ([]byte, error) { if err != nil { return nil, fmt.Errorf("error verifying local metadata; local cache may be corrupt: %w", err) } - targetBytes, err := t.targets.Get(name) if err != nil { return nil, err @@ -379,7 +386,7 @@ func (t *TUF) GetTargetsByMeta(usage UsageKind, fallbacks []string) ([]TargetFil if scm.Sigstore.Usage == usage { target, err := t.GetTarget(name) if err != nil { - return nil, fmt.Errorf("error getting target by usage: %w", err) + return nil, fmt.Errorf("error getting target %s by usage: %w", name, err) } matchedTargets = append(matchedTargets, TargetFile{Target: target, Status: scm.Sigstore.Status}) } @@ -400,30 +407,9 @@ func (t *TUF) GetTargetsByMeta(usage UsageKind, fallbacks []string) ([]TargetFil return matchedTargets, nil } -func localStore(cacheRoot string) (client.LocalStore, error) { - local, err := tuf_leveldbstore.FileLocalStore(cacheRoot) - if err != nil { - return nil, fmt.Errorf("creating cached local store: %w", err) - } - return local, nil -} - -func embeddedLocalStore() (client.LocalStore, error) { - local := client.MemoryLocalStore() - for _, mdFilename := range []string{"root.json", "targets.json", "snapshot.json", "timestamp.json"} { - b, err := embeddedRootRepo.ReadFile(path.Join("repository", mdFilename)) - if err != nil { - return nil, fmt.Errorf("reading embedded file: %w", err) - } - if err := local.SetMeta(mdFilename, b); err != nil { - return nil, fmt.Errorf("setting local meta: %w", err) - } - } - return local, nil -} - func (t *TUF) updateMetadataAndDownloadTargets() error { // Download updated targets and cache new metadata and targets in ${TUF_ROOT}. + // NOTE: This only returns *updated* targets. targetFiles, err := t.client.Update() if err != nil { // Get some extra information for debugging. What was the state of the metadata @@ -459,8 +445,8 @@ func (t *TUF) updateMetadataAndDownloadTargets() error { return fmt.Errorf("error updating to TUF remote mirror: %w\nremote status:%s", err, string(b)) } - // Update the in-memory targets. - // If the cache directory is enabled, update that too. + // Download newly updated targets. + // TODO: Consider lazily downloading these -- be careful with embedded targets if so. for name := range targetFiles { buf := bytes.Buffer{} if err := downloadRemoteTarget(name, t.client, &buf); err != nil { @@ -503,49 +489,175 @@ func rootCacheDir() string { if err != nil { home = "" } - return filepath.Join(home, ".sigstore", "root") + return filepath.FromSlash(filepath.Join(home, ".sigstore", "root")) } return rootDir } func cachedRemote(cacheRoot string) string { - return filepath.Join(cacheRoot, "remote.json") + return filepath.FromSlash(filepath.Join(cacheRoot, "remote.json")) } func cachedTargetsDir(cacheRoot string) string { - return filepath.Join(cacheRoot, "targets") + return filepath.FromSlash(filepath.Join(cacheRoot, "targets")) } -type targetImpl interface { - Get(string) ([]byte, error) - setImpl +// Local store implementations +func newLocalStore() (client.LocalStore, error) { + if noCache() { + return client.MemoryLocalStore(), nil + } + tufDB := filepath.FromSlash(filepath.Join(rootCacheDir(), "tuf.db")) + local, err := tuf_leveldbstore.FileLocalStore(tufDB) + if err != nil { + return nil, fmt.Errorf("creating cached local store: %w", err) + } + return local, nil } -type setImpl interface { - Set(string, []byte) error +type localInitFunc func() (client.LocalStore, error) + +type embeddedLocalStore struct { + // This is where updated repository info will be written to, if needed. + writeable client.LocalStore + writeableFunc localInitFunc } -type memoryCache struct { - targets map[string][]byte +var GetEmbedded = func() fs.FS { + return embeddedRootRepo } -func (m *memoryCache) Set(p string, b []byte) error { - if m.targets == nil { - m.targets = map[string][]byte{} +func wrapEmbeddedLocal(s localInitFunc) client.LocalStore { + return &embeddedLocalStore{writeableFunc: s, writeable: nil} +} + +func (e embeddedLocalStore) GetMeta() (map[string]json.RawMessage, error) { + if e.writeable != nil { + // We are using a writeable store, so use that. + return e.writeable.GetMeta() + } + // We haven't needed to create or write new metadata, so get the embedded metadata. + meta := make(map[string]json.RawMessage) + ed, ok := GetEmbedded().(fs.ReadDirFS) + if !ok { + return nil, errors.New("fs.ReadDirFS unimplemented for embedded repo") + } + entries, err := ed.ReadDir("repository") + if err != nil { + return nil, err + } + rd, ok := GetEmbedded().(fs.ReadFileFS) + if !ok { + return nil, errors.New("fs.ReadFileFS unimplemented for embedded repo") + } + for _, entry := range entries { + if entry.Type().IsRegular() { + b, err := rd.ReadFile(filepath.FromSlash(filepath.Join("repository", entry.Name()))) + if err != nil { + return nil, fmt.Errorf("reading embedded file: %w", err) + } + meta[entry.Name()] = b + } + } + return meta, err +} + +func (e *embeddedLocalStore) SetMeta(name string, meta json.RawMessage) error { + // Only write to the underlying writeable store if the metadata is new. + embeddedMeta, err := e.GetMeta() + if err != nil { + return err + } + if metaContent, ok := embeddedMeta[name]; ok { + stored, err := cjson.EncodeCanonical(metaContent) + if err != nil { + return err + } + toSet, err := cjson.EncodeCanonical(meta) + if err != nil { + return err + } + if bytes.EqualFold(stored, toSet) { + return nil + } + } + // If we reach here, we have to update metadata. + if e.writeable == nil { + // We haven't needed an update yet, so create and populate the writeable store! + meta, err := e.GetMeta() + if err != nil { + return fmt.Errorf("error retrieving embedded repo: %w", err) + } + e.writeable, err = e.writeableFunc() + if err != nil { + return fmt.Errorf("initializing local: %w", err) + } + for m, md := range meta { + if err := e.writeable.SetMeta(m, md); err != nil { + return fmt.Errorf("error transferring to cached repo: %w", err) + } + } + } + // We have a writeable store, so set the metadata. + return e.writeable.SetMeta(name, meta) +} + +func (e embeddedLocalStore) DeleteMeta(name string) error { + if e.writeable != nil { + return e.writeable.DeleteMeta(name) + } + return nil +} + +func (e embeddedLocalStore) Close() error { + if e.writeable != nil { + return e.writeable.Close() } - m.targets[p] = b return nil } //go:embed repository var embeddedRootRepo embed.FS -type embedded struct { - setImpl +// Target Implementations +type targetImpl interface { + Set(string, []byte) error + Get(string) ([]byte, error) +} + +func newFileImpl() targetImpl { + if noCache() { + return &memoryCache{} + } + return &diskCache{base: cachedTargetsDir(rootCacheDir())} } -func (e *embedded) Get(p string) ([]byte, error) { - b, err := embeddedRootRepo.ReadFile(path.Join("repository", "targets", p)) +func wrapEmbedded(t targetImpl) targetImpl { + return &embeddedWrapper{writeable: t, embedded: true} +} + +type embeddedWrapper struct { + // If we have an embedded fallback that needs updates, use + // the writeable targetImpl + writeable targetImpl + // Whether we are using the embedded or writeable + embedded bool +} + +func (e *embeddedWrapper) Get(p string) ([]byte, error) { + if !e.embedded { + // Get it from the writeable target store since there's been updates. + b, err := e.writeable.Get(p) + if err == nil { + return b, nil + } + fmt.Fprintf(os.Stderr, "**Warning** Updated target not found; falling back on embedded target %s\n", p) + } + rd, ok := GetEmbedded().(fs.ReadFileFS) + if !ok { + return nil, errors.New("fs.ReadFileFS unimplemented for embedded repo") + } + b, err := rd.ReadFile(filepath.FromSlash(filepath.Join("repository", "targets", p))) if err != nil { return nil, err } @@ -558,25 +670,75 @@ func (e *embedded) Get(p string) ([]byte, error) { return b, nil } -type file struct { - base string - setImpl +func (e *embeddedWrapper) Set(name string, b []byte) error { + // An embedded targetImpl needs to set a file, copy files to cache. + if e.embedded { + ed, ok := GetEmbedded().(fs.ReadDirFS) + if !ok { + return errors.New("fs.ReadFileFS unimplemented for embedded repo") + } + entries, err := ed.ReadDir(filepath.FromSlash(filepath.Join("repository", "targets"))) + if err != nil { + return err + } + rd, ok := GetEmbedded().(fs.ReadFileFS) + if !ok { + return errors.New("fs.ReadFileFS unimplemented for embedded repo") + } + for _, entry := range entries { + b, err := rd.ReadFile(filepath.FromSlash(filepath.Join("repository", "targets", entry.Name()))) + if err != nil { + return fmt.Errorf("reading embedded file: %w", err) + } + if err := e.writeable.Set(entry.Name(), b); err != nil { + return fmt.Errorf("setting embedded file: %w", err) + } + } + e.embedded = false + } + return e.writeable.Set(name, b) } -func (f *file) Get(p string) ([]byte, error) { - fp := filepath.Join(f.base, p) - return os.ReadFile(fp) +// In-memory cache for targets +type memoryCache struct { + targets map[string][]byte +} + +func (m *memoryCache) Set(p string, b []byte) error { + if m.targets == nil { + m.targets = map[string][]byte{} + } + m.targets[p] = b + return nil +} + +func (m *memoryCache) Get(p string) ([]byte, error) { + if m.targets == nil { + // This should never happen, a memory cache is used only after items are Set. + return nil, fmt.Errorf("no cached targets available, cannot retrieve %s", p) + } + b, ok := m.targets[p] + if !ok { + return nil, fmt.Errorf("missing cached target %s", p) + } + return b, nil } +// On-disk cache for targets type diskCache struct { base string } +func (d *diskCache) Get(p string) ([]byte, error) { + fp := filepath.FromSlash(filepath.Join(d.base, p)) + return os.ReadFile(fp) +} + func (d *diskCache) Set(p string, b []byte) error { if err := os.MkdirAll(d.base, 0700); err != nil { return fmt.Errorf("creating targets dir: %w", err) } - fp := filepath.Join(d.base, p) + fp := filepath.FromSlash(filepath.Join(d.base, p)) return os.WriteFile(fp, b, 0600) } @@ -588,27 +750,6 @@ func noCache() bool { return b } -func newEmbeddedImpl() targetImpl { - e := &embedded{} - if noCache() { - e.setImpl = &memoryCache{} - } else { - e.setImpl = &diskCache{base: cachedTargetsDir(rootCacheDir())} - } - return e -} - -func newFileImpl() targetImpl { - base := cachedTargetsDir(rootCacheDir()) - f := &file{base: base} - if noCache() { - f.setImpl = &memoryCache{} - } else { - f.setImpl = &diskCache{base: base} - } - return f -} - func remoteFromMirror(ctx context.Context, mirror string) (client.RemoteStore, error) { if _, parseErr := url.ParseRequestURI(mirror); parseErr != nil { return GcsRemoteStore(ctx, mirror, nil, nil) diff --git a/pkg/cosign/tuf/client_test.go b/pkg/cosign/tuf/client_test.go index 380441a319e..1a9c7206731 100644 --- a/pkg/cosign/tuf/client_test.go +++ b/pkg/cosign/tuf/client_test.go @@ -19,6 +19,7 @@ import ( "bytes" "context" "encoding/json" + "io/fs" "io/ioutil" "net/http" "net/http/httptest" @@ -28,6 +29,7 @@ import ( "sort" "strings" "testing" + "testing/fstest" "time" "github.com/google/go-cmp/cmp" @@ -66,8 +68,8 @@ func TestNewFromEnv(t *testing.T) { if err != nil { t.Fatal(err) } - tuf.Close() checkTargetsAndMeta(t, tuf) + tuf.Close() if err := Initialize(ctx, DefaultRemoteRoot, nil); err != nil { t.Error() @@ -137,12 +139,12 @@ func TestCache(t *testing.T) { if err != nil { t.Fatal(err) } - tuf.Close() if l := dirLen(t, td); l == 0 { t.Errorf("expected filesystem writes, got %d entries", l) } checkTargetsAndMeta(t, tuf) + tuf.Close() } func TestCustomRoot(t *testing.T) { @@ -308,6 +310,84 @@ func TestGetTargetsByMeta(t *testing.T) { } } +func makeMapFS(repo string) (fs fstest.MapFS) { + fs = make(fstest.MapFS) + _ = filepath.Walk(repo, + func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + rel, _ := filepath.Rel(repo, path) + if info.IsDir() { + fs[filepath.FromSlash(filepath.Join("repository", rel))] = &fstest.MapFile{Mode: os.ModeDir} + } else { + b, _ := os.ReadFile(path) + fs[filepath.FromSlash(filepath.Join("repository", rel))] = &fstest.MapFile{Data: b} + } + return nil + }) + return +} + +// Regression test for failure to locate newly added/renamed targets from a TUF update. +// Previously, the embedded or in-memory targetImpl did not fetch any updated targets. +func TestUpdatedTargetNamesEmbedded(t *testing.T) { + td := t.TempDir() + // Set the TUF_ROOT so we don't interact with other tests and local TUF roots. + t.Setenv("TUF_ROOT", td) + + origEmbedded := GetEmbedded + origDefaultRemote := GetRemoteRoot + defer func() { + GetEmbedded = origEmbedded + GetRemoteRoot = origDefaultRemote + }() + + // Create an "expired" embedded repository. + ctx := context.Background() + store, r := newTufCustomRepo(t, td, "foo") + repository := filepath.FromSlash(filepath.Join(td, "repository")) + mapfs := makeMapFS(repository) + GetEmbedded = func() fs.FS { return mapfs } + + oldIsExpired := verify.IsExpired + isExpiredTimestamp = func(metadata []byte) bool { + m, _ := store.GetMeta() + timestampExpires, _ := getExpiration(m["timestamp.json"]) + metadataExpires, _ := getExpiration(metadata) + return metadataExpires.Sub(*timestampExpires) <= 0 + } + + // Serve an updated remote repository with a new target. + addNewCustomTarget(t, td, r, "newdata") + s := httptest.NewServer(http.FileServer(http.Dir(repository))) + defer s.Close() + GetRemoteRoot = func() string { return s.URL } + + // Initialize. + tufObj, err := NewFromEnv(ctx) + if err != nil { + t.Fatal(err) + } + defer tufObj.Close() + + // Try to retrieve the newly added target. + targets, err := tufObj.GetTargetsByMeta(Fulcio, []string{"fooNoCustom.txt"}) + if err != nil { + t.Fatal(err) + } + if len(targets) != 3 { + t.Fatalf("expected three target without custom metadata, got %d targets", len(targets)) + } + targetBytes := []string{string(targets[0].Target), string(targets[1].Target), string(targets[2].Target)} + expectedTB := []string{"foo", "foo", "newdata"} + if !cmp.Equal(targetBytes, expectedTB, + cmpopts.SortSlices(func(a, b string) bool { return a < b })) { + t.Fatalf("target data mismatched, expected: %v, got: %v", expectedTB, targetBytes) + } + verify.IsExpired = oldIsExpired +} + func checkTargetsAndMeta(t *testing.T, tuf *TUF) { // Check the targets t.Helper() @@ -399,7 +479,7 @@ func newTufCustomRepo(t *testing.T, td string, targetData string) (tuf.LocalStor for name, scm := range map[string]json.RawMessage{ "fooNoCustom.txt": nil, "fooNoCustomOther.txt": nil, "fooActive.txt": scmActive, "fooExpired.txt": scmExpired} { - targetPath := filepath.Join(td, "staged", "targets", name) + targetPath := filepath.FromSlash(filepath.Join(td, "staged", "targets", name)) if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil { t.Error(err) } @@ -422,6 +502,33 @@ func newTufCustomRepo(t *testing.T, td string, targetData string) (tuf.LocalStor return remote, r } +func addNewCustomTarget(t *testing.T, td string, r *tuf.Repo, targetData string) { + scmActive, err := json.Marshal(&sigstoreCustomMetadata{Sigstore: customMetadata{Usage: Fulcio, Status: Active}}) + if err != nil { + t.Error(err) + } + + targetPath := filepath.FromSlash(filepath.Join(td, "staged", "targets", "fooNew.txt")) + if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil { + t.Error(err) + } + if err := ioutil.WriteFile(targetPath, []byte(targetData), 0600); err != nil { + t.Error(err) + } + if err := r.AddTarget("fooNew.txt", scmActive); err != nil { + t.Error(err) + } + if err := r.Snapshot(); err != nil { + t.Error(err) + } + if err := r.Timestamp(); err != nil { + t.Error(err) + } + if err := r.Commit(); err != nil { + t.Error(err) + } +} + func newTufRepo(t *testing.T, td string, targetData string) (tuf.LocalStore, *tuf.Repo) { remote := tuf.FileSystemStore(td, nil) r, err := tuf.NewRepo(remote) @@ -436,7 +543,7 @@ func newTufRepo(t *testing.T, td string, targetData string) (tuf.LocalStore, *tu t.Error(err) } } - targetPath := filepath.Join(td, "staged", "targets", "foo.txt") + targetPath := filepath.FromSlash(filepath.Join(td, "staged", "targets", "foo.txt")) if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil { t.Error(err) } @@ -459,7 +566,7 @@ func newTufRepo(t *testing.T, td string, targetData string) (tuf.LocalStore, *tu } func updateTufRepo(t *testing.T, td string, r *tuf.Repo, targetData string) { - targetPath := filepath.Join(td, "staged", "targets", "foo.txt") + targetPath := filepath.FromSlash(filepath.Join(td, "staged", "targets", "foo.txt")) if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil { t.Error(err) } From 9b337a54aaf6b7741297dc70f7c054c8bb42439d Mon Sep 17 00:00:00 2001 From: Asra Ali Date: Wed, 25 May 2022 18:53:08 -0500 Subject: [PATCH 02/11] update zack comments Signed-off-by: Asra Ali update fix Signed-off-by: Asra Ali update and add some debug Signed-off-by: Asra Ali add debug Signed-off-by: Asra Ali no cache Signed-off-by: Asra Ali remove debug Signed-off-by: Asra Ali --- pkg/cosign/tlog.go | 18 ++-- pkg/cosign/tuf/client.go | 163 ++++++++++++++++++++++------------ pkg/cosign/tuf/client_test.go | 44 +++++---- 3 files changed, 144 insertions(+), 81 deletions(-) diff --git a/pkg/cosign/tlog.go b/pkg/cosign/tlog.go index a23cd2474c6..f1110d10294 100644 --- a/pkg/cosign/tlog.go +++ b/pkg/cosign/tlog.go @@ -78,15 +78,6 @@ func getLogID(pub crypto.PublicKey) (string, error) { // GetRekorPubs retrieves trusted Rekor public keys from the embedded or cached // TUF root. If expired, makes a network call to retrieve the updated targets. func GetRekorPubs(ctx context.Context) (map[string]RekorPubKey, error) { - tufClient, err := tuf.NewFromEnv(ctx) - if err != nil { - return nil, err - } - defer tufClient.Close() - targets, err := tufClient.GetTargetsByMeta(tuf.Rekor, []string{rekorTargetStr}) - if err != nil { - return nil, err - } publicKeys := make(map[string]RekorPubKey) altRekorPub := os.Getenv(altRekorPublicKey) if altRekorPub != "" { @@ -105,6 +96,15 @@ func GetRekorPubs(ctx context.Context) (map[string]RekorPubKey, error) { } publicKeys[keyID] = RekorPubKey{PubKey: extra, Status: tuf.Active} } else { + tufClient, err := tuf.NewFromEnv(ctx) + if err != nil { + return nil, err + } + defer tufClient.Close() + targets, err := tufClient.GetTargetsByMeta(tuf.Rekor, []string{rekorTargetStr}) + if err != nil { + return nil, err + } for _, t := range targets { rekorPubKey, err := PemToECDSAKey(t.Target) if err != nil { diff --git a/pkg/cosign/tuf/client.go b/pkg/cosign/tuf/client.go index cb3abddb180..1d2ac38efc5 100644 --- a/pkg/cosign/tuf/client.go +++ b/pkg/cosign/tuf/client.go @@ -94,6 +94,10 @@ type signedMeta struct { Version int64 `json:"version"` } +var ( + errEmbeddedMetadataNeedsUpdate = fmt.Errorf("new metadata requires an unsupported write operation") +) + // RemoteCache contains information to cache on the location of the remote // repository. type remoteCache struct { @@ -148,6 +152,19 @@ func getMetadataStatus(b []byte) (*MetadataStatus, error) { }, nil } +func isMetaEqual(x json.RawMessage, y json.RawMessage) (bool, error) { + stored, err := cjson.EncodeCanonical(x) + if err != nil { + return false, err + } + toSet, err := cjson.EncodeCanonical(y) + if err != nil { + return false, err + } + cmp := bytes.EqualFold(stored, toSet) + return cmp, nil +} + func (t *TUF) getRootStatus() (*RootStatus, error) { local := "embedded" if !t.embedded { @@ -186,13 +203,13 @@ func (t *TUF) getRootStatus() (*RootStatus, error) { return status, nil } -func getRoot(meta map[string]json.RawMessage) (json.RawMessage, error) { +func getRoot(meta map[string]json.RawMessage, ed fs.FS) (json.RawMessage, error) { trustedRoot, ok := meta["root.json"] if ok { return trustedRoot, nil } // On first initialize, there will be no root in the TUF DB, so read from embedded. - rd, ok := GetEmbedded().(fs.ReadFileFS) + rd, ok := ed.(fs.ReadFileFS) if !ok { return nil, errors.New("fs.ReadFileFS unimplemented for embedded repo") } @@ -236,9 +253,10 @@ func initializeTUF(ctx context.Context, embed bool, mirror string, root []byte, // Lazily init the local store in case we start with an embedded. var initLocal = newLocalStore var err error + embeddedRepo := GetEmbedded() if t.embedded { - t.targets = wrapEmbedded(t.targets) - t.local = wrapEmbeddedLocal(initLocal) + t.targets = wrapEmbedded(embeddedRepo, t.targets) + t.local = wrapEmbeddedLocal(embeddedRepo, initLocal) } else { t.local, err = initLocal() if err != nil { @@ -261,7 +279,7 @@ func initializeTUF(ctx context.Context, embed bool, mirror string, root []byte, } if root == nil { - root, err = getRoot(trustedMeta) + root, err = getRoot(trustedMeta, embeddedRepo) if err != nil { t.Close() return nil, fmt.Errorf("getting trusted root: %w", err) @@ -515,30 +533,20 @@ func newLocalStore() (client.LocalStore, error) { return local, nil } -type localInitFunc func() (client.LocalStore, error) - -type embeddedLocalStore struct { - // This is where updated repository info will be written to, if needed. - writeable client.LocalStore - writeableFunc localInitFunc -} +type localStoreInit func() (client.LocalStore, error) var GetEmbedded = func() fs.FS { return embeddedRootRepo } -func wrapEmbeddedLocal(s localInitFunc) client.LocalStore { - return &embeddedLocalStore{writeableFunc: s, writeable: nil} +// A read-only local store using embedded metadata +type embeddedLocalStore struct { + ed fs.FS } func (e embeddedLocalStore) GetMeta() (map[string]json.RawMessage, error) { - if e.writeable != nil { - // We are using a writeable store, so use that. - return e.writeable.GetMeta() - } - // We haven't needed to create or write new metadata, so get the embedded metadata. meta := make(map[string]json.RawMessage) - ed, ok := GetEmbedded().(fs.ReadDirFS) + ed, ok := e.ed.(fs.ReadDirFS) if !ok { return nil, errors.New("fs.ReadDirFS unimplemented for embedded repo") } @@ -546,49 +554,86 @@ func (e embeddedLocalStore) GetMeta() (map[string]json.RawMessage, error) { if err != nil { return nil, err } - rd, ok := GetEmbedded().(fs.ReadFileFS) + rd, ok := e.ed.(fs.ReadFileFS) if !ok { return nil, errors.New("fs.ReadFileFS unimplemented for embedded repo") } for _, entry := range entries { - if entry.Type().IsRegular() { - b, err := rd.ReadFile(filepath.FromSlash(filepath.Join("repository", entry.Name()))) - if err != nil { - return nil, fmt.Errorf("reading embedded file: %w", err) - } - meta[entry.Name()] = b + if !entry.Type().IsRegular() { + // Skip the target directory or other strange files. + continue + } + b, err := rd.ReadFile(filepath.FromSlash(filepath.Join("repository", entry.Name()))) + if err != nil { + return nil, fmt.Errorf("reading embedded file: %w", err) } + meta[entry.Name()] = b } return meta, err } func (e *embeddedLocalStore) SetMeta(name string, meta json.RawMessage) error { - // Only write to the underlying writeable store if the metadata is new. + // Return no error if no real "write" is required: the meta matches the embedded content. embeddedMeta, err := e.GetMeta() if err != nil { return err } - if metaContent, ok := embeddedMeta[name]; ok { - stored, err := cjson.EncodeCanonical(metaContent) - if err != nil { - return err - } - toSet, err := cjson.EncodeCanonical(meta) - if err != nil { - return err - } - if bytes.EqualFold(stored, toSet) { - return nil - } + metaContent, ok := embeddedMeta[name] + if !ok { + return errEmbeddedMetadataNeedsUpdate } - // If we reach here, we have to update metadata. + equal, err := isMetaEqual(metaContent, meta) + if err != nil { + return fmt.Errorf("error comparing metadata: %w", err) + } + if equal { + return nil + } + return errEmbeddedMetadataNeedsUpdate +} + +func (e embeddedLocalStore) DeleteMeta(name string) error { + return errors.New("attempting to delete embedded metadata") +} + +func (e embeddedLocalStore) Close() error { + return nil +} + +type wrappedEmbeddedLocalStore struct { + // The read-only embedded local store. + embedded client.LocalStore + // Initially nil, initialized with makeWriteableStore once a write operation is needed. + writeable client.LocalStore + makeWriteableStore localStoreInit +} + +func wrapEmbeddedLocal(ed fs.FS, s localStoreInit) client.LocalStore { + return &wrappedEmbeddedLocalStore{embedded: &embeddedLocalStore{ed: ed}, makeWriteableStore: s, writeable: nil} +} + +func (e wrappedEmbeddedLocalStore) GetMeta() (map[string]json.RawMessage, error) { + if e.writeable != nil { + // We are using a writeable store, so use that. + return e.writeable.GetMeta() + } + // We haven't needed to create or write new metadata, so get the embedded metadata. + return e.embedded.GetMeta() +} + +func (e *wrappedEmbeddedLocalStore) SetMeta(name string, meta json.RawMessage) error { if e.writeable == nil { + // Check if we the set operation "succeeds" for the read-only embedded store. + // This only succeeds if the metadata matches the embedded (i.e. no-op). + if err := e.embedded.SetMeta(name, meta); err == nil || !errors.Is(err, errEmbeddedMetadataNeedsUpdate) { + return err + } // We haven't needed an update yet, so create and populate the writeable store! meta, err := e.GetMeta() if err != nil { return fmt.Errorf("error retrieving embedded repo: %w", err) } - e.writeable, err = e.writeableFunc() + e.writeable, err = e.makeWriteableStore() if err != nil { return fmt.Errorf("initializing local: %w", err) } @@ -602,18 +647,18 @@ func (e *embeddedLocalStore) SetMeta(name string, meta json.RawMessage) error { return e.writeable.SetMeta(name, meta) } -func (e embeddedLocalStore) DeleteMeta(name string) error { +func (e wrappedEmbeddedLocalStore) DeleteMeta(name string) error { if e.writeable != nil { return e.writeable.DeleteMeta(name) } - return nil + return e.embedded.DeleteMeta(name) } -func (e embeddedLocalStore) Close() error { +func (e wrappedEmbeddedLocalStore) Close() error { if e.writeable != nil { return e.writeable.Close() } - return nil + return e.embedded.Close() } //go:embed repository @@ -632,20 +677,21 @@ func newFileImpl() targetImpl { return &diskCache{base: cachedTargetsDir(rootCacheDir())} } -func wrapEmbedded(t targetImpl) targetImpl { - return &embeddedWrapper{writeable: t, embedded: true} +func wrapEmbedded(ed fs.FS, t targetImpl) targetImpl { + return &embeddedWrapper{embeddedRepo: ed, writeable: t, modified: false} } type embeddedWrapper struct { + embeddedRepo fs.FS // If we have an embedded fallback that needs updates, use // the writeable targetImpl writeable targetImpl - // Whether we are using the embedded or writeable - embedded bool + // Whether we modified targets and need to fetch from the writeable target store. + modified bool } func (e *embeddedWrapper) Get(p string) ([]byte, error) { - if !e.embedded { + if e.modified { // Get it from the writeable target store since there's been updates. b, err := e.writeable.Get(p) if err == nil { @@ -653,7 +699,7 @@ func (e *embeddedWrapper) Get(p string) ([]byte, error) { } fmt.Fprintf(os.Stderr, "**Warning** Updated target not found; falling back on embedded target %s\n", p) } - rd, ok := GetEmbedded().(fs.ReadFileFS) + rd, ok := e.embeddedRepo.(fs.ReadFileFS) if !ok { return nil, errors.New("fs.ReadFileFS unimplemented for embedded repo") } @@ -671,9 +717,9 @@ func (e *embeddedWrapper) Get(p string) ([]byte, error) { } func (e *embeddedWrapper) Set(name string, b []byte) error { - // An embedded targetImpl needs to set a file, copy files to cache. - if e.embedded { - ed, ok := GetEmbedded().(fs.ReadDirFS) + // If Set is called, our embedded cache is busted so we need to move over to the writeable targetsImpl. + if !e.modified { + ed, ok := e.embeddedRepo.(fs.ReadDirFS) if !ok { return errors.New("fs.ReadFileFS unimplemented for embedded repo") } @@ -681,10 +727,11 @@ func (e *embeddedWrapper) Set(name string, b []byte) error { if err != nil { return err } - rd, ok := GetEmbedded().(fs.ReadFileFS) + rd, ok := e.embeddedRepo.(fs.ReadFileFS) if !ok { return errors.New("fs.ReadFileFS unimplemented for embedded repo") } + // Copy targets to the writeable store so we can find all of them later. for _, entry := range entries { b, err := rd.ReadFile(filepath.FromSlash(filepath.Join("repository", "targets", entry.Name()))) if err != nil { @@ -694,8 +741,10 @@ func (e *embeddedWrapper) Set(name string, b []byte) error { return fmt.Errorf("setting embedded file: %w", err) } } - e.embedded = false } + // Now that we Set a target, we are now in a "modified" state and must check the writeable + // store for targets. + e.modified = true return e.writeable.Set(name, b) } diff --git a/pkg/cosign/tuf/client_test.go b/pkg/cosign/tuf/client_test.go index 1a9c7206731..cd57e54c311 100644 --- a/pkg/cosign/tuf/client_test.go +++ b/pkg/cosign/tuf/client_test.go @@ -329,8 +329,9 @@ func makeMapFS(repo string) (fs fstest.MapFS) { return } -// Regression test for failure to locate newly added/renamed targets from a TUF update. -// Previously, the embedded or in-memory targetImpl did not fetch any updated targets. +// Regression test for failure to fetch a target that does not exist in the embedded +// repository on an update. The new target exists on the remote before the TUF object +// is initialized. func TestUpdatedTargetNamesEmbedded(t *testing.T) { td := t.TempDir() // Set the TUF_ROOT so we don't interact with other tests and local TUF roots. @@ -343,7 +344,7 @@ func TestUpdatedTargetNamesEmbedded(t *testing.T) { GetRemoteRoot = origDefaultRemote }() - // Create an "expired" embedded repository. + // Create an "expired" embedded repository that does not contain newTarget. ctx := context.Background() store, r := newTufCustomRepo(t, td, "foo") repository := filepath.FromSlash(filepath.Join(td, "repository")) @@ -358,8 +359,18 @@ func TestUpdatedTargetNamesEmbedded(t *testing.T) { return metadataExpires.Sub(*timestampExpires) <= 0 } - // Serve an updated remote repository with a new target. - addNewCustomTarget(t, td, r, "newdata") + // Assert that the embedded repository does not contain the newTarget. + newTarget := "fooNew.txt" + rd, ok := GetEmbedded().(fs.ReadFileFS) + if !ok { + t.Fatal("fs.ReadFileFS unimplemented for embedded repo") + } + if _, err := rd.ReadFile(filepath.FromSlash(filepath.Join("repository", "targets", newTarget))); err == nil { + t.Fatal("embedded repository should not contain new target") + } + + // Serve an updated remote repository with the newTarget. + addNewCustomTarget(t, td, r, map[string]string{newTarget: "newdata"}) s := httptest.NewServer(http.FileServer(http.Dir(repository))) defer s.Close() GetRemoteRoot = func() string { return s.URL } @@ -502,22 +513,25 @@ func newTufCustomRepo(t *testing.T, td string, targetData string) (tuf.LocalStor return remote, r } -func addNewCustomTarget(t *testing.T, td string, r *tuf.Repo, targetData string) { +func addNewCustomTarget(t *testing.T, td string, r *tuf.Repo, targetData map[string]string) { scmActive, err := json.Marshal(&sigstoreCustomMetadata{Sigstore: customMetadata{Usage: Fulcio, Status: Active}}) if err != nil { t.Error(err) } - targetPath := filepath.FromSlash(filepath.Join(td, "staged", "targets", "fooNew.txt")) - if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil { - t.Error(err) - } - if err := ioutil.WriteFile(targetPath, []byte(targetData), 0600); err != nil { - t.Error(err) - } - if err := r.AddTarget("fooNew.txt", scmActive); err != nil { - t.Error(err) + for name, data := range targetData { + targetPath := filepath.FromSlash(filepath.Join(td, "staged", "targets", name)) + if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil { + t.Error(err) + } + if err := ioutil.WriteFile(targetPath, []byte(data), 0600); err != nil { + t.Error(err) + } + if err := r.AddTarget(name, scmActive); err != nil { + t.Error(err) + } } + if err := r.Snapshot(); err != nil { t.Error(err) } From 81a86c3c4b79545d0a009ebb4d4ec77a8c1b76e8 Mon Sep 17 00:00:00 2001 From: Asra Ali Date: Fri, 27 May 2022 15:37:19 -0500 Subject: [PATCH 03/11] try haydens comments Signed-off-by: Asra Ali --- pkg/cosign/tuf/client.go | 380 ++++++----------------- pkg/cosign/tuf/client_test.go | 37 ++- pkg/cosign/tuf/repository/1.root.json | 130 -------- pkg/cosign/tuf/repository/2.root.json | 144 --------- pkg/cosign/tuf/repository/rekor.json | 23 -- pkg/cosign/tuf/repository/snapshot.json | 48 --- pkg/cosign/tuf/repository/staging.json | 15 - pkg/cosign/tuf/repository/targets.json | 117 ------- pkg/cosign/tuf/repository/timestamp.json | 24 -- 9 files changed, 120 insertions(+), 798 deletions(-) delete mode 100644 pkg/cosign/tuf/repository/1.root.json delete mode 100644 pkg/cosign/tuf/repository/2.root.json delete mode 100644 pkg/cosign/tuf/repository/rekor.json delete mode 100644 pkg/cosign/tuf/repository/snapshot.json delete mode 100644 pkg/cosign/tuf/repository/staging.json delete mode 100644 pkg/cosign/tuf/repository/targets.json delete mode 100644 pkg/cosign/tuf/repository/timestamp.json diff --git a/pkg/cosign/tuf/client.go b/pkg/cosign/tuf/client.go index 1d2ac38efc5..66d16df6bdf 100644 --- a/pkg/cosign/tuf/client.go +++ b/pkg/cosign/tuf/client.go @@ -27,13 +27,12 @@ import ( "io/ioutil" "net/url" "os" + "path" "path/filepath" - "runtime" "strconv" "strings" "time" - "github.com/secure-systems-lab/go-securesystemslib/cjson" "github.com/theupdateframework/go-tuf/client" tuf_leveldbstore "github.com/theupdateframework/go-tuf/client/leveldbstore" "github.com/theupdateframework/go-tuf/data" @@ -46,6 +45,11 @@ const ( SigstoreNoCache = "SIGSTORE_NO_CACHE" ) +// Global in-memory targets to avoid re-downloading when there is no local cache. +// TODO: Consider using this map even when local caching to avoid reading from disk +// multiple times (e.g. when there are multiple signatures to verify in a single call). +var memoryTargets = map[string][]byte{} + var GetRemoteRoot = func() string { return DefaultRemoteRoot } @@ -55,7 +59,7 @@ type TUF struct { targets targetImpl local client.LocalStore remote client.RemoteStore - embedded bool // local embedded or cache + embedded fs.FS mirror string // location of mirror } @@ -94,10 +98,6 @@ type signedMeta struct { Version int64 `json:"version"` } -var ( - errEmbeddedMetadataNeedsUpdate = fmt.Errorf("new metadata requires an unsupported write operation") -) - // RemoteCache contains information to cache on the location of the remote // repository. type remoteCache struct { @@ -152,23 +152,10 @@ func getMetadataStatus(b []byte) (*MetadataStatus, error) { }, nil } -func isMetaEqual(x json.RawMessage, y json.RawMessage) (bool, error) { - stored, err := cjson.EncodeCanonical(x) - if err != nil { - return false, err - } - toSet, err := cjson.EncodeCanonical(y) - if err != nil { - return false, err - } - cmp := bytes.EqualFold(stored, toSet) - return cmp, nil -} - func (t *TUF) getRootStatus() (*RootStatus, error) { - local := "embedded" - if !t.embedded { - local = rootCacheDir() + local := rootCacheDir() + if noCache() { + local = "in-memory" } status := &RootStatus{ Local: local, @@ -203,13 +190,13 @@ func (t *TUF) getRootStatus() (*RootStatus, error) { return status, nil } -func getRoot(meta map[string]json.RawMessage, ed fs.FS) (json.RawMessage, error) { +func getRoot(meta map[string]json.RawMessage, fallback fs.FS) (json.RawMessage, error) { trustedRoot, ok := meta["root.json"] if ok { return trustedRoot, nil } // On first initialize, there will be no root in the TUF DB, so read from embedded. - rd, ok := ed.(fs.ReadFileFS) + rd, ok := fallback.(fs.ReadFileFS) if !ok { return nil, errors.New("fs.ReadFileFS unimplemented for embedded repo") } @@ -243,25 +230,17 @@ func (t *TUF) Close() error { // defaults to the embedded root.json. // * forceUpdate: indicates checking the remote for an update, even when the local // timestamp.json is up to date. -func initializeTUF(ctx context.Context, embed bool, mirror string, root []byte, forceUpdate bool) (*TUF, error) { +func initializeTUF(ctx context.Context, mirror string, root []byte, forceUpdate bool) (*TUF, error) { t := &TUF{ mirror: mirror, - embedded: embed, + embedded: GetEmbedded(), } t.targets = newFileImpl() - // Lazily init the local store in case we start with an embedded. - var initLocal = newLocalStore var err error - embeddedRepo := GetEmbedded() - if t.embedded { - t.targets = wrapEmbedded(embeddedRepo, t.targets) - t.local = wrapEmbeddedLocal(embeddedRepo, initLocal) - } else { - t.local, err = initLocal() - if err != nil { - return nil, err - } + t.local, err = newLocalStore() + if err != nil { + return nil, err } t.remote, err = remoteFromMirror(ctx, t.mirror) @@ -278,8 +257,10 @@ func initializeTUF(ctx context.Context, embed bool, mirror string, root []byte, return nil, fmt.Errorf("getting trusted meta: %w", err) } + // If the caller does not supply a root, then either use the root in the local store + // or default to the embedded one. if root == nil { - root, err = getRoot(trustedMeta, embeddedRepo) + root, err = getRoot(trustedMeta, t.embedded) if err != nil { t.Close() return nil, fmt.Errorf("getting trusted root: %w", err) @@ -291,14 +272,13 @@ func initializeTUF(ctx context.Context, embed bool, mirror string, root []byte, return nil, fmt.Errorf("unable to initialize client, local cache may be corrupt: %w", err) } - // We have our local store, whether it was embedded or not! - // Now check to see if it needs to be updated. + // We may already have an up-to-date local store! Check to see if it needs to be updated. trustedTimestamp, ok := trustedMeta["timestamp.json"] if ok && !isExpiredTimestamp(trustedTimestamp) && !forceUpdate { return t, nil } - // Update when timestamp is out of date. + // Update if local is not populated or out of date. if err := t.updateMetadataAndDownloadTargets(); err != nil { t.Close() return nil, fmt.Errorf("updating local metadata and targets: %w", err) @@ -308,24 +288,6 @@ func initializeTUF(ctx context.Context, embed bool, mirror string, root []byte, } func NewFromEnv(ctx context.Context) (*TUF, error) { - // Get local and mirror from env - tufDB := filepath.FromSlash(filepath.Join(rootCacheDir(), "tuf.db")) - var embed bool - - // Check for the current local. - _, statErr := os.Stat(tufDB) - switch { - case os.IsNotExist(statErr): - // There is no root at the location, use embedded. - embed = true - case statErr != nil: - // Some other error, bail - return nil, statErr - default: - // There is a local root! Happy path. - embed = false - } - // Check for the current remote mirror. mirror := GetRemoteRoot() b, err := os.ReadFile(cachedRemote(rootCacheDir())) @@ -337,27 +299,41 @@ func NewFromEnv(ctx context.Context) (*TUF, error) { } // Initializes a new TUF object from the local cache or defaults. - return initializeTUF(ctx, embed, mirror, nil, false) + return initializeTUF(ctx, mirror, nil, false) } func Initialize(ctx context.Context, mirror string, root []byte) error { // Initialize the client. Force an update. - t, err := initializeTUF(ctx, false, mirror, root, true) + t, err := initializeTUF(ctx, mirror, root, true) if err != nil { return err } t.Close() - // Store the remote for later. - remoteInfo := &remoteCache{Mirror: mirror} - b, err := json.Marshal(remoteInfo) + // Store the remote for later if we are caching. + if !noCache() { + remoteInfo := &remoteCache{Mirror: mirror} + b, err := json.Marshal(remoteInfo) + if err != nil { + return err + } + if err := os.WriteFile(cachedRemote(rootCacheDir()), b, 0600); err != nil { + return fmt.Errorf("storing remote: %w", err) + } + } + return nil +} + +// Checks if the testTarget matches the valid target file metadata. +func isValidTarget(testTarget []byte, validMeta data.TargetFileMeta) bool { + localMeta, err := util.GenerateTargetFileMeta(bytes.NewReader(testTarget)) if err != nil { - return err + return false } - if err := os.WriteFile(cachedRemote(rootCacheDir()), b, 0600); err != nil { - return fmt.Errorf("storing remote: %w", err) + if err := util.TargetFileMetaEqual(localMeta, validMeta); err != nil { + return false } - return nil + return true } func (t *TUF) GetTarget(name string) ([]byte, error) { @@ -371,12 +347,8 @@ func (t *TUF) GetTarget(name string) ([]byte, error) { return nil, err } - localMeta, err := util.GenerateTargetFileMeta(bytes.NewReader(targetBytes)) - if err != nil { - return nil, err - } - if err := util.TargetFileMetaEqual(localMeta, validMeta); err != nil { - return nil, err + if !isValidTarget(targetBytes, validMeta) { + return nil, fmt.Errorf("cache contains invalid target; local cache may be corrupt") } return targetBytes, nil @@ -463,14 +435,10 @@ func (t *TUF) updateMetadataAndDownloadTargets() error { return fmt.Errorf("error updating to TUF remote mirror: %w\nremote status:%s", err, string(b)) } - // Download newly updated targets. + // Download **newly** updated targets. // TODO: Consider lazily downloading these -- be careful with embedded targets if so. - for name := range targetFiles { - buf := bytes.Buffer{} - if err := downloadRemoteTarget(name, t.client, &buf); err != nil { - return err - } - if err := t.targets.Set(name, buf.Bytes()); err != nil { + for name, targetMeta := range targetFiles { + if err := maybeDownloadRemoteTarget(name, targetMeta, t); err != nil { return err } } @@ -479,7 +447,7 @@ func (t *TUF) updateMetadataAndDownloadTargets() error { } type targetDestination struct { - buf bytes.Buffer + buf *bytes.Buffer } func (t *targetDestination) Write(b []byte) (int, error) { @@ -487,17 +455,47 @@ func (t *targetDestination) Write(b []byte) (int, error) { } func (t *targetDestination) Delete() error { - t.buf = bytes.Buffer{} + t.buf = &bytes.Buffer{} return nil } -func downloadRemoteTarget(name string, c *client.Client, w io.Writer) error { - dest := targetDestination{} - if err := c.Download(name, &dest); err != nil { - return fmt.Errorf("downloading target: %w", err) +func maybeDownloadRemoteTarget(name string, meta data.TargetFileMeta, t *TUF) error { + // If we already have the target locally, don't bother downloading from remote storage. + if cachedTarget, err := t.targets.Get(name); err == nil { + // If the target we have stored matches the meta, use that. + if isValidTarget(cachedTarget, meta) { + return nil + } + } + + // Check if we already have the target in the embedded store. + w := bytes.Buffer{} + rd, ok := t.embedded.(fs.ReadFileFS) + if !ok { + return errors.New("fs.ReadFileFS unimplemented for embedded repo") + } + b, err := rd.ReadFile(path.Join("repository", "targets", name)) + if err == nil { + if isValidTarget(b, meta) { + if _, err := io.Copy(&w, bytes.NewReader(b)); err != nil { + return fmt.Errorf("using embedded target: %w", err) + } + } + } + + // Nope -- no local matching target, go download it. + if w.Len() == 0 { + dest := targetDestination{buf: &w} + if err := t.client.Download(name, &dest); err != nil { + return fmt.Errorf("downloading target: %w", err) + } } - _, err := io.Copy(w, &dest.buf) - return err + + // Set the target in the cache. + if err := t.targets.Set(name, w.Bytes()); err != nil { + return err + } + return nil } func rootCacheDir() string { @@ -533,137 +531,13 @@ func newLocalStore() (client.LocalStore, error) { return local, nil } -type localStoreInit func() (client.LocalStore, error) +//go:embed repository +var embeddedRootRepo embed.FS var GetEmbedded = func() fs.FS { return embeddedRootRepo } -// A read-only local store using embedded metadata -type embeddedLocalStore struct { - ed fs.FS -} - -func (e embeddedLocalStore) GetMeta() (map[string]json.RawMessage, error) { - meta := make(map[string]json.RawMessage) - ed, ok := e.ed.(fs.ReadDirFS) - if !ok { - return nil, errors.New("fs.ReadDirFS unimplemented for embedded repo") - } - entries, err := ed.ReadDir("repository") - if err != nil { - return nil, err - } - rd, ok := e.ed.(fs.ReadFileFS) - if !ok { - return nil, errors.New("fs.ReadFileFS unimplemented for embedded repo") - } - for _, entry := range entries { - if !entry.Type().IsRegular() { - // Skip the target directory or other strange files. - continue - } - b, err := rd.ReadFile(filepath.FromSlash(filepath.Join("repository", entry.Name()))) - if err != nil { - return nil, fmt.Errorf("reading embedded file: %w", err) - } - meta[entry.Name()] = b - } - return meta, err -} - -func (e *embeddedLocalStore) SetMeta(name string, meta json.RawMessage) error { - // Return no error if no real "write" is required: the meta matches the embedded content. - embeddedMeta, err := e.GetMeta() - if err != nil { - return err - } - metaContent, ok := embeddedMeta[name] - if !ok { - return errEmbeddedMetadataNeedsUpdate - } - equal, err := isMetaEqual(metaContent, meta) - if err != nil { - return fmt.Errorf("error comparing metadata: %w", err) - } - if equal { - return nil - } - return errEmbeddedMetadataNeedsUpdate -} - -func (e embeddedLocalStore) DeleteMeta(name string) error { - return errors.New("attempting to delete embedded metadata") -} - -func (e embeddedLocalStore) Close() error { - return nil -} - -type wrappedEmbeddedLocalStore struct { - // The read-only embedded local store. - embedded client.LocalStore - // Initially nil, initialized with makeWriteableStore once a write operation is needed. - writeable client.LocalStore - makeWriteableStore localStoreInit -} - -func wrapEmbeddedLocal(ed fs.FS, s localStoreInit) client.LocalStore { - return &wrappedEmbeddedLocalStore{embedded: &embeddedLocalStore{ed: ed}, makeWriteableStore: s, writeable: nil} -} - -func (e wrappedEmbeddedLocalStore) GetMeta() (map[string]json.RawMessage, error) { - if e.writeable != nil { - // We are using a writeable store, so use that. - return e.writeable.GetMeta() - } - // We haven't needed to create or write new metadata, so get the embedded metadata. - return e.embedded.GetMeta() -} - -func (e *wrappedEmbeddedLocalStore) SetMeta(name string, meta json.RawMessage) error { - if e.writeable == nil { - // Check if we the set operation "succeeds" for the read-only embedded store. - // This only succeeds if the metadata matches the embedded (i.e. no-op). - if err := e.embedded.SetMeta(name, meta); err == nil || !errors.Is(err, errEmbeddedMetadataNeedsUpdate) { - return err - } - // We haven't needed an update yet, so create and populate the writeable store! - meta, err := e.GetMeta() - if err != nil { - return fmt.Errorf("error retrieving embedded repo: %w", err) - } - e.writeable, err = e.makeWriteableStore() - if err != nil { - return fmt.Errorf("initializing local: %w", err) - } - for m, md := range meta { - if err := e.writeable.SetMeta(m, md); err != nil { - return fmt.Errorf("error transferring to cached repo: %w", err) - } - } - } - // We have a writeable store, so set the metadata. - return e.writeable.SetMeta(name, meta) -} - -func (e wrappedEmbeddedLocalStore) DeleteMeta(name string) error { - if e.writeable != nil { - return e.writeable.DeleteMeta(name) - } - return e.embedded.DeleteMeta(name) -} - -func (e wrappedEmbeddedLocalStore) Close() error { - if e.writeable != nil { - return e.writeable.Close() - } - return e.embedded.Close() -} - -//go:embed repository -var embeddedRootRepo embed.FS - // Target Implementations type targetImpl interface { Set(string, []byte) error @@ -672,82 +546,11 @@ type targetImpl interface { func newFileImpl() targetImpl { if noCache() { - return &memoryCache{} + return &memoryCache{targets: memoryTargets} } return &diskCache{base: cachedTargetsDir(rootCacheDir())} } -func wrapEmbedded(ed fs.FS, t targetImpl) targetImpl { - return &embeddedWrapper{embeddedRepo: ed, writeable: t, modified: false} -} - -type embeddedWrapper struct { - embeddedRepo fs.FS - // If we have an embedded fallback that needs updates, use - // the writeable targetImpl - writeable targetImpl - // Whether we modified targets and need to fetch from the writeable target store. - modified bool -} - -func (e *embeddedWrapper) Get(p string) ([]byte, error) { - if e.modified { - // Get it from the writeable target store since there's been updates. - b, err := e.writeable.Get(p) - if err == nil { - return b, nil - } - fmt.Fprintf(os.Stderr, "**Warning** Updated target not found; falling back on embedded target %s\n", p) - } - rd, ok := e.embeddedRepo.(fs.ReadFileFS) - if !ok { - return nil, errors.New("fs.ReadFileFS unimplemented for embedded repo") - } - b, err := rd.ReadFile(filepath.FromSlash(filepath.Join("repository", "targets", p))) - if err != nil { - return nil, err - } - // Unfortunately go:embed appears to somehow replace our line endings on windows, we need to switch them back. - // It should theoretically be safe to do this everywhere - but the files only seem to get mutated on Windows so - // let's only change them back there. - if runtime.GOOS == "windows" { - return bytes.ReplaceAll(b, []byte("\r\n"), []byte("\n")), nil - } - return b, nil -} - -func (e *embeddedWrapper) Set(name string, b []byte) error { - // If Set is called, our embedded cache is busted so we need to move over to the writeable targetsImpl. - if !e.modified { - ed, ok := e.embeddedRepo.(fs.ReadDirFS) - if !ok { - return errors.New("fs.ReadFileFS unimplemented for embedded repo") - } - entries, err := ed.ReadDir(filepath.FromSlash(filepath.Join("repository", "targets"))) - if err != nil { - return err - } - rd, ok := e.embeddedRepo.(fs.ReadFileFS) - if !ok { - return errors.New("fs.ReadFileFS unimplemented for embedded repo") - } - // Copy targets to the writeable store so we can find all of them later. - for _, entry := range entries { - b, err := rd.ReadFile(filepath.FromSlash(filepath.Join("repository", "targets", entry.Name()))) - if err != nil { - return fmt.Errorf("reading embedded file: %w", err) - } - if err := e.writeable.Set(entry.Name(), b); err != nil { - return fmt.Errorf("setting embedded file: %w", err) - } - } - } - // Now that we Set a target, we are now in a "modified" state and must check the writeable - // store for targets. - e.modified = true - return e.writeable.Set(name, b) -} - // In-memory cache for targets type memoryCache struct { targets map[string][]byte @@ -763,7 +566,6 @@ func (m *memoryCache) Set(p string, b []byte) error { func (m *memoryCache) Get(p string) ([]byte, error) { if m.targets == nil { - // This should never happen, a memory cache is used only after items are Set. return nil, fmt.Errorf("no cached targets available, cannot retrieve %s", p) } b, ok := m.targets[p] diff --git a/pkg/cosign/tuf/client_test.go b/pkg/cosign/tuf/client_test.go index cd57e54c311..fb89a7f1404 100644 --- a/pkg/cosign/tuf/client_test.go +++ b/pkg/cosign/tuf/client_test.go @@ -94,16 +94,25 @@ func TestNoCache(t *testing.T) { td := t.TempDir() t.Setenv("TUF_ROOT", td) + // First initialization, populate the cache. + tuf, err := NewFromEnv(ctx) + if err != nil { + t.Fatal(err) + } + checkTargetsAndMeta(t, tuf) + tuf.Close() + // Force expiration so we have some content to download forceExpiration(t, true) - tuf, err := NewFromEnv(ctx) + tuf, err = NewFromEnv(ctx) if err != nil { t.Fatal(err) } checkTargetsAndMeta(t, tuf) tuf.Close() + // No filesystem writes when using SIGSTORE_NO_CACHE. if l := dirLen(t, td); l != 0 { t.Errorf("expected no filesystem writes, got %d entries", l) } @@ -111,7 +120,7 @@ func TestNoCache(t *testing.T) { func TestCache(t *testing.T) { ctx := context.Background() - // Once more with NO_CACHE + // Once more with cache. t.Setenv("SIGSTORE_NO_CACHE", "false") td := t.TempDir() t.Setenv("TUF_ROOT", td) @@ -121,26 +130,38 @@ func TestCache(t *testing.T) { t.Errorf("expected no filesystem writes, got %d entries", l) } - // Nothing should get downloaded if everything is up to date - forceExpiration(t, false) + // First initialization, populate the cache. Expect disk writes. tuf, err := NewFromEnv(ctx) if err != nil { t.Fatal(err) } + checkTargetsAndMeta(t, tuf) tuf.Close() + cachedDirLen := dirLen(t, td) + if cachedDirLen == 0 { + t.Errorf("expected filesystem writes, got %d entries", cachedDirLen) + } - if l := dirLen(t, td); l != 0 { - t.Errorf("expected no filesystem writes, got %d entries", l) + // Nothing should get downloaded if everything is up to date. + forceExpiration(t, false) + tuf, err = NewFromEnv(ctx) + if err != nil { + t.Fatal(err) } + tuf.Close() - // Force expiration so that content gets downloaded. This should write to disk + if l := dirLen(t, td); cachedDirLen != l { + t.Errorf("expected no filesystem writes, got %d entries", l-cachedDirLen) + } + + // Forcing expiration, but expect no disk writes because all targets up to date. forceExpiration(t, true) tuf, err = NewFromEnv(ctx) if err != nil { t.Fatal(err) } - if l := dirLen(t, td); l == 0 { + if l := dirLen(t, td); l != cachedDirLen { t.Errorf("expected filesystem writes, got %d entries", l) } checkTargetsAndMeta(t, tuf) diff --git a/pkg/cosign/tuf/repository/1.root.json b/pkg/cosign/tuf/repository/1.root.json deleted file mode 100644 index dcc71f963a8..00000000000 --- a/pkg/cosign/tuf/repository/1.root.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "signatures": [ - { - "keyid": "2f64fb5eac0cf94dd39bb45308b98920055e9a0d8e012a7220787834c60aef97", - "sig": "30450221008a35d51da0f845301a5eac98ad0df00a934f59b709c1eaf81c86be734d9356f80220742942325599749800f52675f6efe124345980a2a636c0dc76f9caf9fc3123b0" - }, - { - "keyid": "bdde902f5ec668179ff5ca0dabf7657109287d690bf97e230c21d65f99155c62", - "sig": "3045022100ef9157ece2a09baec1eab80adfc00b04da20b1f9a0d1b47c5dabc4506719ef2c022074f72acd57398e4ddc8c2a5040df902961e9615dca48f3fbe38cbb506e500066" - }, - { - "keyid": "eaf22372f417dd618a46f6c627dbc276e9fd30a004fc94f9be946e73f8bd090b", - "sig": "30450220420fdc9a09cd069b8b15fd8db9cedf7d0dee75871bd1cfee77c926d4120a770002210097553b5ad0d6b4a13902ed37509638bb63a9009f78230cd56c802909ffbfead7" - }, - { - "keyid": "f40f32044071a9365505da3d1e3be6561f6f22d0e60cf51df783999f6c3429cb", - "sig": "304502202aaf32e66f90752f658672b085ecfe45cc1ad31ee6cf5c9ad05f3267685f8d88022100b5df02acdaa371123db9d7a42219553fe079b230b168833e951be7ee56ded347" - }, - { - "keyid": "f505595165a177a41750a8e864ed1719b1edfccd5a426fd2c0ffda33ce7ff209", - "sig": "304402205d420c7d05c58980c1c9f7d221f53b5334aae27a447d2a91c2ceddd685269749022039ec83e51f8e1779d7f0142dfa4a5bbecfe327fc0b91b7416090fea2416fd53a" - } - ], - "signed": { - "_type": "root", - "consistent_snapshot": false, - "expires": "2021-12-18T13:28:12.99008-06:00", - "keys": { - "2f64fb5eac0cf94dd39bb45308b98920055e9a0d8e012a7220787834c60aef97": { - "keyid_hash_algorithms": [ - "sha256", - "sha512" - ], - "keytype": "ecdsa-sha2-nistp256", - "keyval": { - "public": "04cbc5cab2684160323c25cd06c3307178a6b1d1c9b949328453ae473c5ba7527e35b13f298b41633382241f3fd8526c262d43b45adee5c618fa0642c82b8a9803" - }, - "scheme": "ecdsa-sha2-nistp256" - }, - "bdde902f5ec668179ff5ca0dabf7657109287d690bf97e230c21d65f99155c62": { - "keyid_hash_algorithms": [ - "sha256", - "sha512" - ], - "keytype": "ecdsa-sha2-nistp256", - "keyval": { - "public": "04a71aacd835dc170ba6db3fa33a1a33dee751d4f8b0217b805b9bd3242921ee93672fdcfd840576c5bb0dc0ed815edf394c1ee48c2b5e02485e59bfc512f3adc7" - }, - "scheme": "ecdsa-sha2-nistp256" - }, - "eaf22372f417dd618a46f6c627dbc276e9fd30a004fc94f9be946e73f8bd090b": { - "keyid_hash_algorithms": [ - "sha256", - "sha512" - ], - "keytype": "ecdsa-sha2-nistp256", - "keyval": { - "public": "04117b33dd265715bf23315e368faa499728db8d1f0a377070a1c7b1aba2cc21be6ab1628e42f2cdd7a35479f2dce07b303a8ba646c55569a8d2a504ba7e86e447" - }, - "scheme": "ecdsa-sha2-nistp256" - }, - "f40f32044071a9365505da3d1e3be6561f6f22d0e60cf51df783999f6c3429cb": { - "keyid_hash_algorithms": [ - "sha256", - "sha512" - ], - "keytype": "ecdsa-sha2-nistp256", - "keyval": { - "public": "04cc1cd53a61c23e88cc54b488dfae168a257c34fac3e88811c55962b24cffbfecb724447999c54670e365883716302e49da57c79a33cd3e16f81fbc66f0bcdf48" - }, - "scheme": "ecdsa-sha2-nistp256" - }, - "f505595165a177a41750a8e864ed1719b1edfccd5a426fd2c0ffda33ce7ff209": { - "keyid_hash_algorithms": [ - "sha256", - "sha512" - ], - "keytype": "ecdsa-sha2-nistp256", - "keyval": { - "public": "048a78a44ac01099890d787e5e62afc29c8ccb69a70ec6549a6b04033b0a8acbfb42ab1ab9c713d225cdb52b858886cf46c8e90a7f3b9e6371882f370c259e1c5b" - }, - "scheme": "ecdsa-sha2-nistp256" - } - }, - "roles": { - "root": { - "keyids": [ - "2f64fb5eac0cf94dd39bb45308b98920055e9a0d8e012a7220787834c60aef97", - "bdde902f5ec668179ff5ca0dabf7657109287d690bf97e230c21d65f99155c62", - "eaf22372f417dd618a46f6c627dbc276e9fd30a004fc94f9be946e73f8bd090b", - "f40f32044071a9365505da3d1e3be6561f6f22d0e60cf51df783999f6c3429cb", - "f505595165a177a41750a8e864ed1719b1edfccd5a426fd2c0ffda33ce7ff209" - ], - "threshold": 3 - }, - "snapshot": { - "keyids": [ - "2f64fb5eac0cf94dd39bb45308b98920055e9a0d8e012a7220787834c60aef97", - "bdde902f5ec668179ff5ca0dabf7657109287d690bf97e230c21d65f99155c62", - "eaf22372f417dd618a46f6c627dbc276e9fd30a004fc94f9be946e73f8bd090b", - "f40f32044071a9365505da3d1e3be6561f6f22d0e60cf51df783999f6c3429cb", - "f505595165a177a41750a8e864ed1719b1edfccd5a426fd2c0ffda33ce7ff209" - ], - "threshold": 3 - }, - "targets": { - "keyids": [ - "2f64fb5eac0cf94dd39bb45308b98920055e9a0d8e012a7220787834c60aef97", - "bdde902f5ec668179ff5ca0dabf7657109287d690bf97e230c21d65f99155c62", - "eaf22372f417dd618a46f6c627dbc276e9fd30a004fc94f9be946e73f8bd090b", - "f40f32044071a9365505da3d1e3be6561f6f22d0e60cf51df783999f6c3429cb", - "f505595165a177a41750a8e864ed1719b1edfccd5a426fd2c0ffda33ce7ff209" - ], - "threshold": 3 - }, - "timestamp": { - "keyids": [ - "2f64fb5eac0cf94dd39bb45308b98920055e9a0d8e012a7220787834c60aef97", - "bdde902f5ec668179ff5ca0dabf7657109287d690bf97e230c21d65f99155c62", - "eaf22372f417dd618a46f6c627dbc276e9fd30a004fc94f9be946e73f8bd090b", - "f40f32044071a9365505da3d1e3be6561f6f22d0e60cf51df783999f6c3429cb", - "f505595165a177a41750a8e864ed1719b1edfccd5a426fd2c0ffda33ce7ff209" - ], - "threshold": 3 - } - }, - "spec_version": "1.0", - "version": 1 - } -} \ No newline at end of file diff --git a/pkg/cosign/tuf/repository/2.root.json b/pkg/cosign/tuf/repository/2.root.json deleted file mode 100644 index 386ebe62c1e..00000000000 --- a/pkg/cosign/tuf/repository/2.root.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "signatures": [ - { - "keyid": "2f64fb5eac0cf94dd39bb45308b98920055e9a0d8e012a7220787834c60aef97", - "sig": "3046022100d3ea59490b253beae0926c6fa63f54336dea1ed700555be9f27ff55cd347639c0221009157d1ba012cead81948a4ab777d355451d57f5c4a2d333fc68d2e3f358093c2" - }, - { - "keyid": "bdde902f5ec668179ff5ca0dabf7657109287d690bf97e230c21d65f99155c62", - "sig": "304502206eaef40564403ce572c6d062e0c9b0aab5e0223576133e081e1b495e8deb9efd02210080fd6f3464d759601b4afec596bbd5952f3a224cd06ed1cdfc3c399118752ba2" - }, - { - "keyid": "eaf22372f417dd618a46f6c627dbc276e9fd30a004fc94f9be946e73f8bd090b", - "sig": "304502207baace02f56d8e6069f10b6ff098a26e7f53a7f9324ad62cffa0557bdeb9036c022100fb3032baaa090d0040c3f2fd872571c84479309b773208601d65948df87a9720" - }, - { - "keyid": "f40f32044071a9365505da3d1e3be6561f6f22d0e60cf51df783999f6c3429cb", - "sig": "304402205180c01905505dd88acd7a2dad979dd75c979b3722513a7bdedac88c6ae8dbeb022056d1ddf7a192f0b1c2c90ff487de2fb3ec9f0c03f66ea937c78d3b6a493504ca" - }, - { - "keyid": "f505595165a177a41750a8e864ed1719b1edfccd5a426fd2c0ffda33ce7ff209", - "sig": "3046022100c8806d4647c514d80fd8f707d3369444c4fd1d0812a2d25f828e564c99790e3f022100bb51f12e862ef17a7d3da2ac103bebc5c7e792237006c4cafacd76267b249c2f" - } - ], - "signed": { - "_type": "root", - "consistent_snapshot": false, - "expires": "2022-05-11T19:09:02.663975009Z", - "keys": { - "2f64fb5eac0cf94dd39bb45308b98920055e9a0d8e012a7220787834c60aef97": { - "keyid_hash_algorithms": [ - "sha256", - "sha512" - ], - "keytype": "ecdsa-sha2-nistp256", - "keyval": { - "public": "04cbc5cab2684160323c25cd06c3307178a6b1d1c9b949328453ae473c5ba7527e35b13f298b41633382241f3fd8526c262d43b45adee5c618fa0642c82b8a9803" - }, - "scheme": "ecdsa-sha2-nistp256" - }, - "b6710623a30c010738e64c5209d367df1c0a18cf90e6ab5292fb01680f83453d": { - "keyid_hash_algorithms": [ - "sha256", - "sha512" - ], - "keytype": "ecdsa-sha2-nistp256", - "keyval": { - "public": "04fa1a3e42f2300cd3c5487a61509348feb1e936920fef2f83b7cd5dbe7ba045f538725ab8f18a666e6233edb7e0db8766c8dc336633449c5e1bbe0c182b02df0b" - }, - "scheme": "ecdsa-sha2-nistp256" - }, - "bdde902f5ec668179ff5ca0dabf7657109287d690bf97e230c21d65f99155c62": { - "keyid_hash_algorithms": [ - "sha256", - "sha512" - ], - "keytype": "ecdsa-sha2-nistp256", - "keyval": { - "public": "04a71aacd835dc170ba6db3fa33a1a33dee751d4f8b0217b805b9bd3242921ee93672fdcfd840576c5bb0dc0ed815edf394c1ee48c2b5e02485e59bfc512f3adc7" - }, - "scheme": "ecdsa-sha2-nistp256" - }, - "eaf22372f417dd618a46f6c627dbc276e9fd30a004fc94f9be946e73f8bd090b": { - "keyid_hash_algorithms": [ - "sha256", - "sha512" - ], - "keytype": "ecdsa-sha2-nistp256", - "keyval": { - "public": "04117b33dd265715bf23315e368faa499728db8d1f0a377070a1c7b1aba2cc21be6ab1628e42f2cdd7a35479f2dce07b303a8ba646c55569a8d2a504ba7e86e447" - }, - "scheme": "ecdsa-sha2-nistp256" - }, - "f40f32044071a9365505da3d1e3be6561f6f22d0e60cf51df783999f6c3429cb": { - "keyid_hash_algorithms": [ - "sha256", - "sha512" - ], - "keytype": "ecdsa-sha2-nistp256", - "keyval": { - "public": "04cc1cd53a61c23e88cc54b488dfae168a257c34fac3e88811c55962b24cffbfecb724447999c54670e365883716302e49da57c79a33cd3e16f81fbc66f0bcdf48" - }, - "scheme": "ecdsa-sha2-nistp256" - }, - "f505595165a177a41750a8e864ed1719b1edfccd5a426fd2c0ffda33ce7ff209": { - "keyid_hash_algorithms": [ - "sha256", - "sha512" - ], - "keytype": "ecdsa-sha2-nistp256", - "keyval": { - "public": "048a78a44ac01099890d787e5e62afc29c8ccb69a70ec6549a6b04033b0a8acbfb42ab1ab9c713d225cdb52b858886cf46c8e90a7f3b9e6371882f370c259e1c5b" - }, - "scheme": "ecdsa-sha2-nistp256" - }, - "fc61191ba8a516fe386c7d6c97d918e1d241e1589729add09b122725b8c32451": { - "keyid_hash_algorithms": [ - "sha256", - "sha512" - ], - "keytype": "ecdsa-sha2-nistp256", - "keyval": { - "public": "044c7793ab74b9ddd713054e587b8d9c75c5f6025633d0fef7ca855ed5b8d5a474b23598fe33eb4a63630d526f74d4bdaec8adcb51993ed65652d651d7c49203eb" - }, - "scheme": "ecdsa-sha2-nistp256" - } - }, - "roles": { - "root": { - "keyids": [ - "2f64fb5eac0cf94dd39bb45308b98920055e9a0d8e012a7220787834c60aef97", - "bdde902f5ec668179ff5ca0dabf7657109287d690bf97e230c21d65f99155c62", - "eaf22372f417dd618a46f6c627dbc276e9fd30a004fc94f9be946e73f8bd090b", - "f40f32044071a9365505da3d1e3be6561f6f22d0e60cf51df783999f6c3429cb", - "f505595165a177a41750a8e864ed1719b1edfccd5a426fd2c0ffda33ce7ff209" - ], - "threshold": 3 - }, - "snapshot": { - "keyids": [ - "fc61191ba8a516fe386c7d6c97d918e1d241e1589729add09b122725b8c32451" - ], - "threshold": 1 - }, - "targets": { - "keyids": [ - "2f64fb5eac0cf94dd39bb45308b98920055e9a0d8e012a7220787834c60aef97", - "bdde902f5ec668179ff5ca0dabf7657109287d690bf97e230c21d65f99155c62", - "eaf22372f417dd618a46f6c627dbc276e9fd30a004fc94f9be946e73f8bd090b", - "f40f32044071a9365505da3d1e3be6561f6f22d0e60cf51df783999f6c3429cb", - "f505595165a177a41750a8e864ed1719b1edfccd5a426fd2c0ffda33ce7ff209" - ], - "threshold": 3 - }, - "timestamp": { - "keyids": [ - "b6710623a30c010738e64c5209d367df1c0a18cf90e6ab5292fb01680f83453d" - ], - "threshold": 1 - } - }, - "spec_version": "1.0", - "version": 2 - } -} \ No newline at end of file diff --git a/pkg/cosign/tuf/repository/rekor.json b/pkg/cosign/tuf/repository/rekor.json deleted file mode 100644 index f86930d537b..00000000000 --- a/pkg/cosign/tuf/repository/rekor.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "signatures": [ - { - "keyid": "ae0c689c6347ada7359df48934991f4e013193d6ddf3482a5ffb293f74f3b217", - "sig": "3045022076eadd73f6664bac5cc91f12d3a7ddcdd53f9bde661f147651196ff66e7235d1022100f7b3143792405f9e8a75331a05d4128bdf083de302801e99c3d027919a4b03da" - } - ], - "signed": { - "_type": "targets", - "expires": "2022-05-11T19:10:11Z", - "spec_version": "1.0", - "targets": { - "rekor.0.pub": { - "hashes": { - "sha256": "dce5ef715502ec9f3cdfd11f8cc384b31a6141023d3e7595e9908a81cb6241bd", - "sha512": "0ae7705e02db33e814329746a4a0e5603c5bdcd91c96d072158d71011a2695788866565a2fec0fe363eb72cbcaeda39e54c5fe8d416daf9f3101fdba4217ef35" - }, - "length": 178 - } - }, - "version": 1 - } -} \ No newline at end of file diff --git a/pkg/cosign/tuf/repository/snapshot.json b/pkg/cosign/tuf/repository/snapshot.json deleted file mode 100644 index c0e3ca3030d..00000000000 --- a/pkg/cosign/tuf/repository/snapshot.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "signatures": [ - { - "keyid": "fc61191ba8a516fe386c7d6c97d918e1d241e1589729add09b122725b8c32451", - "sig": "3045022100cffd67c2d0339acb9d045f06dfbc81cfedbb49c0b68f95df3d449198d7957a6d022071a28a25789bbe13b8838c88f1f48d401fb15151a0689f51007dbca502408d38" - } - ], - "signed": { - "_type": "snapshot", - "expires": "2022-01-29T00:38:36Z", - "meta": { - "rekor.json": { - "hashes": { - "sha256": "a7412a87f8d7b330e0380b19a4a76c00357c39a1aa7f56fd87445d4e12faafe4", - "sha512": "720cb3c42bac50c5bc3cb7076e730301ef29f1893ea52e25f9393fc05851c7a531638c42d9fc992969805982a2bf51d676e33d28a7382ea589b5a9f87474c63f" - }, - "length": 697, - "version": 1 - }, - "root.json": { - "hashes": { - "sha256": "f5ad897c9414cca99629f400ac3585e41bd8ebb44c5af07fb08dd636a9eced9c", - "sha512": "7445ddfdd338ef786c324fc3d68f75be28cb95b7fb581d2a383e3e5dde18aa17029a5636ec0a22e9631931bbcb34057788311718ea41e21e7cdd3c0de13ede42" - }, - "length": 5297, - "version": 2 - }, - "staging.json": { - "hashes": { - "sha256": "c7f32379c2a76f0ec0af84e86794a8f4fe285e44fb62f336d598810dccdc7343", - "sha512": "5462cb15fe5248a12cc12387a732ad43caf42391361f36113ea3d4b7e5e193cdf39fbe91c309c0691134377cb83afeba50cf6d711537d8280ce16ce9cd8752ba" - }, - "length": 399, - "version": 1 - }, - "targets.json": { - "hashes": { - "sha256": "18d10c07c8d6bd7484772b02dcc988d0abf8a0fa379d5893a502410590c17fe6", - "sha512": "c2ba2a84820288997c8fae264776df7b262dde97c4f9e0320ad354879ce5afabd1d43494734fecffd23253442a14cfe217787de8b65cf7fd1f03130b72a0767c" - }, - "length": 4167, - "version": 2 - } - }, - "spec_version": "1.0", - "version": 10 - } -} \ No newline at end of file diff --git a/pkg/cosign/tuf/repository/staging.json b/pkg/cosign/tuf/repository/staging.json deleted file mode 100644 index 084010de75c..00000000000 --- a/pkg/cosign/tuf/repository/staging.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "signatures": [ - { - "keyid": "b811bd53f2d7adcf5d93e6bb4a8ed2e0ca0f83d454a3e51f105c8e8376bc80d4", - "sig": "304502204486f7b23eadb69df87776ac7a4938ac75a8a2b2e93c84c05d962373837ea91c022100aaeb0fa587430f49618711bb4bd0c1092637c22c223d03c0f1b5a09baea0ed9f" - } - ], - "signed": { - "_type": "targets", - "expires": "2022-02-11T20:10:16Z", - "spec_version": "1.0", - "targets": {}, - "version": 1 - } -} \ No newline at end of file diff --git a/pkg/cosign/tuf/repository/targets.json b/pkg/cosign/tuf/repository/targets.json deleted file mode 100644 index b26926a438f..00000000000 --- a/pkg/cosign/tuf/repository/targets.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "signatures": [ - { - "keyid": "2f64fb5eac0cf94dd39bb45308b98920055e9a0d8e012a7220787834c60aef97", - "sig": "3046022100cc1b2ed390e75a112c0fdd6bcbd8bb775300a410f5737ae39996b1858753c8e4022100b591f73370e9378914fb2fab837f700661abd1a74c680f139f6164ec12cb538f" - }, - { - "keyid": "bdde902f5ec668179ff5ca0dabf7657109287d690bf97e230c21d65f99155c62", - "sig": "3045022100bc6c45a125e45507339af96aa63983e847565c769f20d7d71bcd2deb7bd36ea902202bf6bd3b76d434c318287899e53f64b4dc178eb0ba403080f1c4fba88a2177ca" - }, - { - "keyid": "eaf22372f417dd618a46f6c627dbc276e9fd30a004fc94f9be946e73f8bd090b", - "sig": "304502210085d5bc8a158d31536b4e76cddceef25185c7abbe9091b84f5f2b0d615d9b4ee90220136a36fed2d5986c2519b7d165556f20dfe41fddececda48dffa8dec5258cb95" - }, - { - "keyid": "f40f32044071a9365505da3d1e3be6561f6f22d0e60cf51df783999f6c3429cb", - "sig": "304402202fe73a61dfe05b4202bc50f66e52bba3d3475134434dab9576735caed659b03c0220449755a87f4dab9961566f10477204637b2415f87e162b58a23b13327dec53e3" - }, - { - "keyid": "f505595165a177a41750a8e864ed1719b1edfccd5a426fd2c0ffda33ce7ff209", - "sig": "304602210091f453ef75c5178299175734355a65a2fc2d0ee137410f46ba8439d99037fc08022100fc800d15f0b751fa225a77542928f4264835c013054a5c409c674e2ea5a70384" - } - ], - "signed": { - "_type": "targets", - "delegations": { - "keys": { - "ae0c689c6347ada7359df48934991f4e013193d6ddf3482a5ffb293f74f3b217": { - "keyid_hash_algorithms": [ - "sha256", - "sha512" - ], - "keytype": "ecdsa-sha2-nistp256", - "keyval": { - "public": "043463588ae9df33a419d1099761245af52aaf7e638b2047bc0f739a62de9808c50a21ea8a1a273799f857f31a1bcb66e6661dd9d5ac7ac3ca260b0b8130c3fed8" - }, - "scheme": "ecdsa-sha2-nistp256" - }, - "b811bd53f2d7adcf5d93e6bb4a8ed2e0ca0f83d454a3e51f105c8e8376bc80d4": { - "keyid_hash_algorithms": [ - "sha256", - "sha512" - ], - "keytype": "ecdsa-sha2-nistp256", - "keyval": { - "public": "041b4b13a6e7110292d284c0dbfc3962a12d2a779a800c99aff59c6afe779296943c75d84aa5bad0be28e4061cf93e0cd3d372d9b2f75ea9f29b907cbccd82006f" - }, - "scheme": "ecdsa-sha2-nistp256" - } - }, - "roles": [ - { - "keyids": [ - "ae0c689c6347ada7359df48934991f4e013193d6ddf3482a5ffb293f74f3b217" - ], - "name": "rekor", - "paths": [ - "rekor.*.pub" - ], - "terminating": true, - "threshold": 1 - }, - { - "keyids": [ - "b811bd53f2d7adcf5d93e6bb4a8ed2e0ca0f83d454a3e51f105c8e8376bc80d4" - ], - "name": "staging", - "paths": [ - "*" - ], - "terminating": false, - "threshold": 1 - } - ] - }, - "expires": "2022-05-11T19:10:16Z", - "spec_version": "1.0", - "targets": { - "artifact.pub": { - "hashes": { - "sha256": "59ebf97a9850aecec4bc39c1f5c1dc46e6490a6b5fd2a6cacdcac0c3a6fc4cbf", - "sha512": "308fd1d1d95d7f80aa33b837795251cc3e886792982275e062409e13e4e236ffc34d676682aa96fdc751414de99c864bf132dde71581fa651c6343905e3bf988" - }, - "length": 177 - }, - "ctfe.pub": { - "hashes": { - "sha256": "7fcb94a5d0ed541260473b990b99a6c39864c1fb16f3f3e594a5a3cebbfe138a", - "sha512": "4b20747d1afe2544238ad38cc0cc3010921b177d60ac743767e0ef675b915489bd01a36606c0ff83c06448622d7160f0d866c83d20f0c0f44653dcc3f9aa0bd4" - }, - "length": 177 - }, - "fulcio.crt.pem": { - "hashes": { - "sha256": "f360c53b2e13495a628b9b8096455badcb6d375b185c4816d95a5d746ff29908", - "sha512": "0713252a7fd17f7f3ab12f88a64accf2eb14b8ad40ca711d7fe8b4ecba3b24db9e9dffadb997b196d3867b8f9ff217faf930d80e4dab4e235c7fc3f07be69224" - }, - "length": 744 - }, - "fulcio_v1.crt.pem": { - "hashes": { - "sha256": "f989aa23def87c549404eadba767768d2a3c8d6d30a8b793f9f518a8eafd2cf5", - "sha512": "f2e33a6dc208cee1f51d33bbea675ab0f0ced269617497985f9a0680689ee7073e4b6f8fef64c91bda590d30c129b3070dddce824c05bc165ac9802f0705cab6" - }, - "length": 740 - }, - "rekor.pub": { - "hashes": { - "sha256": "dce5ef715502ec9f3cdfd11f8cc384b31a6141023d3e7595e9908a81cb6241bd", - "sha512": "0ae7705e02db33e814329746a4a0e5603c5bdcd91c96d072158d71011a2695788866565a2fec0fe363eb72cbcaeda39e54c5fe8d416daf9f3101fdba4217ef35" - }, - "length": 178 - } - }, - "version": 2 - } -} \ No newline at end of file diff --git a/pkg/cosign/tuf/repository/timestamp.json b/pkg/cosign/tuf/repository/timestamp.json deleted file mode 100644 index 2030e49d9a6..00000000000 --- a/pkg/cosign/tuf/repository/timestamp.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "signatures": [ - { - "keyid": "b6710623a30c010738e64c5209d367df1c0a18cf90e6ab5292fb01680f83453d", - "sig": "3045022100b4cda580b371e32ce938d03676208b394942f6205eb8ebe8650ec5b366ac5cd7022038ce51f5e3de90849ff728c21179484135b286563f60c906f0db5d999fd676d7" - } - ], - "signed": { - "_type": "timestamp", - "expires": "2022-01-29T00:38:38Z", - "meta": { - "snapshot.json": { - "hashes": { - "sha256": "c4d26ba0e5c0b142c26b9cb11caedac1f29134275190da8f2cb981f2d8a13236", - "sha512": "db3c7a61418fd050a6af04f9d3bb2b359785fd5b4b323516740c76b413df0deae0d81f4bab0abce7abe077584110c00e0f35eeee5155abb7be1b74975794a8fe" - }, - "length": 1657, - "version": 10 - } - }, - "spec_version": "1.0", - "version": 10 - } -} \ No newline at end of file From 8d77efe478c40fed1cb7b74b128512a82d9e3ecb Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Mon, 30 May 2022 11:15:59 +0300 Subject: [PATCH 04/11] Use Rekor API for pubkeys before TUF if so specified. Signed-off-by: Ville Aikas --- pkg/cosign/tlog.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/pkg/cosign/tlog.go b/pkg/cosign/tlog.go index f1110d10294..e1540ab372a 100644 --- a/pkg/cosign/tlog.go +++ b/pkg/cosign/tlog.go @@ -358,11 +358,9 @@ func VerifyTLogEntry(ctx context.Context, rekorClient *client.Rekor, e *models.L LogID: *e.LogID, } - rekorPubKeys, err := GetRekorPubs(ctx) - if err != nil { - return fmt.Errorf("unable to fetch Rekor public keys from TUF repository: %w", err) - } - + // If we've been told to fetch the Public Key from Rekor, fetch it here + // first before using the TUF code below. + rekorPubKeys := make(map[string]RekorPubKey) addRekorPublic := os.Getenv(addRekorPublicKeyFromRekor) if addRekorPublic != "" { pubOK, err := rekorClient.Pubkey.GetPublicKey(nil) @@ -380,6 +378,17 @@ func VerifyTLogEntry(ctx context.Context, rekorClient *client.Rekor, e *models.L rekorPubKeys[keyID] = RekorPubKey{PubKey: pubFromAPI, Status: tuf.Active} } + rekorPubKeysTuf, err := GetRekorPubs(ctx) + if err != nil { + if len(rekorPubKeys) == 0 { + return fmt.Errorf("unable to fetch Rekor public keys from TUF repository, and not trusting the Rekor API for fetching public keys: %w", err) + } + fmt.Fprintf(os.Stderr, "**Warning** Failed to fetch Rekor public keys from TUF but using the public key from Rekor API because %s was specified", addRekorPublicKeyFromRekor) + } + + for k, v := range rekorPubKeysTuf { + rekorPubKeys[k] = v + } pubKey, ok := rekorPubKeys[payload.LogID] if !ok { return errors.New("rekor log public key not found for payload") From 153cfdeb1781a2a9b63e74416ca21dd6acb20883 Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Mon, 30 May 2022 12:12:39 +0300 Subject: [PATCH 05/11] Address PR feedback, bump golangci-lint from 1.46.0 to 1.46.2 Signed-off-by: Ville Aikas --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a47074f75ca..439257d959d 100644 --- a/Makefile +++ b/Makefile @@ -118,7 +118,7 @@ cross: golangci-lint: rm -f $(GOLANGCI_LINT_BIN) || : set -e ;\ - GOBIN=$(GOLANGCI_LINT_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.46.0 ;\ + GOBIN=$(GOLANGCI_LINT_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.46.2 ;\ lint: golangci-lint ## Run golangci-lint linter $(GOLANGCI_LINT_BIN) run -n From 90798bd4ad9e0a634f14099cc393a7cac08eb8c4 Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Mon, 30 May 2022 13:16:53 +0300 Subject: [PATCH 06/11] Add comments for the env variables. Signed-off-by: Ville Aikas --- pkg/cosign/tlog.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/cosign/tlog.go b/pkg/cosign/tlog.go index e1540ab372a..197e881b5e9 100644 --- a/pkg/cosign/tlog.go +++ b/pkg/cosign/tlog.go @@ -77,9 +77,13 @@ func getLogID(pub crypto.PublicKey) (string, error) { // GetRekorPubs retrieves trusted Rekor public keys from the embedded or cached // TUF root. If expired, makes a network call to retrieve the updated targets. +// There is an Env variable that can be used to override this behaviour: +// SIGSTORE_REKOR_PUBLIC_KEY - If specified, location of the file that contains +// the Rekor Public Key on local filesystem func GetRekorPubs(ctx context.Context) (map[string]RekorPubKey, error) { publicKeys := make(map[string]RekorPubKey) altRekorPub := os.Getenv(altRekorPublicKey) + if altRekorPub != "" { fmt.Fprintf(os.Stderr, "**Warning** Using a non-standard public key for Rekor: %s\n", altRekorPub) raw, err := os.ReadFile(altRekorPub) @@ -326,6 +330,9 @@ func FindTLogEntriesByPayload(ctx context.Context, rekorClient *client.Rekor, pa return searchIndex.GetPayload(), nil } +// VerityTLogEntry verifies a TLog entry. If the Env variable +// SIGSTORE_TRUST_REKOR_API_PUBLIC_KEY is specified, fetches the Rekor public +// key from the Rekor server using the provided rekorClient. func VerifyTLogEntry(ctx context.Context, rekorClient *client.Rekor, e *models.LogEntryAnon) error { if e.Verification == nil || e.Verification.InclusionProof == nil { return errors.New("inclusion proof not provided") @@ -383,7 +390,7 @@ func VerifyTLogEntry(ctx context.Context, rekorClient *client.Rekor, e *models.L if len(rekorPubKeys) == 0 { return fmt.Errorf("unable to fetch Rekor public keys from TUF repository, and not trusting the Rekor API for fetching public keys: %w", err) } - fmt.Fprintf(os.Stderr, "**Warning** Failed to fetch Rekor public keys from TUF but using the public key from Rekor API because %s was specified", addRekorPublicKeyFromRekor) + fmt.Fprintf(os.Stderr, "**Warning** Failed to fetch Rekor public keys from TUF, using the public key from Rekor API because %s was specified", addRekorPublicKeyFromRekor) } for k, v := range rekorPubKeysTuf { From 4dd5b896b57a37781462fb2e9a74a6dfc0350b2e Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Tue, 31 May 2022 11:41:54 +0300 Subject: [PATCH 07/11] Use path instead of filepath, basically revert to what it was before. Signed-off-by: Ville Aikas --- pkg/cosign/tuf/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cosign/tuf/client.go b/pkg/cosign/tuf/client.go index 66d16df6bdf..a4cab529149 100644 --- a/pkg/cosign/tuf/client.go +++ b/pkg/cosign/tuf/client.go @@ -200,7 +200,7 @@ func getRoot(meta map[string]json.RawMessage, fallback fs.FS) (json.RawMessage, if !ok { return nil, errors.New("fs.ReadFileFS unimplemented for embedded repo") } - trustedRoot, err := rd.ReadFile(filepath.FromSlash(filepath.Join("repository", "root.json"))) + trustedRoot, err := rd.ReadFile(filepath.Join("repository", "root.json")) if err != nil { return nil, err } From 530c29e1538c746513840dc035253b44315a8762 Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Tue, 31 May 2022 12:08:31 +0300 Subject: [PATCH 08/11] ho hum, really just use the path. Signed-off-by: Ville Aikas --- pkg/cosign/tuf/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cosign/tuf/client.go b/pkg/cosign/tuf/client.go index a4cab529149..7f302f0841b 100644 --- a/pkg/cosign/tuf/client.go +++ b/pkg/cosign/tuf/client.go @@ -200,7 +200,7 @@ func getRoot(meta map[string]json.RawMessage, fallback fs.FS) (json.RawMessage, if !ok { return nil, errors.New("fs.ReadFileFS unimplemented for embedded repo") } - trustedRoot, err := rd.ReadFile(filepath.Join("repository", "root.json")) + trustedRoot, err := rd.ReadFile(path.Join("repository", "root.json")) if err != nil { return nil, err } From b8a50755902ece7cd715b9f54ce98893aa31e7bf Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Tue, 31 May 2022 13:12:30 +0300 Subject: [PATCH 09/11] When interacting with fs do not use OS specific separators. Signed-off-by: Ville Aikas --- pkg/cosign/tuf/client.go | 3 +-- pkg/cosign/tuf/client_test.go | 13 +++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/cosign/tuf/client.go b/pkg/cosign/tuf/client.go index 7f302f0841b..631fe1309af 100644 --- a/pkg/cosign/tuf/client.go +++ b/pkg/cosign/tuf/client.go @@ -191,8 +191,7 @@ func (t *TUF) getRootStatus() (*RootStatus, error) { } func getRoot(meta map[string]json.RawMessage, fallback fs.FS) (json.RawMessage, error) { - trustedRoot, ok := meta["root.json"] - if ok { + if trustedRoot, ok := meta["root.json"]; ok { return trustedRoot, nil } // On first initialize, there will be no root in the TUF DB, so read from embedded. diff --git a/pkg/cosign/tuf/client_test.go b/pkg/cosign/tuf/client_test.go index fb89a7f1404..e7ebb68faa8 100644 --- a/pkg/cosign/tuf/client_test.go +++ b/pkg/cosign/tuf/client_test.go @@ -24,6 +24,7 @@ import ( "net/http" "net/http/httptest" "os" + "path" "path/filepath" "reflect" "sort" @@ -334,16 +335,16 @@ func TestGetTargetsByMeta(t *testing.T) { func makeMapFS(repo string) (fs fstest.MapFS) { fs = make(fstest.MapFS) _ = filepath.Walk(repo, - func(path string, info os.FileInfo, err error) error { + func(fpath string, info os.FileInfo, err error) error { if err != nil { return err } - rel, _ := filepath.Rel(repo, path) + rel, _ := filepath.Rel(repo, fpath) if info.IsDir() { - fs[filepath.FromSlash(filepath.Join("repository", rel))] = &fstest.MapFile{Mode: os.ModeDir} + fs[path.Join("repository", rel)] = &fstest.MapFile{Mode: os.ModeDir} } else { - b, _ := os.ReadFile(path) - fs[filepath.FromSlash(filepath.Join("repository", rel))] = &fstest.MapFile{Data: b} + b, _ := os.ReadFile(fpath) + fs[path.Join("repository", rel)] = &fstest.MapFile{Data: b} } return nil }) @@ -386,7 +387,7 @@ func TestUpdatedTargetNamesEmbedded(t *testing.T) { if !ok { t.Fatal("fs.ReadFileFS unimplemented for embedded repo") } - if _, err := rd.ReadFile(filepath.FromSlash(filepath.Join("repository", "targets", newTarget))); err == nil { + if _, err := rd.ReadFile(path.Join("repository", "targets", newTarget)); err == nil { t.Fatal("embedded repository should not contain new target") } From 410d6cf9cef895df5fc7897abb0df8e48c00c803 Mon Sep 17 00:00:00 2001 From: Asra Ali Date: Tue, 31 May 2022 10:00:40 -0500 Subject: [PATCH 10/11] fix windows line endings Signed-off-by: Asra Ali --- pkg/cosign/tuf/client.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/cosign/tuf/client.go b/pkg/cosign/tuf/client.go index 631fe1309af..37b1fb62bcb 100644 --- a/pkg/cosign/tuf/client.go +++ b/pkg/cosign/tuf/client.go @@ -29,6 +29,7 @@ import ( "os" "path" "path/filepath" + "runtime" "strconv" "strings" "time" @@ -474,7 +475,15 @@ func maybeDownloadRemoteTarget(name string, meta data.TargetFileMeta, t *TUF) er return errors.New("fs.ReadFileFS unimplemented for embedded repo") } b, err := rd.ReadFile(path.Join("repository", "targets", name)) + if err == nil { + // Unfortunately go:embed appears to somehow replace our line endings on windows, we need to switch them back. + // It should theoretically be safe to do this everywhere - but the files only seem to get mutated on Windows so + // let's only change them back there. + if runtime.GOOS == "windows" { + b = bytes.ReplaceAll(b, []byte("\r\n"), []byte("\n")) + } + if isValidTarget(b, meta) { if _, err := io.Copy(&w, bytes.NewReader(b)); err != nil { return fmt.Errorf("using embedded target: %w", err) From e7bcb6961178788ceb5a2e63e33388a188b574c9 Mon Sep 17 00:00:00 2001 From: Asra Ali Date: Tue, 31 May 2022 10:04:43 -0500 Subject: [PATCH 11/11] pass embedded into initialization Signed-off-by: Asra Ali --- pkg/cosign/tuf/client.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/cosign/tuf/client.go b/pkg/cosign/tuf/client.go index 37b1fb62bcb..c5df317589f 100644 --- a/pkg/cosign/tuf/client.go +++ b/pkg/cosign/tuf/client.go @@ -228,12 +228,14 @@ func (t *TUF) Close() error { // * mirror: provides a reference to a remote GCS or HTTP mirror. // * root: provides an external initial root.json. When this is not provided, this // defaults to the embedded root.json. +// * embedded: An embedded filesystem that provides a trusted root and pre-downloaded +// targets in a targets/ subfolder. // * forceUpdate: indicates checking the remote for an update, even when the local // timestamp.json is up to date. -func initializeTUF(ctx context.Context, mirror string, root []byte, forceUpdate bool) (*TUF, error) { +func initializeTUF(ctx context.Context, mirror string, root []byte, embedded fs.FS, forceUpdate bool) (*TUF, error) { t := &TUF{ mirror: mirror, - embedded: GetEmbedded(), + embedded: embedded, } t.targets = newFileImpl() @@ -299,12 +301,12 @@ func NewFromEnv(ctx context.Context) (*TUF, error) { } // Initializes a new TUF object from the local cache or defaults. - return initializeTUF(ctx, mirror, nil, false) + return initializeTUF(ctx, mirror, nil, GetEmbedded(), false) } func Initialize(ctx context.Context, mirror string, root []byte) error { // Initialize the client. Force an update. - t, err := initializeTUF(ctx, mirror, root, true) + t, err := initializeTUF(ctx, mirror, root, GetEmbedded(), true) if err != nil { return err }