From 1373f97acd518c444e7504f73ecf8391745cb241 Mon Sep 17 00:00:00 2001 From: Hayden B Date: Thu, 3 Nov 2022 11:14:48 -0700 Subject: [PATCH 1/3] Add script to backfill Redis from Rekor (#1163) * Add script to backfill Redis from Rekor Signed-off-by: Hayden Blauzvern * Address comments Signed-off-by: Hayden Blauzvern * Fix bugs in script Signed-off-by: Hayden Blauzvern * Clean up, change flag name Signed-off-by: Hayden Blauzvern * Add value removal before upload, print all errors Signed-off-by: Hayden Blauzvern * Remove intoto 0.0.2 from backfill Signed-off-by: Hayden Blauzvern Signed-off-by: Hayden Blauzvern --- cmd/backfill-redis/main.go | 169 +++++++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 cmd/backfill-redis/main.go diff --git a/cmd/backfill-redis/main.go b/cmd/backfill-redis/main.go new file mode 100644 index 000000000..e87f82dfa --- /dev/null +++ b/cmd/backfill-redis/main.go @@ -0,0 +1,169 @@ +// Copyright 2022 The Sigstore Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* + backfill-redis is a script to populate the Redis index with entries + from Rekor. This is sometimes necessary because Redis caching is best + effort. If Redis returns an error, Rekor will not, and so sometimes + we need to backfill missing entries into Redis for the search API. + + To run: + go run cmd/backfill-redis/main.go --rekor-address
\ + --hostname --port + --start --end +*/ + +package main + +import ( + "bytes" + "context" + "encoding/base64" + "flag" + "fmt" + "log" + + "github.com/go-openapi/runtime" + radix "github.com/mediocregopher/radix/v4" + "github.com/sigstore/rekor/pkg/client" + "github.com/sigstore/rekor/pkg/generated/client/entries" + "github.com/sigstore/rekor/pkg/generated/models" + "github.com/sigstore/rekor/pkg/types" + + // these imports are to call the packages' init methods + _ "github.com/sigstore/rekor/pkg/types/alpine/v0.0.1" + _ "github.com/sigstore/rekor/pkg/types/cose/v0.0.1" + _ "github.com/sigstore/rekor/pkg/types/hashedrekord/v0.0.1" + _ "github.com/sigstore/rekor/pkg/types/helm/v0.0.1" + _ "github.com/sigstore/rekor/pkg/types/intoto/v0.0.1" + + // remove 0.0.2 intoto type due to bugs + // _ "github.com/sigstore/rekor/pkg/types/intoto/v0.0.2" + _ "github.com/sigstore/rekor/pkg/types/jar/v0.0.1" + _ "github.com/sigstore/rekor/pkg/types/rekord/v0.0.1" + _ "github.com/sigstore/rekor/pkg/types/rfc3161/v0.0.1" + _ "github.com/sigstore/rekor/pkg/types/rpm/v0.0.1" + _ "github.com/sigstore/rekor/pkg/types/tuf/v0.0.1" +) + +var ( + redisHostname = flag.String("hostname", "", "Hostname for Redis application") + redisPort = flag.String("port", "", "Port to Redis application") + startIndex = flag.Int("start", -1, "First index to backfill") + endIndex = flag.Int("end", -1, "Last index to backfill") + rekorAddress = flag.String("rekor-address", "", "Address for Rekor, e.g. https://rekor.sigstore.dev") +) + +func main() { + flag.Parse() + + if *redisHostname == "" { + log.Fatal("address must be set") + } + if *redisPort == "" { + log.Fatal("port must be set") + } + if *startIndex == -1 { + log.Fatal("start must be set to >=0") + } + if *endIndex == -1 { + log.Fatal("end must be set to >=0") + } + if *rekorAddress == "" { + log.Fatal("rekor-address must be set") + } + + cfg := radix.PoolConfig{} + redisClient, err := cfg.New(context.Background(), "tcp", fmt.Sprintf("%s:%s", *redisHostname, *redisPort)) + if err != nil { + log.Fatal(err) + } + + rekorClient, err := client.GetRekorClient(*rekorAddress) + if err != nil { + log.Fatalf("creating rekor client: %v", err) + } + + for i := *startIndex; i <= *endIndex; i++ { + params := entries.NewGetLogEntryByIndexParamsWithContext(context.Background()) + params.SetLogIndex(int64(i)) + resp, err := rekorClient.Entries.GetLogEntryByIndex(params) + if err != nil { + log.Fatalf("retrieving log uuid by index: %v", err) + } + var insertErrs []error + for uuid, entry := range resp.Payload { + // uuid is the global UUID - tree ID and entry UUID + e, _, _, err := unmarshalEntryImpl(entry.Body.(string)) + if err != nil { + insertErrs = append(insertErrs, fmt.Errorf("error unmarshalling entry for %s: %v", uuid, err)) + continue + } + keys, err := e.IndexKeys() + if err != nil { + insertErrs = append(insertErrs, fmt.Errorf("error building index keys for %s: %v", uuid, err)) + continue + } + for _, key := range keys { + // remove the key-value pair from the index in case it already exists + if err := removeFromIndex(context.Background(), redisClient, key, uuid); err != nil { + insertErrs = append(insertErrs, fmt.Errorf("error removing UUID %s with key %s: %v", uuid, key, err)) + } + if err := addToIndex(context.Background(), redisClient, key, uuid); err != nil { + insertErrs = append(insertErrs, fmt.Errorf("error inserting UUID %s with key %s: %v", uuid, key, err)) + } + fmt.Printf("Uploaded Redis entry %s, index %d, key %s\n", uuid, i, key) + } + } + if len(insertErrs) != 0 { + fmt.Printf("Errors with log index %d:\n", i) + for _, e := range insertErrs { + fmt.Println(e) + } + } else { + fmt.Printf("Completed log index %d\n", i) + } + } +} + +// unmarshalEntryImpl decodes the base64-encoded entry to a specific entry type (types.EntryImpl). +// Taken from Cosign +func unmarshalEntryImpl(e string) (types.EntryImpl, string, string, error) { + b, err := base64.StdEncoding.DecodeString(e) + if err != nil { + return nil, "", "", err + } + + pe, err := models.UnmarshalProposedEntry(bytes.NewReader(b), runtime.JSONConsumer()) + if err != nil { + return nil, "", "", err + } + + entry, err := types.UnmarshalEntry(pe) + if err != nil { + return nil, "", "", err + } + return entry, pe.Kind(), entry.APIVersion(), nil +} + +// removeFromIndex removes all occurrences of a value from a given key. This guards against +// multiple invocations of backfilling creating duplicates. +func removeFromIndex(ctx context.Context, redisClient radix.Client, key, value string) error { + return redisClient.Do(ctx, radix.Cmd(nil, "LREM", key, "0", value)) +} + +// addToIndex pushes a value onto a key of type list. +func addToIndex(ctx context.Context, redisClient radix.Client, key, value string) error { + return redisClient.Do(ctx, radix.Cmd(nil, "LPUSH", key, value)) +} From c02cf87029e32c55e7c6125aa85451cc3233be1e Mon Sep 17 00:00:00 2001 From: cpanato Date: Fri, 4 Nov 2022 10:53:43 +0100 Subject: [PATCH 2/3] add backfill-redis to the release Signed-off-by: cpanato --- .ko.yaml | 15 +++++++++++++++ Makefile | 14 ++++++++++++++ release/ko-sign-release-images.sh | 7 +++++++ release/release.mk | 10 +++++++--- 4 files changed, 43 insertions(+), 3 deletions(-) diff --git a/.ko.yaml b/.ko.yaml index ecf440910..18c35c779 100644 --- a/.ko.yaml +++ b/.ko.yaml @@ -46,3 +46,18 @@ builds: ldflags: - -extldflags "-static" - "{{ .Env.LDFLAGS }}" + +- id: backfill-redis + dir: . + main: ./cmd/backfill-redis + env: + - CGO_ENABLED=0 + flags: + - -trimpath + - --tags + - "{{ .Env.GIT_HASH }}" + - --tags + - "{{ .Env.GIT_VERSION }}" + ldflags: + - -extldflags "-static" + - "{{ .Env.LDFLAGS }}" diff --git a/Makefile b/Makefile index ca25f716d..6f6306f5d 100644 --- a/Makefile +++ b/Makefile @@ -81,6 +81,9 @@ rekor-cli: $(SRCS) rekor-server: $(SRCS) CGO_ENABLED=0 go build -trimpath -ldflags "$(SERVER_LDFLAGS)" -o rekor-server ./cmd/rekor-server +backfill-redis: $(SRCS) + CGO_ENABLED=0 go build -trimpath -ldflags "$(SERVER_LDFLAGS)" -o rekor-server ./cmd/backfill-redis + test: go test ./... @@ -122,6 +125,12 @@ ko: --platform=all --tags $(GIT_VERSION) --tags $(GIT_HASH) \ --image-refs rekorCliImagerefs github.com/sigstore/rekor/cmd/rekor-cli + # backfill-redis + LDFLAGS="$(SERVER_LDFLAGS)" GIT_HASH=$(GIT_HASH) GIT_VERSION=$(GIT_VERSION) \ + ko publish --base-import-paths \ + --platform=all --tags $(GIT_VERSION) --tags $(GIT_HASH) \ + --image-refs bRedisImagerefs github.com/sigstore/rekor/cmd/backfill-redis + deploy: LDFLAGS="$(SERVER_LDFLAGS)" GIT_HASH=$(GIT_HASH) GIT_VERSION=$(GIT_VERSION) ko apply -f config/ @@ -146,6 +155,11 @@ ko-local: --tags $(GIT_VERSION) --tags $(GIT_HASH) --local \ github.com/sigstore/rekor/cmd/rekor-cli + LDFLAGS="$(SERVER_LDFLAGS)" GIT_HASH=$(GIT_HASH) GIT_VERSION=$(GIT_VERSION) \ + ko publish --base-import-paths \ + --tags $(GIT_VERSION) --tags $(GIT_HASH) --local \ + github.com/sigstore/rekor/cmd/backfill-redis + # This builds the trillian containers we rely on using ko for cross platform support .PHONY: ko-trillian ko-trillian: diff --git a/release/ko-sign-release-images.sh b/release/ko-sign-release-images.sh index ca6b6f177..1688d0e77 100755 --- a/release/ko-sign-release-images.sh +++ b/release/ko-sign-release-images.sh @@ -36,6 +36,11 @@ if [[ ! -f rekorCliImagerefs ]]; then exit 1 fi +if [[ ! -f bRedisImagerefs ]]; then + echo "bRedisImagerefs not found" + exit 1 +fi + if [[ ! -f trillianServerImagerefs ]]; then echo "trillianServerImagerefs not found" exit 1 @@ -49,11 +54,13 @@ fi echo "Signing images with GCP KMS Key..." cosign sign --force --key "gcpkms://projects/$PROJECT_ID/locations/$KEY_LOCATION/keyRings/$KEY_RING/cryptoKeys/$KEY_NAME/versions/$KEY_VERSION" -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat rekorServerImagerefs) cosign sign --force --key "gcpkms://projects/$PROJECT_ID/locations/$KEY_LOCATION/keyRings/$KEY_RING/cryptoKeys/$KEY_NAME/versions/$KEY_VERSION" -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat rekorCliImagerefs) +cosign sign --force --key "gcpkms://projects/$PROJECT_ID/locations/$KEY_LOCATION/keyRings/$KEY_RING/cryptoKeys/$KEY_NAME/versions/$KEY_VERSION" -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat bRedisImagerefs) cosign sign --force --key "gcpkms://projects/$PROJECT_ID/locations/$KEY_LOCATION/keyRings/$KEY_RING/cryptoKeys/$KEY_NAME/versions/$KEY_VERSION" -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat trillianServerImagerefs) cosign sign --force --key "gcpkms://projects/$PROJECT_ID/locations/$KEY_LOCATION/keyRings/$KEY_RING/cryptoKeys/$KEY_NAME/versions/$KEY_VERSION" -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat trillianSignerImagerefs) echo "Signing images with Keyless..." cosign sign --force -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat rekorServerImagerefs) cosign sign --force -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat rekorCliImagerefs) +cosign sign --force -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat bRedisImagerefs) cosign sign --force -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat trillianServerImagerefs) cosign sign --force -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat trillianSignerImagerefs) diff --git a/release/release.mk b/release/release.mk index 19dc18209..19a1c6d63 100644 --- a/release/release.mk +++ b/release/release.mk @@ -5,12 +5,12 @@ # used when releasing together with GCP CloudBuild .PHONY: release release: - CLI_LDFLAGS="$(CLI_LDFLAGS)" SERVER_LDFLAGS="$(SERVER_LDFLAGS)" goreleaser release --rm-dist --timeout 60m + CLI_LDFLAGS="$(CLI_LDFLAGS)" SERVER_LDFLAGS="$(SERVER_LDFLAGS)" goreleaser release --rm-dist --timeout 120m # used when need to validate the goreleaser .PHONY: snapshot snapshot: - CLI_LDFLAGS="$(CLI_LDFLAGS)" SERVER_LDFLAGS="$(SERVER_LDFLAGS)" goreleaser release --skip-sign --skip-publish --snapshot --rm-dist + CLI_LDFLAGS="$(CLI_LDFLAGS)" SERVER_LDFLAGS="$(SERVER_LDFLAGS)" goreleaser release --skip-sign --skip-publish --snapshot --rm-dist --timeout 120m ########################### # sign section @@ -33,6 +33,10 @@ copy-rekor-server-signed-release-to-ghcr: copy-rekor-cli-signed-release-to-ghcr: cosign copy $(KO_PREFIX)/rekor-cli:$(GIT_VERSION) $(GHCR_PREFIX)/rekor-cli:$(GIT_VERSION) +.PHONY: copy-backfill-redis-signed-release-to-ghcr +copy-backfill-redis-signed-release-to-ghcr: + cosign copy $(KO_PREFIX)/backfill-redis:$(GIT_VERSION) $(GHCR_PREFIX)/backfill-redis:$(GIT_VERSION) + .PHONY: copy-trillian-log-server-signed-release-to-ghcr copy-trillian-log-server-signed-release-to-ghcr: cosign copy $(KO_PREFIX)/trillian_log_server:$(GIT_VERSION) $(GHCR_PREFIX)/trillian_log_server:$(GIT_VERSION) @@ -42,7 +46,7 @@ copy-trillian-log-signer-signed-release-to-ghcr: cosign copy $(KO_PREFIX)/trillian_log_signer:$(GIT_VERSION) $(GHCR_PREFIX)/trillian_log_signer:$(GIT_VERSION) .PHONY: copy-signed-release-to-ghcr -copy-signed-release-to-ghcr: copy-rekor-server-signed-release-to-ghcr copy-rekor-cli-signed-release-to-ghcr copy-trillian-log-signer-signed-release-to-ghcr copy-trillian-log-server-signed-release-to-ghcr +copy-signed-release-to-ghcr: copy-rekor-server-signed-release-to-ghcr copy-rekor-cli-signed-release-to-ghcr copy-backfill-redis-signed-release-to-ghcr copy-trillian-log-signer-signed-release-to-ghcr copy-trillian-log-server-signed-release-to-ghcr ## -------------------------------------- ## Dist / maybe we can deprecate From 4d6f6f2d36434a5f5cba4d896dc0e468021823e8 Mon Sep 17 00:00:00 2001 From: cpanato Date: Fri, 4 Nov 2022 10:54:10 +0100 Subject: [PATCH 3/3] add version information for backfill-redis Signed-off-by: cpanato --- .github/workflows/main.yml | 6 ++++-- .gitignore | 3 +++ Makefile | 6 +++--- cmd/backfill-redis/main.go | 12 ++++++++++++ 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 19f8a01c6..4f954b781 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -67,8 +67,10 @@ jobs: - name: container run: | - make ko-local 2>&1 | tee output.txt - docker run --rm $(tail -1 output.txt) version + make ko-local + docker run --rm $(cat rekorImagerefs) version + docker run --rm $(cat cliImagerefs) version + docker run --rm $(cat redisImagerefs) --version e2e: runs-on: ubuntu-20.04 diff --git a/.gitignore b/.gitignore index 55fce30e7..dc1389003 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,9 @@ rekorServerImagerefs rekorCliImagerefs trillianServerImagerefs trillianSignerImagerefs +rekorImagerefs +cliImagerefs +redisImagerefs cosign.* signature rekor.pub diff --git a/Makefile b/Makefile index 6f6306f5d..4a4e9ced9 100644 --- a/Makefile +++ b/Makefile @@ -147,17 +147,17 @@ sign-keyless-ci: ko ko-local: LDFLAGS="$(SERVER_LDFLAGS)" GIT_HASH=$(GIT_HASH) GIT_VERSION=$(GIT_VERSION) \ ko publish --base-import-paths \ - --tags $(GIT_VERSION) --tags $(GIT_HASH) --local \ + --tags $(GIT_VERSION) --tags $(GIT_HASH) --local --image-refs rekorImagerefs \ github.com/sigstore/rekor/cmd/rekor-server LDFLAGS="$(CLI_LDFLAGS)" GIT_HASH=$(GIT_HASH) GIT_VERSION=$(GIT_VERSION) \ ko publish --base-import-paths \ - --tags $(GIT_VERSION) --tags $(GIT_HASH) --local \ + --tags $(GIT_VERSION) --tags $(GIT_HASH) --local --image-refs cliImagerefs \ github.com/sigstore/rekor/cmd/rekor-cli LDFLAGS="$(SERVER_LDFLAGS)" GIT_HASH=$(GIT_HASH) GIT_VERSION=$(GIT_VERSION) \ ko publish --base-import-paths \ - --tags $(GIT_VERSION) --tags $(GIT_HASH) --local \ + --tags $(GIT_VERSION) --tags $(GIT_HASH) --local --image-refs redisImagerefs \ github.com/sigstore/rekor/cmd/backfill-redis # This builds the trillian containers we rely on using ko for cross platform support diff --git a/cmd/backfill-redis/main.go b/cmd/backfill-redis/main.go index e87f82dfa..58f3c1797 100644 --- a/cmd/backfill-redis/main.go +++ b/cmd/backfill-redis/main.go @@ -33,9 +33,12 @@ import ( "flag" "fmt" "log" + "os" "github.com/go-openapi/runtime" radix "github.com/mediocregopher/radix/v4" + "sigs.k8s.io/release-utils/version" + "github.com/sigstore/rekor/pkg/client" "github.com/sigstore/rekor/pkg/generated/client/entries" "github.com/sigstore/rekor/pkg/generated/models" @@ -63,11 +66,18 @@ var ( startIndex = flag.Int("start", -1, "First index to backfill") endIndex = flag.Int("end", -1, "Last index to backfill") rekorAddress = flag.String("rekor-address", "", "Address for Rekor, e.g. https://rekor.sigstore.dev") + versionFlag = flag.Bool("version", false, "Print the current version of Backfill Redis") ) func main() { flag.Parse() + versionInfo := version.GetVersionInfo() + if *versionFlag { + fmt.Println(versionInfo.String()) + os.Exit(0) + } + if *redisHostname == "" { log.Fatal("address must be set") } @@ -84,6 +94,8 @@ func main() { log.Fatal("rekor-address must be set") } + log.Printf("running backfill redis Version: %s GitCommit: %s BuildDate: %s", versionInfo.GitVersion, versionInfo.GitCommit, versionInfo.BuildDate) + cfg := radix.PoolConfig{} redisClient, err := cfg.New(context.Background(), "tcp", fmt.Sprintf("%s:%s", *redisHostname, *redisPort)) if err != nil {