Cherry pick #1163 and #1174 into release-1.0

.github/workflows/main.yml

@@ -67,8 +67,10 @@ jobs:
 
       - name: container
         run: |
-          make ko-local 2>&1 | tee output.txt
-          docker run --rm $(tail -1 output.txt) version
+          make ko-local
+          docker run --rm $(cat rekorImagerefs) version
+          docker run --rm $(cat cliImagerefs) version
+          docker run --rm $(cat redisImagerefs) --version
 
   e2e:
     runs-on: ubuntu-20.04

.gitignore

@@ -17,6 +17,9 @@ rekorServerImagerefs
 rekorCliImagerefs
 trillianServerImagerefs
 trillianSignerImagerefs
+rekorImagerefs
+cliImagerefs
+redisImagerefs
 cosign.*
 signature
 rekor.pub

.ko.yaml

@@ -46,3 +46,18 @@ builds:
   ldflags:
   - -extldflags "-static"
   - "{{ .Env.LDFLAGS }}"
+
+- id: backfill-redis
+  dir: .
+  main: ./cmd/backfill-redis
+  env:
+  - CGO_ENABLED=0
+  flags:
+  - -trimpath
+  - --tags
+  - "{{ .Env.GIT_HASH }}"
+  - --tags
+  - "{{ .Env.GIT_VERSION }}"
+  ldflags:
+  - -extldflags "-static"
+  - "{{ .Env.LDFLAGS }}"

Makefile

@@ -81,6 +81,9 @@ rekor-cli: $(SRCS)
 rekor-server: $(SRCS)
 	CGO_ENABLED=0 go build -trimpath -ldflags "$(SERVER_LDFLAGS)" -o rekor-server ./cmd/rekor-server
 
+backfill-redis: $(SRCS)
+	CGO_ENABLED=0 go build -trimpath -ldflags "$(SERVER_LDFLAGS)" -o rekor-server ./cmd/backfill-redis
+
 test:
 	go test ./...
 
@@ -122,6 +125,12 @@ ko:
 		--platform=all --tags $(GIT_VERSION) --tags $(GIT_HASH) \
 		--image-refs rekorCliImagerefs github.com/sigstore/rekor/cmd/rekor-cli
 
+	# backfill-redis
+	LDFLAGS="$(SERVER_LDFLAGS)" GIT_HASH=$(GIT_HASH) GIT_VERSION=$(GIT_VERSION) \
+	ko publish --base-import-paths \
+		--platform=all --tags $(GIT_VERSION) --tags $(GIT_HASH) \
+		--image-refs bRedisImagerefs github.com/sigstore/rekor/cmd/backfill-redis
+
 deploy:
 	LDFLAGS="$(SERVER_LDFLAGS)" GIT_HASH=$(GIT_HASH) GIT_VERSION=$(GIT_VERSION) ko apply -f config/
 
@@ -138,14 +147,19 @@ sign-keyless-ci: ko
 ko-local:
 	LDFLAGS="$(SERVER_LDFLAGS)" GIT_HASH=$(GIT_HASH) GIT_VERSION=$(GIT_VERSION) \
 	ko publish --base-import-paths \
-		--tags $(GIT_VERSION) --tags $(GIT_HASH) --local \
+		--tags $(GIT_VERSION) --tags $(GIT_HASH) --local --image-refs rekorImagerefs \
 		github.com/sigstore/rekor/cmd/rekor-server
 
 	LDFLAGS="$(CLI_LDFLAGS)" GIT_HASH=$(GIT_HASH) GIT_VERSION=$(GIT_VERSION) \
 	ko publish --base-import-paths \
-		--tags $(GIT_VERSION) --tags $(GIT_HASH) --local \
+		--tags $(GIT_VERSION) --tags $(GIT_HASH) --local --image-refs cliImagerefs \
 		github.com/sigstore/rekor/cmd/rekor-cli
 
+	LDFLAGS="$(SERVER_LDFLAGS)" GIT_HASH=$(GIT_HASH) GIT_VERSION=$(GIT_VERSION) \
+	ko publish --base-import-paths \
+		--tags $(GIT_VERSION) --tags $(GIT_HASH) --local --image-refs redisImagerefs \
+		github.com/sigstore/rekor/cmd/backfill-redis
+
 # This builds the trillian containers we rely on using ko for cross platform support
 .PHONY: ko-trillian
 ko-trillian:

cmd/backfill-redis/main.go

@@ -0,0 +1,181 @@
+// Copyright 2022 The Sigstore Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/*
+	backfill-redis is a script to populate the Redis index with entries
+	from Rekor. This is sometimes necessary because Redis caching is best
+	effort. If Redis returns an error, Rekor will not, and so sometimes
+	we need to backfill missing entries into Redis for the search API.
+
+	To run:
+	go run cmd/backfill-redis/main.go --rekor-address <address> \
+	    --hostname <redis-hostname> --port <redis-port>
+		--start <first index to backfill> --end <last index to backfill>
+*/
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/go-openapi/runtime"
+	radix "github.com/mediocregopher/radix/v4"
+	"sigs.k8s.io/release-utils/version"
+
+	"github.com/sigstore/rekor/pkg/client"
+	"github.com/sigstore/rekor/pkg/generated/client/entries"
+	"github.com/sigstore/rekor/pkg/generated/models"
+	"github.com/sigstore/rekor/pkg/types"
+
+	// these imports are to call the packages' init methods
+	_ "github.com/sigstore/rekor/pkg/types/alpine/v0.0.1"
+	_ "github.com/sigstore/rekor/pkg/types/cose/v0.0.1"
+	_ "github.com/sigstore/rekor/pkg/types/hashedrekord/v0.0.1"
+	_ "github.com/sigstore/rekor/pkg/types/helm/v0.0.1"
+	_ "github.com/sigstore/rekor/pkg/types/intoto/v0.0.1"
+
+	// remove 0.0.2 intoto type due to bugs
+	// _ "github.com/sigstore/rekor/pkg/types/intoto/v0.0.2"
+	_ "github.com/sigstore/rekor/pkg/types/jar/v0.0.1"
+	_ "github.com/sigstore/rekor/pkg/types/rekord/v0.0.1"
+	_ "github.com/sigstore/rekor/pkg/types/rfc3161/v0.0.1"
+	_ "github.com/sigstore/rekor/pkg/types/rpm/v0.0.1"
+	_ "github.com/sigstore/rekor/pkg/types/tuf/v0.0.1"
+)
+
+var (
+	redisHostname = flag.String("hostname", "", "Hostname for Redis application")
+	redisPort     = flag.String("port", "", "Port to Redis application")
+	startIndex    = flag.Int("start", -1, "First index to backfill")
+	endIndex      = flag.Int("end", -1, "Last index to backfill")
+	rekorAddress  = flag.String("rekor-address", "", "Address for Rekor, e.g. https://rekor.sigstore.dev")
+	versionFlag   = flag.Bool("version", false, "Print the current version of Backfill Redis")
+)
+
+func main() {
+	flag.Parse()
+
+	versionInfo := version.GetVersionInfo()
+	if *versionFlag {
+		fmt.Println(versionInfo.String())
+		os.Exit(0)
+	}
+
+	if *redisHostname == "" {
+		log.Fatal("address must be set")
+	}
+	if *redisPort == "" {
+		log.Fatal("port must be set")
+	}
+	if *startIndex == -1 {
+		log.Fatal("start must be set to >=0")
+	}
+	if *endIndex == -1 {
+		log.Fatal("end must be set to >=0")
+	}
+	if *rekorAddress == "" {
+		log.Fatal("rekor-address must be set")
+	}
+
+	log.Printf("running backfill redis Version: %s GitCommit: %s BuildDate: %s", versionInfo.GitVersion, versionInfo.GitCommit, versionInfo.BuildDate)
+
+	cfg := radix.PoolConfig{}
+	redisClient, err := cfg.New(context.Background(), "tcp", fmt.Sprintf("%s:%s", *redisHostname, *redisPort))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	rekorClient, err := client.GetRekorClient(*rekorAddress)
+	if err != nil {
+		log.Fatalf("creating rekor client: %v", err)
+	}
+
+	for i := *startIndex; i <= *endIndex; i++ {
+		params := entries.NewGetLogEntryByIndexParamsWithContext(context.Background())
+		params.SetLogIndex(int64(i))
+		resp, err := rekorClient.Entries.GetLogEntryByIndex(params)
+		if err != nil {
+			log.Fatalf("retrieving log uuid by index: %v", err)
+		}
+		var insertErrs []error
+		for uuid, entry := range resp.Payload {
+			// uuid is the global UUID - tree ID and entry UUID
+			e, _, _, err := unmarshalEntryImpl(entry.Body.(string))
+			if err != nil {
+				insertErrs = append(insertErrs, fmt.Errorf("error unmarshalling entry for %s: %v", uuid, err))
+				continue
+			}
+			keys, err := e.IndexKeys()
+			if err != nil {
+				insertErrs = append(insertErrs, fmt.Errorf("error building index keys for %s: %v", uuid, err))
+				continue
+			}
+			for _, key := range keys {
+				// remove the key-value pair from the index in case it already exists
+				if err := removeFromIndex(context.Background(), redisClient, key, uuid); err != nil {
+					insertErrs = append(insertErrs, fmt.Errorf("error removing UUID %s with key %s: %v", uuid, key, err))
+				}
+				if err := addToIndex(context.Background(), redisClient, key, uuid); err != nil {
+					insertErrs = append(insertErrs, fmt.Errorf("error inserting UUID %s with key %s: %v", uuid, key, err))
+				}
+				fmt.Printf("Uploaded Redis entry %s, index %d, key %s\n", uuid, i, key)
+			}
+		}
+		if len(insertErrs) != 0 {
+			fmt.Printf("Errors with log index %d:\n", i)
+			for _, e := range insertErrs {
+				fmt.Println(e)
+			}
+		} else {
+			fmt.Printf("Completed log index %d\n", i)
+		}
+	}
+}
+
+// unmarshalEntryImpl decodes the base64-encoded entry to a specific entry type (types.EntryImpl).
+// Taken from Cosign
+func unmarshalEntryImpl(e string) (types.EntryImpl, string, string, error) {
+	b, err := base64.StdEncoding.DecodeString(e)
+	if err != nil {
+		return nil, "", "", err
+	}
+
+	pe, err := models.UnmarshalProposedEntry(bytes.NewReader(b), runtime.JSONConsumer())
+	if err != nil {
+		return nil, "", "", err
+	}
+
+	entry, err := types.UnmarshalEntry(pe)
+	if err != nil {
+		return nil, "", "", err
+	}
+	return entry, pe.Kind(), entry.APIVersion(), nil
+}
+
+// removeFromIndex removes all occurrences of a value from a given key. This guards against
+// multiple invocations of backfilling creating duplicates.
+func removeFromIndex(ctx context.Context, redisClient radix.Client, key, value string) error {
+	return redisClient.Do(ctx, radix.Cmd(nil, "LREM", key, "0", value))
+}
+
+// addToIndex pushes a value onto a key of type list.
+func addToIndex(ctx context.Context, redisClient radix.Client, key, value string) error {
+	return redisClient.Do(ctx, radix.Cmd(nil, "LPUSH", key, value))
+}

release/ko-sign-release-images.sh

@@ -36,6 +36,11 @@ if [[ ! -f rekorCliImagerefs ]]; then
     exit 1
 fi
 
+if [[ ! -f bRedisImagerefs ]]; then
+    echo "bRedisImagerefs not found"
+    exit 1
+fi
+
 if [[ ! -f trillianServerImagerefs ]]; then
     echo "trillianServerImagerefs not found"
     exit 1
@@ -49,11 +54,13 @@ fi
 echo "Signing images with GCP KMS Key..."
 cosign sign --force --key "gcpkms://projects/$PROJECT_ID/locations/$KEY_LOCATION/keyRings/$KEY_RING/cryptoKeys/$KEY_NAME/versions/$KEY_VERSION" -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat rekorServerImagerefs)
 cosign sign --force --key "gcpkms://projects/$PROJECT_ID/locations/$KEY_LOCATION/keyRings/$KEY_RING/cryptoKeys/$KEY_NAME/versions/$KEY_VERSION" -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat rekorCliImagerefs)
+cosign sign --force --key "gcpkms://projects/$PROJECT_ID/locations/$KEY_LOCATION/keyRings/$KEY_RING/cryptoKeys/$KEY_NAME/versions/$KEY_VERSION" -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat bRedisImagerefs)
 cosign sign --force --key "gcpkms://projects/$PROJECT_ID/locations/$KEY_LOCATION/keyRings/$KEY_RING/cryptoKeys/$KEY_NAME/versions/$KEY_VERSION" -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat trillianServerImagerefs)
 cosign sign --force --key "gcpkms://projects/$PROJECT_ID/locations/$KEY_LOCATION/keyRings/$KEY_RING/cryptoKeys/$KEY_NAME/versions/$KEY_VERSION" -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat trillianSignerImagerefs)
 
 echo "Signing images with Keyless..."
 cosign sign --force -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat rekorServerImagerefs)
 cosign sign --force -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat rekorCliImagerefs)
+cosign sign --force -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat bRedisImagerefs)
 cosign sign --force -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat trillianServerImagerefs)
 cosign sign --force -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat trillianSignerImagerefs)

release/release.mk

@@ -5,12 +5,12 @@
 # used when releasing together with GCP CloudBuild
 .PHONY: release
 release:
-	CLI_LDFLAGS="$(CLI_LDFLAGS)" SERVER_LDFLAGS="$(SERVER_LDFLAGS)" goreleaser release --rm-dist --timeout 60m
+	CLI_LDFLAGS="$(CLI_LDFLAGS)" SERVER_LDFLAGS="$(SERVER_LDFLAGS)" goreleaser release --rm-dist --timeout 120m
 
 # used when need to validate the goreleaser
 .PHONY: snapshot
 snapshot:
-	CLI_LDFLAGS="$(CLI_LDFLAGS)" SERVER_LDFLAGS="$(SERVER_LDFLAGS)" goreleaser release --skip-sign --skip-publish --snapshot --rm-dist
+	CLI_LDFLAGS="$(CLI_LDFLAGS)" SERVER_LDFLAGS="$(SERVER_LDFLAGS)" goreleaser release --skip-sign --skip-publish --snapshot --rm-dist --timeout 120m
 
 ###########################
 # sign section
@@ -33,6 +33,10 @@ copy-rekor-server-signed-release-to-ghcr:
 copy-rekor-cli-signed-release-to-ghcr:
 	cosign copy $(KO_PREFIX)/rekor-cli:$(GIT_VERSION) $(GHCR_PREFIX)/rekor-cli:$(GIT_VERSION)
 
+.PHONY: copy-backfill-redis-signed-release-to-ghcr
+copy-backfill-redis-signed-release-to-ghcr:
+	cosign copy $(KO_PREFIX)/backfill-redis:$(GIT_VERSION) $(GHCR_PREFIX)/backfill-redis:$(GIT_VERSION)
+
 .PHONY: copy-trillian-log-server-signed-release-to-ghcr
 copy-trillian-log-server-signed-release-to-ghcr:
 	cosign copy $(KO_PREFIX)/trillian_log_server:$(GIT_VERSION) $(GHCR_PREFIX)/trillian_log_server:$(GIT_VERSION)
@@ -42,7 +46,7 @@ copy-trillian-log-signer-signed-release-to-ghcr:
 	cosign copy $(KO_PREFIX)/trillian_log_signer:$(GIT_VERSION) $(GHCR_PREFIX)/trillian_log_signer:$(GIT_VERSION)
 
 .PHONY: copy-signed-release-to-ghcr
-copy-signed-release-to-ghcr: copy-rekor-server-signed-release-to-ghcr copy-rekor-cli-signed-release-to-ghcr copy-trillian-log-signer-signed-release-to-ghcr copy-trillian-log-server-signed-release-to-ghcr
+copy-signed-release-to-ghcr: copy-rekor-server-signed-release-to-ghcr copy-rekor-cli-signed-release-to-ghcr copy-backfill-redis-signed-release-to-ghcr copy-trillian-log-signer-signed-release-to-ghcr copy-trillian-log-server-signed-release-to-ghcr
 
 ## --------------------------------------
 ## Dist / maybe we can deprecate