Search through all shards when searching by hash

.gitignore

@@ -17,3 +17,5 @@ rekorServerImagerefs
 rekorCliImagerefs
 trillianServerImagerefs
 trillianSignerImagerefs
+cosign.*
+signature

pkg/api/entries.go

@@ -415,23 +415,27 @@ func SearchLogQueryHandler(params entries.SearchLogQueryParams) middleware.Respo
 			searchHashes = append(searchHashes, hash)
 		}
 
-		searchByHashResults := make([]*trillian.GetEntryAndProofResponse, len(searchHashes))
+		searchByHashResults := make([][]*trillian.GetEntryAndProofResponse, len(searchHashes))
 		g, _ = errgroup.WithContext(httpReqCtx)
 		for i, hash := range searchHashes {
 			i, hash := i, hash // https://golang.org/doc/faq#closures_and_goroutines
 			g.Go(func() error {
-				resp := tc.getLeafAndProofByHash(hash)
-				switch resp.status {
-				case codes.OK:
-				case codes.NotFound:
-					code = http.StatusNotFound
-					return resp.err
-				default:
+				var results []*trillian.GetEntryAndProofResponse
+				for _, shard := range api.logRanges.AllShards() {
+					tcs := NewTrillianClientFromTreeID(httpReqCtx, shard)
+					resp := tcs.getLeafAndProofByHash(hash)
+					if resp.status != codes.OK {
+						continue
+					}
+					leafResult := resp.getLeafAndProofResult
+					if leafResult != nil && leafResult.Leaf != nil {
+						results = append(results, resp.getLeafAndProofResult)
+					}
 				}
-				leafResult := resp.getLeafAndProofResult
-				if leafResult != nil && leafResult.Leaf != nil {
-					searchByHashResults[i] = leafResult
+				if results == nil {
+					return fmt.Errorf("no responses found")
 				}
+				searchByHashResults[i] = results
 				return nil
 			})
 		}
@@ -440,14 +444,20 @@ func SearchLogQueryHandler(params entries.SearchLogQueryParams) middleware.Respo
 			return handleRekorAPIError(params, code, err, err.Error())
 		}
 
-		for _, leafResp := range searchByHashResults {
-			if leafResp != nil {
-				logEntry, err := logEntryFromLeaf(httpReqCtx, api.signer, tc, leafResp.Leaf, leafResp.SignedLogRoot, leafResp.Proof, api.logRanges.ActiveTreeID(), api.logRanges)
+		var flattenedHashResults []*trillian.GetEntryAndProofResponse
+		for _, s := range searchByHashResults {
+			flattenedHashResults = append(flattenedHashResults, s...)
+		}
+
+		for _, leafResp := range flattenedHashResults {
+			if leafResp == nil {
+				continue
+			}
+			for _, shard := range api.logRanges.AllShards() {
+				logEntry, err := logEntryFromLeaf(httpReqCtx, api.signer, tc, leafResp.Leaf, leafResp.SignedLogRoot, leafResp.Proof, shard, api.logRanges)
 				if err != nil {
-					code = http.StatusInternalServerError
-					return handleRekorAPIError(params, code, err, err.Error())
+					continue
 				}
-
 				resultPayload = append(resultPayload, logEntry)
 			}
 		}

pkg/sharding/ranges.go

@@ -138,6 +138,15 @@ func (l *LogRanges) NoInactive() bool {
 	return l.inactive == nil
 }
 
+// AllShards returns all shards, starting with the active shard and then the inactive shards
+func (l *LogRanges) AllShards() []int64 {
+	shards := []int64{l.ActiveTreeID()}
+	for _, in := range l.GetInactive() {
+		shards = append(shards, in.TreeID)
+	}
+	return shards
+}
+
 // TotalInactiveLength returns the total length across all inactive shards;
 // we don't know the length of the active shard.
 func (l *LogRanges) TotalInactiveLength() int64 {

tests/sharding-e2e-test.sh

@@ -24,6 +24,8 @@ set -ex
 echo "Installing createtree..."
 go install github.com/google/trillian/cmd/createtree@latest
 
+echo "Installing cosign..."
+go install github.com/sigstore/cosign/cmd/cosign@latest
 
 echo "starting services"
 docker-compose up -d
@@ -34,6 +36,7 @@ go build -o rekor-cli ./cmd/rekor-cli
 REKOR_CLI=$(pwd)/rekor-cli
 go build -o rekor-server ./cmd/rekor-server
 
+
 function check_log_index () {
   logIndex=$1
   # make sure we can get this log index from rekor
@@ -99,10 +102,10 @@ trap "collectLogsOnFailure $?" EXIT
 echo "Waiting for rekor server to come up..."
 waitForRekorServer
 
-# Add some things to the tlog :)
-pushd tests
-$REKOR_CLI upload --artifact test_file.txt --signature test_file.sig --public-key test_public_key.key --rekor_server http://localhost:3000
-popd
+# Sign a blob and push to the log with cosign
+rm cosign.*
+COSIGN_PASSWORD=pw cosign generate-key-pair
+COSIGN_PASSWORD=pw COSIGN_EXPERIMENTAL=1 cosign sign-blob README.md --key cosign.key --rekor-url http://localhost:3000 --output-signature ./signature
 
 # Make sure we can prove consistency
 $REKOR_CLI loginfo --rekor_server http://localhost:3000 
@@ -251,6 +254,9 @@ ENTRY_ID_2=$($REKOR_CLI get --log-index 3 --rekor_server http://localhost:3000 -
 NUM_ELEMENTS=$(curl -f http://localhost:3000/api/v1/log/entries/retrieve -H "Content-Type: application/json" -H "Accept: application/json" -d "{ \"entryUUIDs\": [\"$ENTRY_ID_1\"]}" | jq '. | length')
 stringsMatch $NUM_ELEMENTS "1"
 
+# Make sure we can verify the blob we entered into the now-inactive shard
+COSIGN_EXPERIMENTAL=1 cosign verify-blob README.md --key cosign.pub --rekor-url http://localhost:3000  --signature ./signature
+
 # -f makes sure we exit on failure
 NUM_ELEMENTS=$(curl -f http://localhost:3000/api/v1/log/entries/retrieve -H "Content-Type: application/json" -H "Accept: application/json" -d "{ \"entryUUIDs\": [\"$ENTRY_ID_1\", \"$ENTRY_ID_2\"]}" | jq '. | length')
 stringsMatch $NUM_ELEMENTS "2"