Revamping rekor e2e - part 1 of N (#1089)

* Refactor - e2e tests

- Refactor e2e tests to move it to specific folders.

Signed-off-by: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com>

* More tweaks to fix the e2e

Signed-off-by: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com>

* Increased the timeout.

Signed-off-by: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com>

* Updated the code with the coverage information.

Signed-off-by: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com>

* Fixed the coverage issues.

Signed-off-by: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com>

* Fixed coverage flags based on comments.

Signed-off-by: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com>

* Removed forced-recreate option.

Signed-off-by: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com>

Signed-off-by: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com>
Signed-off-by: Naveen <172697+naveensrinivasan@users.noreply.github.com>

.github/workflows/main.yml

@@ -84,6 +84,8 @@ jobs:
 
       - name: CLI
         run: ./tests/e2e-test.sh
+      - name: PKG-CLI # this will a WIP to move all the CLI tests to the pkg repo
+        run: ./pkg/e2e-test.sh
       - name: Upload logs if they exist
         uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # v3.1.1
         if: failure()
@@ -93,7 +95,7 @@ jobs:
       - name: Upload Coverage Report
         uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0
         with:
-          files: /tmp/rekor-merged.cov
+          files: /tmp/rekor-merged.cov,/tmp/pkg-rekor-merged.cov
           flags: e2etests
 
   sharding-e2e:

.gitignore

@@ -3,8 +3,8 @@
 .vscode/*
 /cli
 logid
-/rekor-cli
-/rekor-server
+rekor-cli
+rekor-server
 /tests/rekor-server
 /server
 swagger

pkg/e2e-test.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+#
+# Copyright 2022 The Sigstore Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+testdir=$(dirname "$0")
+
+rm -f /tmp/pkg-rekor-*.cov
+echo "installing gocovmerge"
+make gocovmerge
+docker kill $(docker ps -q) || true
+echo "starting services"
+docker-compose -f docker-compose.yml -f docker-compose.test.yml up -d --build
+
+echo "building CLI and server"
+# set the path to the root of the repo
+dir=$(git rev-parse --show-toplevel)
+go test -c ./cmd/rekor-cli -o rekor-cli -cover -covermode=count -coverpkg=./...
+go test -c ./cmd/rekor-server -o rekor-server -covermode=count -coverpkg=./...
+
+count=0
+echo -n "waiting up to 120 sec for system to start"
+until [ $(docker-compose ps | grep -c "(healthy)") == 3 ];
+do
+    if [ $count -eq 12 ]; then
+       echo "! timeout reached"
+       exit 1
+    else
+       echo -n "."
+       sleep 10
+       let 'count+=1'
+    fi
+done
+
+echo
+echo "running tests"
+REKORTMPDIR="$(mktemp -d -t rekor_test.XXXXXX)"
+cp $dir/rekor-cli $REKORTMPDIR/rekor-cli
+touch $REKORTMPDIR.rekor.yaml
+trap "rm -rf $REKORTMPDIR" EXIT
+if ! REKORTMPDIR=$REKORTMPDIR go test -tags=e2e ./pkg/...; then
+   docker-compose logs --no-color > /tmp/docker-compose.log
+   exit 1
+fi
+if docker-compose logs --no-color | grep -q "panic: runtime error:" ; then
+   # if we're here, we found a panic
+   echo "Failing due to panics detected in logs"
+   docker-compose logs --no-color > /tmp/docker-compose.log
+   exit 1
+fi
+
+echo "generating code coverage"
+curl -X GET 0.0.0.0:2345/kill
+sleep 5
+
+if ! docker cp $(docker ps -aqf "name=rekor_rekor-server"):go/rekor-server.cov /tmp/pkg-rekor-server.cov ; then
+   # failed to copy code coverage report from server
+   echo "Failed to retrieve server code coverage report"
+   docker-compose logs --no-color > /tmp/docker-compose.log
+   exit 1
+fi
+
+# merging coverage reports and filtering out /pkg/generated from final report
+hack/tools/bin/gocovmerge /tmp/pkg-rekor-*.cov | grep -v "/pkg/generated/" > /tmp/pkg-rekor-merged.cov
+echo "code coverage $(go tool cover -func=/tmp/pkg-rekor-merged.cov | grep -E '^total\:' | sed -E 's/\s+/ /g')"

pkg/pki/tuf/e2e_test.go

@@ -0,0 +1,44 @@
+//
+// Copyright 2022 The Sigstore Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build e2e
+// +build e2e
+
+package tuf
+
+import (
+	"github.com/sigstore/rekor/pkg/util"
+	"path/filepath"
+	"testing"
+)
+
+func TestTufVerifyUpload(t *testing.T) {
+	artifactPath := filepath.Join(t.TempDir(), "timestamp.json")
+	rootPath := filepath.Join(t.TempDir(), "root.json")
+
+	createTufSignedArtifact(t, artifactPath, rootPath)
+
+	// Now upload to rekor!
+	out := util.RunCli(t, "upload", "--artifact", artifactPath, "--public-key", rootPath, "--type", "tuf")
+	util.OutputContains(t, out, "Created entry at")
+
+	uuid := util.GetUUIDFromUploadOutput(t, out)
+
+	out = util.RunCli(t, "verify", "--artifact", artifactPath, "--public-key", rootPath, "--type", "tuf")
+	util.OutputContains(t, out, "Inclusion Proof")
+
+	out = util.RunCli(t, "search", "--public-key", rootPath, "--pki-format", "tuf")
+	util.OutputContains(t, out, uuid)
+}

tests/tuf.go → pkg/pki/tuf/tuf_e2e_test.go

@@ -13,9 +13,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build e2e
 // +build e2e
 
-package e2e
+package tuf
 
 import (
 	"io/ioutil"

pkg/util/util.go

@@ -0,0 +1,206 @@
+//
+// Copyright 2022 The Sigstore Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build e2e
+// +build e2e
+
+package util
+
+import (
+	"encoding/base64"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/sigstore/rekor/pkg/generated/models"
+)
+
+var (
+	cli         = "rekor-cli"
+	server      = "rekor-server"
+	nodeDataDir = "node"
+)
+
+func init() {
+	p := os.Getenv("REKORTMPDIR")
+	if p != "" {
+		cli = path.Join(p, cli)
+		server = path.Join(p, server)
+	}
+}
+
+func OutputContains(t *testing.T, output, sub string) {
+	t.Helper()
+	if !strings.Contains(output, sub) {
+		t.Errorf("Expected [%s] in response, got %s", sub, output)
+	}
+}
+
+func Run(t *testing.T, stdin, cmd string, arg ...string) string {
+	t.Helper()
+	arg = append([]string{coverageFlag()}, arg...)
+	c := exec.Command(cmd, arg...)
+	if stdin != "" {
+		c.Stdin = strings.NewReader(stdin)
+	}
+	if os.Getenv("REKORTMPDIR") != "" {
+		// ensure that we use a clean state.json file for each Run
+		c.Env = append(c.Env, "HOME="+os.Getenv("REKORTMPDIR"))
+	}
+	b, err := c.CombinedOutput()
+	if err != nil {
+		t.Log(string(b))
+		t.Fatal(err)
+	}
+	return stripCoverageOutput(string(b))
+}
+
+func RunCli(t *testing.T, arg ...string) string {
+	t.Helper()
+	arg = append(arg, rekorServerFlag())
+	// use a blank config file to ensure no collision
+	if os.Getenv("REKORTMPDIR") != "" {
+		arg = append(arg, "--config="+os.Getenv("REKORTMPDIR")+".rekor.yaml")
+	}
+	return Run(t, "", cli, arg...)
+}
+
+func RunCliStdout(t *testing.T, arg ...string) string {
+	t.Helper()
+	arg = append([]string{coverageFlag()}, arg...)
+	arg = append(arg, rekorServerFlag())
+	c := exec.Command(cli, arg...)
+
+	if os.Getenv("REKORTMPDIR") != "" {
+		// ensure that we use a clean state.json file for each Run
+		c.Env = append(c.Env, "HOME="+os.Getenv("REKORTMPDIR"))
+	}
+	b, err := c.Output()
+	if err != nil {
+		t.Log(string(b))
+		t.Fatal(err)
+	}
+	return stripCoverageOutput(string(b))
+}
+
+func RunCliErr(t *testing.T, arg ...string) string {
+	t.Helper()
+	arg = append([]string{coverageFlag()}, arg...)
+	arg = append(arg, rekorServerFlag())
+	// use a blank config file to ensure no collision
+	if os.Getenv("REKORTMPDIR") != "" {
+		arg = append(arg, "--config="+os.Getenv("REKORTMPDIR")+".rekor.yaml")
+	}
+	cmd := exec.Command(cli, arg...)
+	b, err := cmd.CombinedOutput()
+	if err == nil {
+		t.Log(string(b))
+		t.Fatalf("expected error, got %s", string(b))
+	}
+	return stripCoverageOutput(string(b))
+}
+
+func rekorServerFlag() string {
+	return fmt.Sprintf("--rekor_server=%s", rekorServer())
+}
+
+func rekorServer() string {
+	if s := os.Getenv("REKOR_SERVER"); s != "" {
+		return s
+	}
+	return "http://localhost:3000"
+}
+
+func coverageFlag() string {
+	return "-test.coverprofile=/tmp/pkg-rekor-cli." + randomSuffix(8) + ".cov"
+}
+
+func stripCoverageOutput(out string) string {
+	return strings.Split(strings.Split(out, "PASS")[0], "FAIL")[0]
+}
+
+func readFile(t *testing.T, p string) string {
+	b, err := ioutil.ReadFile(p)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return strings.TrimSpace(string(b))
+}
+
+func randomSuffix(n int) string {
+	const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = letterBytes[rand.Intn(len(letterBytes))]
+	}
+	return string(b)
+}
+
+func randomData(t *testing.T, n int) []byte {
+	t.Helper()
+	rand.Seed(time.Now().UnixNano())
+	data := make([]byte, n)
+	if _, err := rand.Read(data[:]); err != nil {
+		t.Fatal(err)
+	}
+	return data
+}
+
+func createArtifact(t *testing.T, artifactPath string) string {
+	t.Helper()
+	// First let's generate some random data so we don't have to worry about dupes.
+	data := randomData(t, 100)
+
+	artifact := base64.StdEncoding.EncodeToString(data[:])
+	// Write this to a file
+	write(t, artifact, artifactPath)
+	return artifact
+}
+
+func extractLogEntry(t *testing.T, le models.LogEntry) models.LogEntryAnon {
+	t.Helper()
+
+	if len(le) != 1 {
+		t.Fatal("expected length to be 1, is actually", len(le))
+	}
+	for _, v := range le {
+		return v
+	}
+	// this should never happen
+	return models.LogEntryAnon{}
+}
+
+func write(t *testing.T, data string, path string) {
+	t.Helper()
+	if err := ioutil.WriteFile(path, []byte(data), 0644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func GetUUIDFromUploadOutput(t *testing.T, out string) string {
+	t.Helper()
+	// Output looks like "Artifact timestamped at ...\m Wrote response \n Created entry at index X, available at $URL/UUID", so grab the UUID:
+	urlTokens := strings.Split(strings.TrimSpace(out), " ")
+	url := urlTokens[len(urlTokens)-1]
+	splitUrl := strings.Split(url, "/")
+	return splitUrl[len(splitUrl)-1]
+}

tests/e2e_test.go

@@ -1076,25 +1076,6 @@ func TestEntryUpload(t *testing.T) {
 	outputContains(t, out, "Created entry at")
 }
 
-func TestTufVerifyUpload(t *testing.T) {
-	artifactPath := filepath.Join(t.TempDir(), "timestamp.json")
-	rootPath := filepath.Join(t.TempDir(), "root.json")
-
-	createTufSignedArtifact(t, artifactPath, rootPath)
-
-	// Now upload to rekor!
-	out := runCli(t, "upload", "--artifact", artifactPath, "--public-key", rootPath, "--type", "tuf")
-	outputContains(t, out, "Created entry at")
-
-	uuid := getUUIDFromUploadOutput(t, out)
-
-	out = runCli(t, "verify", "--artifact", artifactPath, "--public-key", rootPath, "--type", "tuf")
-	outputContains(t, out, "Inclusion Proof")
-
-	out = runCli(t, "search", "--public-key", rootPath, "--pki-format", "tuf")
-	outputContains(t, out, uuid)
-}
-
 // Regression test for https://github.com/sigstore/rekor/pull/956
 // Requesting an inclusion proof concurrently with an entry write triggers
 // a race where the inclusion proof returned does not verify because the