From 6551890ccfd442b6e1352fdc5df3d602bb2a7c78 Mon Sep 17 00:00:00 2001 From: Naveen <172697+naveensrinivasan@users.noreply.github.com> Date: Mon, 24 Oct 2022 19:05:58 -0500 Subject: [PATCH] Revamping rekor e2e - part 1 of N (#1089) * Refactor - e2e tests - Refactor e2e tests to move it to specific folders. Signed-off-by: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com> * More tweaks to fix the e2e Signed-off-by: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com> * Increased the timeout. Signed-off-by: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com> * Updated the code with the coverage information. Signed-off-by: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com> * Fixed the coverage issues. Signed-off-by: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com> * Fixed coverage flags based on comments. Signed-off-by: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com> * Removed forced-recreate option. Signed-off-by: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com> Signed-off-by: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com> Signed-off-by: Naveen <172697+naveensrinivasan@users.noreply.github.com> --- .github/workflows/main.yml | 4 +- .gitignore | 4 +- pkg/e2e-test.sh | 77 ++++++++ pkg/pki/tuf/e2e_test.go | 44 +++++ tests/tuf.go => pkg/pki/tuf/tuf_e2e_test.go | 3 +- pkg/util/util.go | 206 ++++++++++++++++++++ tests/e2e_test.go | 19 -- 7 files changed, 334 insertions(+), 23 deletions(-) create mode 100755 pkg/e2e-test.sh create mode 100644 pkg/pki/tuf/e2e_test.go rename tests/tuf.go => pkg/pki/tuf/tuf_e2e_test.go (98%) create mode 100644 pkg/util/util.go diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9ce0d699c..6c3c27a7d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -84,6 +84,8 @@ jobs: - name: CLI run: ./tests/e2e-test.sh + - name: PKG-CLI # this will a WIP to move all the CLI tests to the pkg repo + run: ./pkg/e2e-test.sh - name: Upload logs if they exist uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # v3.1.1 if: failure() @@ -93,7 +95,7 @@ jobs: - name: Upload Coverage Report uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 with: - files: /tmp/rekor-merged.cov + files: /tmp/rekor-merged.cov,/tmp/pkg-rekor-merged.cov flags: e2etests sharding-e2e: diff --git a/.gitignore b/.gitignore index 1c0c64c22..ba4e04ea6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,8 +3,8 @@ .vscode/* /cli logid -/rekor-cli -/rekor-server +rekor-cli +rekor-server /tests/rekor-server /server swagger diff --git a/pkg/e2e-test.sh b/pkg/e2e-test.sh new file mode 100755 index 000000000..51e42b814 --- /dev/null +++ b/pkg/e2e-test.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# +# Copyright 2022 The Sigstore Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +testdir=$(dirname "$0") + +rm -f /tmp/pkg-rekor-*.cov +echo "installing gocovmerge" +make gocovmerge +docker kill $(docker ps -q) || true +echo "starting services" +docker-compose -f docker-compose.yml -f docker-compose.test.yml up -d --build + +echo "building CLI and server" +# set the path to the root of the repo +dir=$(git rev-parse --show-toplevel) +go test -c ./cmd/rekor-cli -o rekor-cli -cover -covermode=count -coverpkg=./... +go test -c ./cmd/rekor-server -o rekor-server -covermode=count -coverpkg=./... + +count=0 +echo -n "waiting up to 120 sec for system to start" +until [ $(docker-compose ps | grep -c "(healthy)") == 3 ]; +do + if [ $count -eq 12 ]; then + echo "! timeout reached" + exit 1 + else + echo -n "." + sleep 10 + let 'count+=1' + fi +done + +echo +echo "running tests" +REKORTMPDIR="$(mktemp -d -t rekor_test.XXXXXX)" +cp $dir/rekor-cli $REKORTMPDIR/rekor-cli +touch $REKORTMPDIR.rekor.yaml +trap "rm -rf $REKORTMPDIR" EXIT +if ! REKORTMPDIR=$REKORTMPDIR go test -tags=e2e ./pkg/...; then + docker-compose logs --no-color > /tmp/docker-compose.log + exit 1 +fi +if docker-compose logs --no-color | grep -q "panic: runtime error:" ; then + # if we're here, we found a panic + echo "Failing due to panics detected in logs" + docker-compose logs --no-color > /tmp/docker-compose.log + exit 1 +fi + +echo "generating code coverage" +curl -X GET 0.0.0.0:2345/kill +sleep 5 + +if ! docker cp $(docker ps -aqf "name=rekor_rekor-server"):go/rekor-server.cov /tmp/pkg-rekor-server.cov ; then + # failed to copy code coverage report from server + echo "Failed to retrieve server code coverage report" + docker-compose logs --no-color > /tmp/docker-compose.log + exit 1 +fi + +# merging coverage reports and filtering out /pkg/generated from final report +hack/tools/bin/gocovmerge /tmp/pkg-rekor-*.cov | grep -v "/pkg/generated/" > /tmp/pkg-rekor-merged.cov +echo "code coverage $(go tool cover -func=/tmp/pkg-rekor-merged.cov | grep -E '^total\:' | sed -E 's/\s+/ /g')" diff --git a/pkg/pki/tuf/e2e_test.go b/pkg/pki/tuf/e2e_test.go new file mode 100644 index 000000000..fd5aa6dc1 --- /dev/null +++ b/pkg/pki/tuf/e2e_test.go @@ -0,0 +1,44 @@ +// +// Copyright 2022 The Sigstore Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build e2e +// +build e2e + +package tuf + +import ( + "github.com/sigstore/rekor/pkg/util" + "path/filepath" + "testing" +) + +func TestTufVerifyUpload(t *testing.T) { + artifactPath := filepath.Join(t.TempDir(), "timestamp.json") + rootPath := filepath.Join(t.TempDir(), "root.json") + + createTufSignedArtifact(t, artifactPath, rootPath) + + // Now upload to rekor! + out := util.RunCli(t, "upload", "--artifact", artifactPath, "--public-key", rootPath, "--type", "tuf") + util.OutputContains(t, out, "Created entry at") + + uuid := util.GetUUIDFromUploadOutput(t, out) + + out = util.RunCli(t, "verify", "--artifact", artifactPath, "--public-key", rootPath, "--type", "tuf") + util.OutputContains(t, out, "Inclusion Proof") + + out = util.RunCli(t, "search", "--public-key", rootPath, "--pki-format", "tuf") + util.OutputContains(t, out, uuid) +} diff --git a/tests/tuf.go b/pkg/pki/tuf/tuf_e2e_test.go similarity index 98% rename from tests/tuf.go rename to pkg/pki/tuf/tuf_e2e_test.go index 3714057f3..8cdf32cec 100644 --- a/tests/tuf.go +++ b/pkg/pki/tuf/tuf_e2e_test.go @@ -13,9 +13,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build e2e // +build e2e -package e2e +package tuf import ( "io/ioutil" diff --git a/pkg/util/util.go b/pkg/util/util.go new file mode 100644 index 000000000..26fb131d6 --- /dev/null +++ b/pkg/util/util.go @@ -0,0 +1,206 @@ +// +// Copyright 2022 The Sigstore Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build e2e +// +build e2e + +package util + +import ( + "encoding/base64" + "fmt" + "io/ioutil" + "math/rand" + "os" + "os/exec" + "path" + "strings" + "testing" + "time" + + "github.com/sigstore/rekor/pkg/generated/models" +) + +var ( + cli = "rekor-cli" + server = "rekor-server" + nodeDataDir = "node" +) + +func init() { + p := os.Getenv("REKORTMPDIR") + if p != "" { + cli = path.Join(p, cli) + server = path.Join(p, server) + } +} + +func OutputContains(t *testing.T, output, sub string) { + t.Helper() + if !strings.Contains(output, sub) { + t.Errorf("Expected [%s] in response, got %s", sub, output) + } +} + +func Run(t *testing.T, stdin, cmd string, arg ...string) string { + t.Helper() + arg = append([]string{coverageFlag()}, arg...) + c := exec.Command(cmd, arg...) + if stdin != "" { + c.Stdin = strings.NewReader(stdin) + } + if os.Getenv("REKORTMPDIR") != "" { + // ensure that we use a clean state.json file for each Run + c.Env = append(c.Env, "HOME="+os.Getenv("REKORTMPDIR")) + } + b, err := c.CombinedOutput() + if err != nil { + t.Log(string(b)) + t.Fatal(err) + } + return stripCoverageOutput(string(b)) +} + +func RunCli(t *testing.T, arg ...string) string { + t.Helper() + arg = append(arg, rekorServerFlag()) + // use a blank config file to ensure no collision + if os.Getenv("REKORTMPDIR") != "" { + arg = append(arg, "--config="+os.Getenv("REKORTMPDIR")+".rekor.yaml") + } + return Run(t, "", cli, arg...) +} + +func RunCliStdout(t *testing.T, arg ...string) string { + t.Helper() + arg = append([]string{coverageFlag()}, arg...) + arg = append(arg, rekorServerFlag()) + c := exec.Command(cli, arg...) + + if os.Getenv("REKORTMPDIR") != "" { + // ensure that we use a clean state.json file for each Run + c.Env = append(c.Env, "HOME="+os.Getenv("REKORTMPDIR")) + } + b, err := c.Output() + if err != nil { + t.Log(string(b)) + t.Fatal(err) + } + return stripCoverageOutput(string(b)) +} + +func RunCliErr(t *testing.T, arg ...string) string { + t.Helper() + arg = append([]string{coverageFlag()}, arg...) + arg = append(arg, rekorServerFlag()) + // use a blank config file to ensure no collision + if os.Getenv("REKORTMPDIR") != "" { + arg = append(arg, "--config="+os.Getenv("REKORTMPDIR")+".rekor.yaml") + } + cmd := exec.Command(cli, arg...) + b, err := cmd.CombinedOutput() + if err == nil { + t.Log(string(b)) + t.Fatalf("expected error, got %s", string(b)) + } + return stripCoverageOutput(string(b)) +} + +func rekorServerFlag() string { + return fmt.Sprintf("--rekor_server=%s", rekorServer()) +} + +func rekorServer() string { + if s := os.Getenv("REKOR_SERVER"); s != "" { + return s + } + return "http://localhost:3000" +} + +func coverageFlag() string { + return "-test.coverprofile=/tmp/pkg-rekor-cli." + randomSuffix(8) + ".cov" +} + +func stripCoverageOutput(out string) string { + return strings.Split(strings.Split(out, "PASS")[0], "FAIL")[0] +} + +func readFile(t *testing.T, p string) string { + b, err := ioutil.ReadFile(p) + if err != nil { + t.Fatal(err) + } + return strings.TrimSpace(string(b)) +} + +func randomSuffix(n int) string { + const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + + b := make([]byte, n) + for i := range b { + b[i] = letterBytes[rand.Intn(len(letterBytes))] + } + return string(b) +} + +func randomData(t *testing.T, n int) []byte { + t.Helper() + rand.Seed(time.Now().UnixNano()) + data := make([]byte, n) + if _, err := rand.Read(data[:]); err != nil { + t.Fatal(err) + } + return data +} + +func createArtifact(t *testing.T, artifactPath string) string { + t.Helper() + // First let's generate some random data so we don't have to worry about dupes. + data := randomData(t, 100) + + artifact := base64.StdEncoding.EncodeToString(data[:]) + // Write this to a file + write(t, artifact, artifactPath) + return artifact +} + +func extractLogEntry(t *testing.T, le models.LogEntry) models.LogEntryAnon { + t.Helper() + + if len(le) != 1 { + t.Fatal("expected length to be 1, is actually", len(le)) + } + for _, v := range le { + return v + } + // this should never happen + return models.LogEntryAnon{} +} + +func write(t *testing.T, data string, path string) { + t.Helper() + if err := ioutil.WriteFile(path, []byte(data), 0644); err != nil { + t.Fatal(err) + } +} + +func GetUUIDFromUploadOutput(t *testing.T, out string) string { + t.Helper() + // Output looks like "Artifact timestamped at ...\m Wrote response \n Created entry at index X, available at $URL/UUID", so grab the UUID: + urlTokens := strings.Split(strings.TrimSpace(out), " ") + url := urlTokens[len(urlTokens)-1] + splitUrl := strings.Split(url, "/") + return splitUrl[len(splitUrl)-1] +} diff --git a/tests/e2e_test.go b/tests/e2e_test.go index f6f568ab5..7196dc600 100644 --- a/tests/e2e_test.go +++ b/tests/e2e_test.go @@ -1076,25 +1076,6 @@ func TestEntryUpload(t *testing.T) { outputContains(t, out, "Created entry at") } -func TestTufVerifyUpload(t *testing.T) { - artifactPath := filepath.Join(t.TempDir(), "timestamp.json") - rootPath := filepath.Join(t.TempDir(), "root.json") - - createTufSignedArtifact(t, artifactPath, rootPath) - - // Now upload to rekor! - out := runCli(t, "upload", "--artifact", artifactPath, "--public-key", rootPath, "--type", "tuf") - outputContains(t, out, "Created entry at") - - uuid := getUUIDFromUploadOutput(t, out) - - out = runCli(t, "verify", "--artifact", artifactPath, "--public-key", rootPath, "--type", "tuf") - outputContains(t, out, "Inclusion Proof") - - out = runCli(t, "search", "--public-key", rootPath, "--pki-format", "tuf") - outputContains(t, out, uuid) -} - // Regression test for https://github.com/sigstore/rekor/pull/956 // Requesting an inclusion proof concurrently with an entry write triggers // a race where the inclusion proof returned does not verify because the