Adding e2e test coverage (#1071)

* configuring app for test coverage

Signed-off-by: Ceridwen Driskill <cdriskill@google.com>

* update docker files to build and run with coverage

Signed-off-by: Ceridwen Driskill <cdriskill@google.com>

* updating tests to account for test coverage output

Signed-off-by: Ceridwen Driskill <cdriskill@google.com>

* adding e2e coverage to github actions in theory

Signed-off-by: Ceridwen Driskill <cdriskill@google.com>

* removing /pkg/generated/ from coverage report

Signed-off-by: Ceridwen Driskill <cdriskill@google.com>

* lint

Signed-off-by: Ceridwen Driskill <cdriskill@google.com>

* separate port for server test killswitch

Signed-off-by: Ceridwen Driskill <cdriskill@google.com>

* hide killswitch flag

Signed-off-by: Ceridwen Driskill <cdriskill@google.com>

* pin gocovmerge, factor out stripCoverageOutput

Signed-off-by: Ceridwen Driskill <cdriskill@google.com>

Signed-off-by: Ceridwen Driskill <cdriskill@google.com>

.github/workflows/main.yml

@@ -44,6 +44,8 @@ jobs:
         run: go test -v -coverprofile=coverage.txt -covermode=atomic ./...
       - name: Upload Coverage Report
         uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.0
+        with:
+          flags: unittests
       - name: Ensure no files were modified as a result of the build
         run: git update-index --refresh && git diff-index --quiet HEAD -- || git diff --exit-code
 
@@ -72,13 +74,13 @@ jobs:
       - name: download minisign
         run: sudo add-apt-repository ppa:dysfunctionalprogramming/minisign && sudo apt-get update && sudo apt-get install minisign
       - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v3.0.2
-      - name: Docker Build
-        run: docker-compose build
       - name: Extract version of Go to use
         run: echo "GOVERSION=$(cat Dockerfile|grep golang | awk ' { print $2 } ' | cut -d '@' -f 1 | cut -d ':' -f 2 | uniq)" >> $GITHUB_ENV
       - uses: actions/setup-go@268d8c0ca0432bb2cf416faae41297df9d262d7f # v3.1.0
         with:
           go-version: ${{ env.GOVERSION }}
+      - name: install gocovmerge
+        run: go install github.com/wadey/gocovmerge@b5bfa59ec0adc420475f97f89b58045c721d761c
 
       - name: CLI
         run: ./tests/e2e-test.sh
@@ -88,6 +90,11 @@ jobs:
         with:
           name: E2E Docker Compose logs
           path: /tmp/docker-compose.log
+      - name: Upload Coverage Report
+        uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0
+        with:
+          files: /tmp/rekor-merged.cov
+          flags: e2etests
 
   sharding-e2e:
     runs-on: ubuntu-20.04

Dockerfile

@@ -28,6 +28,7 @@ ADD ./pkg/ $APP_ROOT/src/pkg/
 ARG SERVER_LDFLAGS
 RUN go build -ldflags "${SERVER_LDFLAGS}" ./cmd/rekor-server
 RUN CGO_ENABLED=0 go build -gcflags "all=-N -l" -ldflags "${SERVER_LDFLAGS}" -o rekor-server_debug ./cmd/rekor-server
+RUN go test -c -ldflags "${SERVER_LDFLAGS}" -cover -covermode=count -coverpkg=./... -o rekor-server_test ./cmd/rekor-server
 
 # Multi-Stage production build
 FROM golang:1.19.1@sha256:2d17ffd12a2cdb25d4a633ad25f8dc29608ed84f31b3b983427d825280427095 as deploy
@@ -44,3 +45,7 @@ RUN go install github.com/go-delve/delve/cmd/dlv@v1.8.0
 
 # overwrite server and include debugger
 COPY --from=builder /opt/app-root/src/rekor-server_debug /usr/local/bin/rekor-server
+
+FROM deploy as test
+# overwrite server with test build with code coverage
+COPY --from=builder /opt/app-root/src/rekor-server_test /usr/local/bin/rekor-server

cmd/rekor-cli/main_test.go

@@ -0,0 +1,26 @@
+//
+// Copyright 2022 The Sigstore Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"testing"
+
+	"github.com/sigstore/rekor/cmd/rekor-cli/app"
+)
+
+func TestCover(t *testing.T) {
+	app.Execute()
+}

cmd/rekor-server/app/root.go

@@ -61,6 +61,8 @@ func init() {
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.rekor-server.yaml)")
 	rootCmd.PersistentFlags().StringVar(&logType, "log_type", "dev", "logger type to use (dev/prod)")
 	rootCmd.PersistentFlags().BoolVar(&enablePprof, "enable_pprof", false, "enable pprof for profiling on port 6060")
+	rootCmd.PersistentFlags().Bool("enable_killswitch", false, "enable killswitch for TESTING ONLY on port 2345")
+	_ = rootCmd.PersistentFlags().MarkHidden("enable_killswitch")
 
 	rootCmd.PersistentFlags().String("trillian_log_server.address", "127.0.0.1", "Trillian log server address")
 	rootCmd.PersistentFlags().Uint16("trillian_log_server.port", 8090, "Trillian log server port")

cmd/rekor-server/app/serve.go

@@ -123,6 +123,27 @@ var serveCmd = &cobra.Command{
 			_ = srv.ListenAndServe()
 		}()
 
+		if viper.GetBool("enable_killswitch") {
+			go func() {
+				mux := http.NewServeMux()
+				mux.Handle("/kill", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+					if err := server.Shutdown(); err != nil {
+						log.Logger.Error(err)
+					}
+					w.WriteHeader(http.StatusOK)
+				}))
+
+				srv := &http.Server{
+					Addr:         ":2345",
+					ReadTimeout:  10 * time.Second,
+					WriteTimeout: 10 * time.Second,
+					Handler:      mux,
+				}
+
+				_ = srv.ListenAndServe()
+			}()
+		}
+
 		if err := server.Serve(); err != nil {
 			log.Logger.Fatal(err)
 		}

cmd/rekor-server/main_test.go

@@ -0,0 +1,26 @@
+//
+// Copyright 2022 The Sigstore Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"testing"
+
+	"github.com/sigstore/rekor/cmd/rekor-server/app"
+)
+
+func TestCover(t *testing.T) {
+	app.Execute()
+}

docker-compose.test.yml

@@ -0,0 +1,39 @@
+#
+# Copyright 2022 The Sigstore Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: '3.4'
+services:
+  rekor-server:
+    build:
+      context: .
+      target: "test"
+    command: [
+      "rekor-server",
+      "-test.coverprofile=rekor-server.cov",
+      "serve",
+      "--trillian_log_server.address=trillian-log-server",
+      "--trillian_log_server.port=8090",
+      "--redis_server.address=redis-server",
+      "--redis_server.port=6379",
+      "--rekor_server.address=0.0.0.0",
+      "--rekor_server.signer=memory",
+      "--enable_attestation_storage",
+      "--attestation_storage_bucket=file:///var/run/attestations",
+      "--enable_killswitch",
+      ]
+    ports:
+      - "3000:3000"
+      - "2112:2112"
+      - "2345:2345"

tests/apk.go

@@ -1,3 +1,4 @@
+//go:build e2e
 // +build e2e
 
 //
@@ -51,7 +52,7 @@ func createSignedApk(t *testing.T, artifactPath string) {
 	datahash := sha256.Sum256(dataTGZBuf.Bytes())
 
 	ctlData := strings.Builder{}
-	ctlData.WriteString("name = " + randomRpmSuffix())
+	ctlData.WriteString("name = " + randomSuffix(16))
 	ctlData.WriteRune('\n')
 	ctlData.WriteString("datahash = " + hex.EncodeToString(datahash[:]))
 	ctlData.WriteRune('\n')

tests/e2e-test.sh

@@ -17,12 +17,14 @@
 set -e
 testdir=$(dirname "$0")
 
+rm -f /tmp/rekor-*.cov
+
 echo "starting services"
-docker-compose up -d
+docker-compose -f docker-compose.yml -f docker-compose.test.yml up -d --force-recreate --build
 
 echo "building CLI and server"
-go build -o rekor-cli ./cmd/rekor-cli
-go build -o rekor-server ./cmd/rekor-server
+go test -c ./cmd/rekor-cli -o rekor-cli -cover -covermode=count -coverpkg=./...
+go test -c ./cmd/rekor-server -o rekor-server -covermode=count -coverpkg=./...
 
 count=0
 
@@ -54,3 +56,18 @@ if docker-compose logs --no-color | grep -q "panic: runtime error:" ; then
    docker-compose logs --no-color > /tmp/docker-compose.log
    exit 1
 fi
+
+echo "generating code coverage"
+curl -X GET 0.0.0.0:2345/kill
+sleep 5
+
+if ! docker cp $(docker ps -aqf "name=rekor_rekor-server"):go/rekor-server.cov /tmp/rekor-server.cov ; then
+   # failed to copy code coverage report from server
+   echo "Failed to retrieve server code coverage report"
+   docker-compose logs --no-color > /tmp/docker-compose.log
+   exit 1
+fi
+
+# merging coverage reports and filtering out /pkg/generated from final report
+gocovmerge /tmp/rekor-*.cov | grep -v "/pkg/generated/" > /tmp/rekor-merged.cov
+echo "code coverage $(go tool cover -func=/tmp/rekor-merged.cov | grep -E '^total\:' | sed -E 's/\s+/ /g')"

tests/jar.go

@@ -13,6 +13,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build e2e
 // +build e2e
 
 package e2e
@@ -60,7 +61,7 @@ func createSignedJar(t *testing.T, artifactPath string) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	randManifest := strings.Replace(manifest, "REPLACE", randomRpmSuffix(), 1)
+	randManifest := strings.Replace(manifest, "REPLACE", randomSuffix(16), 1)
 	mf.Write([]byte(randManifest))
 	if err := zw.Close(); err != nil {
 		t.Fatal(err)

tests/rpm.go

@@ -1,3 +1,4 @@
+//go:build e2e
 // +build e2e
 
 //
@@ -20,28 +21,17 @@ package e2e
 import (
 	"bytes"
 	"io/ioutil"
-	"math/rand"
 	"os"
 	"testing"
 
 	"github.com/google/rpmpack"
 )
 
-func randomRpmSuffix() string {
-	const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
-
-	b := make([]byte, 16)
-	for i := range b {
-		b[i] = letterBytes[rand.Intn(len(letterBytes))]
-	}
-	return string(b)
-}
-
 func createSignedRpm(t *testing.T, artifactPath string) {
 	t.Helper()
 
 	rpmMetadata := rpmpack.RPMMetaData{
-		Name:    "test-rpm-" + randomRpmSuffix(),
+		Name:    "test-rpm-" + randomSuffix(16),
 		Epoch:   0,
 		Version: "1",
 		Release: "2",
@@ -57,7 +47,7 @@ func createSignedRpm(t *testing.T, artifactPath string) {
 	data := randomData(t, 100)
 
 	rpm.AddFile(rpmpack.RPMFile{
-		Name:  randomRpmSuffix(),
+		Name:  randomSuffix(16),
 		Body:  data,
 		Type:  rpmpack.GenericFile,
 		Owner: "testOwner",

tests/util.go

@@ -47,6 +47,9 @@ func outputContains(t *testing.T, output, sub string) {
 
 func run(t *testing.T, stdin, cmd string, arg ...string) string {
 	t.Helper()
+	// Coverage flag must be the first arg passed to coverage binary
+	// No impact when running with regular binary
+	arg = append([]string{coverageFlag()}, arg...)
 	c := exec.Command(cmd, arg...)
 	if stdin != "" {
 		c.Stdin = strings.NewReader(stdin)
@@ -60,8 +63,7 @@ func run(t *testing.T, stdin, cmd string, arg ...string) string {
 		t.Log(string(b))
 		t.Fatal(err)
 	}
-
-	return string(b)
+	return stripCoverageOutput(string(b))
 }
 
 func runCli(t *testing.T, arg ...string) string {
@@ -76,6 +78,9 @@ func runCli(t *testing.T, arg ...string) string {
 
 func runCliStdout(t *testing.T, arg ...string) string {
 	t.Helper()
+	// Coverage flag must be the first arg passed to coverage binary
+	// No impact when running with regular binary
+	arg = append([]string{coverageFlag()}, arg...)
 	arg = append(arg, rekorServerFlag())
 	c := exec.Command(cli, arg...)
 
@@ -88,11 +93,14 @@ func runCliStdout(t *testing.T, arg ...string) string {
 		t.Log(string(b))
 		t.Fatal(err)
 	}
-	return string(b)
+	return stripCoverageOutput(string(b))
 }
 
 func runCliErr(t *testing.T, arg ...string) string {
 	t.Helper()
+	// Coverage flag must be the first arg passed to coverage binary
+	// No impact when running with regular binary
+	arg = append([]string{coverageFlag()}, arg...)
 	arg = append(arg, rekorServerFlag())
 	// use a blank config file to ensure no collision
 	if os.Getenv("REKORTMPDIR") != "" {
@@ -104,7 +112,7 @@ func runCliErr(t *testing.T, arg ...string) string {
 		t.Log(string(b))
 		t.Fatalf("expected error, got %s", string(b))
 	}
-	return string(b)
+	return stripCoverageOutput(string(b))
 }
 
 func rekorServerFlag() string {
@@ -118,6 +126,14 @@ func rekorServer() string {
 	return "http://localhost:3000"
 }
 
+func coverageFlag() string {
+	return "-test.coverprofile=/tmp/rekor-cli."+randomSuffix(8)+".cov"
+}
+
+func stripCoverageOutput(out string) string {
+	return strings.Split(strings.Split(out, "PASS")[0], "FAIL")[0]
+}
+
 func readFile(t *testing.T, p string) string {
 	b, err := ioutil.ReadFile(p)
 	if err != nil {
@@ -126,6 +142,16 @@ func readFile(t *testing.T, p string) string {
 	return strings.TrimSpace(string(b))
 }
 
+func randomSuffix(n int) string {
+	const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = letterBytes[rand.Intn(len(letterBytes))]
+	}
+	return string(b)
+}
+
 func randomData(t *testing.T, n int) []byte {
 	t.Helper()
 	rand.Seed(time.Now().UnixNano())