add matching parity integration tests for all supported formats

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

Makefile

@@ -16,6 +16,8 @@ SUCCESS := $(BOLD)$(GREEN)
 # the quality gate lower threshold for unit test total % coverage (by function statements)
 COVERAGE_THRESHOLD := 47
 BOOTSTRAP_CACHE="c7afb99ad"
+INTEGRATION_CACHE_BUSTER="894d8ca"
+
 
 ## Build variables
 DISTDIR=./dist
@@ -152,7 +154,7 @@ integration: ## Run integration tests
 # note: this is used by CI to determine if the integration test fixture cache (docker image tars) should be busted
 .PHONY: integration-fingerprint
 integration-fingerprint:
-	find test/integration/*.go test/integration/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee test/integration/test-fixtures/cache.fingerprint
+	find test/integration/*.go test/integration/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | tee /dev/stderr | md5sum | tee test/integration/test-fixtures/cache.fingerprint && echo "$(INTEGRATION_CACHE_BUSTER)" >> test/integration/test-fixtures/cache.fingerprint
 
 # note: this is used by CI to determine if the cli test fixture cache (docker image tars) should be busted
 .PHONY: cli-fingerprint

go.mod

@@ -9,8 +9,9 @@ require (
 	github.com/alicebob/sqlittle v1.4.0
 	github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04
 	github.com/anchore/go-version v1.2.2-0.20210903204242-51efa5b487c4
-	github.com/anchore/stereoscope v0.0.0-20220201190559-f162f1e96f45
-	github.com/anchore/syft v0.37.11-0.20220209191120-76f8205936b2
+	github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29
+	github.com/anchore/stereoscope v0.0.0-20220209180455-403dd709a3fb
+	github.com/anchore/syft v0.37.11-0.20220210184638-ca032434b39f
 	github.com/bmatcuk/doublestar/v2 v2.0.4
 	github.com/docker/docker v20.10.12+incompatible
 	github.com/dustin/go-humanize v1.0.0

go.sum

@@ -111,6 +111,7 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
 github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=
 github.com/alicebob/sqlittle v1.4.0 h1:vgYt0nAjhdf/hg52MjKJ84g/uTzBPfrvI+VUBrIghxA=
 github.com/alicebob/sqlittle v1.4.0/go.mod h1:Co1L1qxHqCwf41puWhk2HOodojR0mcsAV4BIt8byZh8=
+github.com/anchore/client-go v0.0.0-20210222170800-9c70f9b80bcf h1:DYssiUV1pBmKqzKsm4mqXx8artqC0Q8HgZsVI3lMsAg=
 github.com/anchore/client-go v0.0.0-20210222170800-9c70f9b80bcf/go.mod h1:FaODhIA06mxO1E6R32JE0TL1JWZZkmjRIAd4ULvHUKk=
 github.com/anchore/go-rpmdb v0.0.0-20210914181456-a9c52348da63 h1:C9W/LAydEz/qdUhx1MdjO9l8NEcFKYknkxDVyo9LAoM=
 github.com/anchore/go-rpmdb v0.0.0-20210914181456-a9c52348da63/go.mod h1:6qH8c6U/3CBVvDDDBZnPSTbTINq3cIdADUYTaVf75EM=
@@ -121,25 +122,20 @@ github.com/anchore/go-version v1.2.2-0.20210903204242-51efa5b487c4 h1:rmZG77uXgE
 github.com/anchore/go-version v1.2.2-0.20210903204242-51efa5b487c4/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
 github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29 h1:K9LfnxwhqvihqU0+MF325FNy7fsKV9EGaUxdfR4gnWk=
 github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29/go.mod h1:Oc1UkGaJwY6ND6vtAqPSlYrptKRJngHwkwB6W7l1uP0=
+github.com/anchore/stereoscope v0.0.0-20220209160132-2e595043fa19 h1:INJWzjqSo4uF5NrYISnIfIpnmgV+nfYwbrL8nnmIz7g=
+github.com/anchore/stereoscope v0.0.0-20220209160132-2e595043fa19/go.mod h1:QpDHHV2h1NNfu7klzU75XC8RvSlaPK6HHgi0dy8A6sk=
 github.com/anchore/stereoscope v0.0.0-20220209160132-2e595043fa19/go.mod h1:QpDHHV2h1NNfu7klzU75XC8RvSlaPK6HHgi0dy8A6sk=
 github.com/anchore/stereoscope v0.0.0-20220209180455-403dd709a3fb h1:yicFaC7dVBS4uYvU7sxsnEVi/2rndM0axZUgfhx+1qs=
 github.com/anchore/stereoscope v0.0.0-20220209180455-403dd709a3fb/go.mod h1:QpDHHV2h1NNfu7klzU75XC8RvSlaPK6HHgi0dy8A6sk=
-github.com/anchore/syft v0.37.11-0.20220209193326-5ab872c73281 h1:QWRCTTPfLHa6ks9gp3nh5/mG0PrC4X6xPoX2vdFDzGA=
-github.com/anchore/syft v0.37.11-0.20220209193326-5ab872c73281/go.mod h1:vjP8jxwgvL91DxhkoEH8GgEIUCumuPOuZuS/DWeYy0s=
-github.com/anchore/stereoscope v0.0.0-20220110181730-c91cf94a3718 h1:46+DtmTaPlOCuY5KY3H6zazuz3+E/DSwc+ZpfPhyj50=
-github.com/anchore/stereoscope v0.0.0-20220110181730-c91cf94a3718/go.mod h1:OHhT0g7HQlELWJgZE80dJ0rCbMPIR+jIM8KNwN7ReKU=
-github.com/anchore/syft v0.36.1-0.20220126161937-9f7104d4f194 h1:WlwDAT8AC3RDMpLvNgTOAnmFSZy/GDk/G0CAYZqKw+A=
-github.com/anchore/syft v0.36.1-0.20220126161937-9f7104d4f194/go.mod h1:qox7ntCZuKQ8mtHQoG40Waccjp2gZNJ6bYDrwgAkKp4=
-github.com/anchore/stereoscope v0.0.0-20220201190559-f162f1e96f45 h1:GYwI1qXcGh7fmyUWLK41suUXYfmrvGWWKGh0cw+k6ug=
-github.com/anchore/stereoscope v0.0.0-20220201190559-f162f1e96f45/go.mod h1:QpDHHV2h1NNfu7klzU75XC8RvSlaPK6HHgi0dy8A6sk=
-github.com/anchore/syft v0.37.11-0.20220209191120-76f8205936b2 h1:iqXZ72nqUf293HBxOhSn7JIhmCvoYMHbOtxdq3XuG3w=
-github.com/anchore/syft v0.37.11-0.20220209191120-76f8205936b2/go.mod h1:v45oVrOTnoSyJdmBRfQwl6Sf/HGqK2xCq7CrQd2ew5w=
+github.com/anchore/syft v0.37.11-0.20220210184638-ca032434b39f h1:l778WhJp0kKYxG8D9g9n5NkJBsT9qQNlQj1tucchWZQ=
+github.com/anchore/syft v0.37.11-0.20220210184638-ca032434b39f/go.mod h1:vjP8jxwgvL91DxhkoEH8GgEIUCumuPOuZuS/DWeYy0s=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
 github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
 github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=

test/cli/trait_assertions_test.go

@@ -29,6 +29,6 @@ func assertFailingReturnCode(tb testing.TB, _, _ string, rc int) {
 func assertSucceedingReturnCode(tb testing.TB, _, _ string, rc int) {
 	tb.Helper()
 	if rc != 0 {
-		tb.Errorf("expected a failure but got rc=%d", rc)
+		tb.Errorf("expected to succeed but got rc=%d", rc)
 	}
 }

test/integration/compare_sbom_input_vs_lib_test.go

@@ -2,6 +2,7 @@ package integration
 
 import (
 	"fmt"
+	"github.com/anchore/syft/syft/format"
 	"os"
 	"testing"
 
@@ -15,15 +16,20 @@ import (
 	"github.com/scylladb/go-set/strset"
 )
 
+var imagesWithVulnerabilities = []string{
+	"anchore/test_images:vulnerabilities-alpine",
+	"anchore/test_images:gems",
+	"anchore/test_images:vulnerabilities-debian",
+	"anchore/test_images:vulnerabilities-centos",
+	"anchore/test_images:npm",
+	"anchore/test_images:java",
+	"anchore/test_images:golang-56d52bc",
+}
+
 func TestCompareSBOMInputToLibResults(t *testing.T) {
-	images := []string{
-		"anchore/test_images:vulnerabilities-alpine",
-		"anchore/test_images:gems",
-		"anchore/test_images:vulnerabilities-debian",
-		"anchore/test_images:vulnerabilities-centos",
-		"anchore/test_images:npm",
-		"anchore/test_images:java",
-		"anchore/test_images:golang-56d52bc",
+	formats := []format.Option{
+		format.SPDXJSONOption,
+		format.SPDXTagValueOption,
 	}
 
 	// get a grype DB
@@ -43,46 +49,50 @@ func TestCompareSBOMInputToLibResults(t *testing.T) {
 		string(syftPkg.RustPkg),
 		string(syftPkg.KbPkg),
 		string(syftPkg.PhpComposerPkg),
+		string(syftPkg.JenkinsPluginPkg), // package type cannot be inferred for all formats
 	)
 	observedPkgTypes := strset.New()
 
-	for _, image := range images {
-		t.Run(image, func(t *testing.T) {
-			imageArchive := PullThroughImageCache(t, image)
-			imageSource := fmt.Sprintf("docker-archive:%s", imageArchive)
-
-			// get SBOM from syft, write to temp file
-			sbomBytes := getSyftSBOM(t, imageSource)
-			sbomFile, err := os.CreateTemp("", "")
-			assert.NoError(t, err)
-			t.Cleanup(func() {
-				assert.NoError(t, os.Remove(sbomFile.Name()))
-			})
-			_, err = sbomFile.WriteString(sbomBytes)
-			assert.NoError(t, err)
-			assert.NoError(t, sbomFile.Close())
+	for _, image := range imagesWithVulnerabilities {
+		imageArchive := PullThroughImageCache(t, image)
+		imageSource := fmt.Sprintf("docker-archive:%s", imageArchive)
 
-			// get vulns (sbom)
-			matchesFromSbom, _, pkgsFromSbom, err := grype.FindVulnerabilities(vulnProvider, fmt.Sprintf("sbom:%s", sbomFile.Name()), source.SquashedScope, nil)
-			assert.NoError(t, err)
+		for _, f := range formats {
+			t.Run(fmt.Sprintf("%s/%s", image, f), func(t *testing.T) {
 
-			// get vulns (image)
-			matchesFromImage, _, _, err := grype.FindVulnerabilities(vulnProvider, imageSource, source.SquashedScope, nil)
-			assert.NoError(t, err)
+				// get SBOM from syft, write to temp file
+				sbomBytes := getSyftSBOM(t, imageSource, f)
+				sbomFile, err := os.CreateTemp("", "")
+				assert.NoError(t, err)
+				t.Cleanup(func() {
+					assert.NoError(t, os.Remove(sbomFile.Name()))
+				})
+				_, err = sbomFile.WriteString(sbomBytes)
+				assert.NoError(t, err)
+				assert.NoError(t, sbomFile.Close())
 
-			// compare packages (shallow)
-			matchSetFromSbom := getMatchSet(matchesFromSbom)
-			matchSetFromImage := getMatchSet(matchesFromImage)
+				// get vulns (sbom)
+				matchesFromSbom, _, pkgsFromSbom, err := grype.FindVulnerabilities(vulnProvider, fmt.Sprintf("sbom:%s", sbomFile.Name()), source.SquashedScope, nil)
+				assert.NoError(t, err)
 
-			assert.Empty(t, strset.Difference(matchSetFromSbom, matchSetFromImage).List(), "vulnerabilities present only in results when using sbom as input")
-			assert.Empty(t, strset.Difference(matchSetFromImage, matchSetFromSbom).List(), "vulnerabilities present only in results when using image as input")
+				// get vulns (image)
+				matchesFromImage, _, _, err := grype.FindVulnerabilities(vulnProvider, imageSource, source.SquashedScope, nil)
+				assert.NoError(t, err)
 
-			// track all covered package types (for use after the test)
-			for _, p := range pkgsFromSbom {
-				observedPkgTypes.Add(string(p.Type))
-			}
+				// compare packages (shallow)
+				matchSetFromSbom := getMatchSet(matchesFromSbom)
+				matchSetFromImage := getMatchSet(matchesFromImage)
 
-		})
+				assert.Empty(t, strset.Difference(matchSetFromSbom, matchSetFromImage).List(), "vulnerabilities present only in results when using sbom as input")
+				assert.Empty(t, strset.Difference(matchSetFromImage, matchSetFromSbom).List(), "vulnerabilities present only in results when using image as input")
+
+				// track all covered package types (for use after the test)
+				for _, p := range pkgsFromSbom {
+					observedPkgTypes.Add(string(p.Type))
+				}
+
+			})
+		}
 	}
 
 	// ensure we've covered all package types (-rust, -kb)

test/integration/utils_test.go

@@ -61,7 +61,7 @@ func saveImage(t testing.TB, imageName string, destPath string) {
 	t.Logf("Stdout: %s\n", out)
 }
 
-func getSyftSBOM(t testing.TB, image string) string {
+func getSyftSBOM(t testing.TB, image string, formatOption format.Option) string {
 	src, cleanup, err := source.New(image, nil, nil)
 	if err != nil {
 		t.Fatalf("can't get the source: %+v", err)
@@ -81,7 +81,7 @@ func getSyftSBOM(t testing.TB, image string) string {
 		Source: src.Metadata,
 	}
 
-	bytes, err := syft.Encode(sbom, format.JSONOption)
+	bytes, err := syft.Encode(sbom, formatOption)
 	if err != nil {
 		t.Fatalf("presenter failed: %+v", err)
 	}
@@ -92,7 +92,7 @@ func getSyftSBOM(t testing.TB, image string) string {
 func getMatchSet(matches match.Matches) *strset.Set {
 	s := strset.New()
 	for _, m := range matches.Sorted() {
-		s.Add(fmt.Sprintf("%s-%s-%s-%s", m.Vulnerability.ID, m.Package.Name, m.Package.Version, string(m.Package.Type)))
+		s.Add(fmt.Sprintf("%s-%s-%s", m.Vulnerability.ID, m.Package.Name, m.Package.Version))
 	}
 	return s
 }