Abstract upstream package before matching (#607)

* add metadata extraction from pURLs

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* extract upstream packages before matching

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* put pkg.UpstreamPackages under test

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* remove pURL related processing

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* pull in syft spdx decoding

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* allow for more flexible GHSA namespace and source extraction

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add matching parity integration tests for all supported formats

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* bump syft to get spdx tv fix

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

Makefile

@@ -16,6 +16,8 @@ SUCCESS := $(BOLD)$(GREEN)
 # the quality gate lower threshold for unit test total % coverage (by function statements)
 COVERAGE_THRESHOLD := 47
 BOOTSTRAP_CACHE="c7afb99ad"
+INTEGRATION_CACHE_BUSTER="894d8ca"
+
 
 ## Build variables
 DISTDIR=./dist
@@ -152,7 +154,7 @@ integration: ## Run integration tests
 # note: this is used by CI to determine if the integration test fixture cache (docker image tars) should be busted
 .PHONY: integration-fingerprint
 integration-fingerprint:
-	find test/integration/*.go test/integration/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee test/integration/test-fixtures/cache.fingerprint
+	find test/integration/*.go test/integration/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | tee /dev/stderr | md5sum | tee test/integration/test-fixtures/cache.fingerprint && echo "$(INTEGRATION_CACHE_BUSTER)" >> test/integration/test-fixtures/cache.fingerprint
 
 # note: this is used by CI to determine if the cli test fixture cache (docker image tars) should be busted
 .PHONY: cli-fingerprint

go.mod

@@ -9,8 +9,9 @@ require (
 	github.com/alicebob/sqlittle v1.4.0
 	github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04
 	github.com/anchore/go-version v1.2.2-0.20210903204242-51efa5b487c4
+	github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29
 	github.com/anchore/stereoscope v0.0.0-20220209180455-403dd709a3fb
-	github.com/anchore/syft v0.37.11-0.20220209193326-5ab872c73281
+	github.com/anchore/syft v0.37.11-0.20220210211800-220f3a24fdf5
 	github.com/bmatcuk/doublestar/v2 v2.0.4
 	github.com/docker/docker v20.10.12+incompatible
 	github.com/dustin/go-humanize v1.0.0
@@ -25,7 +26,6 @@ require (
 	github.com/hashicorp/go-multierror v1.1.0
 	github.com/hashicorp/go-version v1.2.0
 	github.com/huandu/xstrings v1.3.2 // indirect
-	github.com/jinzhu/copier v0.3.2
 	github.com/jinzhu/gorm v1.9.14
 	github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f
 	github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d

go.sum

@@ -122,10 +122,11 @@ github.com/anchore/go-version v1.2.2-0.20210903204242-51efa5b487c4/go.mod h1:Bkc
 github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29 h1:K9LfnxwhqvihqU0+MF325FNy7fsKV9EGaUxdfR4gnWk=
 github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29/go.mod h1:Oc1UkGaJwY6ND6vtAqPSlYrptKRJngHwkwB6W7l1uP0=
 github.com/anchore/stereoscope v0.0.0-20220209160132-2e595043fa19/go.mod h1:QpDHHV2h1NNfu7klzU75XC8RvSlaPK6HHgi0dy8A6sk=
+github.com/anchore/stereoscope v0.0.0-20220209160132-2e595043fa19/go.mod h1:QpDHHV2h1NNfu7klzU75XC8RvSlaPK6HHgi0dy8A6sk=
 github.com/anchore/stereoscope v0.0.0-20220209180455-403dd709a3fb h1:yicFaC7dVBS4uYvU7sxsnEVi/2rndM0axZUgfhx+1qs=
 github.com/anchore/stereoscope v0.0.0-20220209180455-403dd709a3fb/go.mod h1:QpDHHV2h1NNfu7klzU75XC8RvSlaPK6HHgi0dy8A6sk=
-github.com/anchore/syft v0.37.11-0.20220209193326-5ab872c73281 h1:QWRCTTPfLHa6ks9gp3nh5/mG0PrC4X6xPoX2vdFDzGA=
-github.com/anchore/syft v0.37.11-0.20220209193326-5ab872c73281/go.mod h1:vjP8jxwgvL91DxhkoEH8GgEIUCumuPOuZuS/DWeYy0s=
+github.com/anchore/syft v0.37.11-0.20220210211800-220f3a24fdf5 h1:GLShI62a8Y5pW+SIWnwsoXC4szWIj98rzwzEurKem84=
+github.com/anchore/syft v0.37.11-0.20220210211800-220f3a24fdf5/go.mod h1:vjP8jxwgvL91DxhkoEH8GgEIUCumuPOuZuS/DWeYy0s=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=

grype/db/v3/namespace.go

@@ -7,6 +7,8 @@ import (
 	"github.com/anchore/grype/grype/distro"
 	"github.com/anchore/grype/grype/pkg"
 	"github.com/anchore/grype/internal"
+	"github.com/anchore/grype/internal/log"
+	"github.com/anchore/packageurl-go"
 	syftPkg "github.com/anchore/syft/syft/pkg"
 )
 
@@ -116,5 +118,14 @@ func githubJavaPackageNamer(p pkg.Package) []string {
 		}
 	}
 
+	if p.PURL != "" {
+		purl, err := packageurl.FromString(p.PURL)
+		if err != nil {
+			log.Warnf("unable to extract GHSA java package information from purl=%q: %+v", p.PURL, err)
+		} else {
+			names.Add(fmt.Sprintf("%s:%s", purl.Namespace, purl.Name))
+		}
+	}
+
 	return names.ToSlice()
 }

grype/db/v3/namespace_test.go

@@ -405,6 +405,31 @@ func Test_githubJavaPackageNamer(t *testing.T) {
 			},
 			expected: []string{},
 		},
+		{
+			name: "with valid purl",
+			namerInput: pkg.Package{
+				ID:   pkg.ID(uuid.NewString()),
+				Name: "a-name",
+				PURL: "pkg:maven/org.anchore/b-name@0.2",
+			},
+			expected: []string{"org.anchore:b-name"},
+		},
+		{
+			name: "ignore invalid pURLs",
+			namerInput: pkg.Package{
+				ID:   pkg.ID(uuid.NewString()),
+				Name: "a-name",
+				PURL: "pkg:BAD/",
+				Metadata: pkg.JavaMetadata{
+					VirtualPath:   "v-path",
+					PomArtifactID: "art-id",
+					PomGroupID:    "g-id",
+				},
+			},
+			expected: []string{
+				"g-id:art-id",
+			},
+		},
 	}
 
 	for _, test := range tests {

grype/matcher/apk/matcher.go

@@ -1,9 +1,7 @@
 package apk
 
 import (
-	"errors"
 	"fmt"
-	"strings"
 
 	"github.com/anchore/grype/grype/distro"
 	"github.com/anchore/grype/grype/match"
@@ -12,8 +10,6 @@ import (
 	"github.com/anchore/grype/grype/version"
 	"github.com/anchore/grype/grype/vulnerability"
 	syftPkg "github.com/anchore/syft/syft/pkg"
-	"github.com/jinzhu/copier"
-	"github.com/scylladb/go-set/strset"
 )
 
 type Matcher struct {
@@ -161,19 +157,14 @@ func (m *Matcher) findApkPackage(store vulnerability.Provider, d *distro.Distro,
 }
 
 func (m *Matcher) matchBySourceIndirection(store vulnerability.Provider, d *distro.Distro, p pkg.Package) ([]match.Match, error) {
-	// build indirect package for matching against source package
-	indirectPackage, err := buildIndirectPackage(p)
-	if err != nil {
-		// If the err is that there no indirect package return empty slice
-		if errors.Is(err, errNoIndirectPackage) {
-			return nil, nil
-		}
-		return nil, fmt.Errorf("failed to build an indirect package for: %s", p.Name)
-	}
+	var matches []match.Match
 
-	matches, err := m.findApkPackage(store, d, indirectPackage)
-	if err != nil {
-		return nil, fmt.Errorf("failed to find vulnerabilities by apk source indirection: %w", err)
+	for _, indirectPackage := range pkg.UpstreamPackages(p) {
+		indirectMatches, err := m.findApkPackage(store, d, indirectPackage)
+		if err != nil {
+			return nil, fmt.Errorf("failed to find vulnerabilities for apk upstream source package: %w", err)
+		}
+		matches = append(matches, indirectMatches...)
 	}
 
 	// we want to make certain that we are tracking the match based on the package from the SBOM (not the indirect package)
@@ -182,40 +173,3 @@ func (m *Matcher) matchBySourceIndirection(store vulnerability.Provider, d *dist
 
 	return matches, nil
 }
-
-// Custom error for when indirect package is not present or is identical to package
-var errNoIndirectPackage = errors.New("source package is either identical to pkg or not present")
-
-func buildIndirectPackage(p pkg.Package) (pkg.Package, error) {
-	metadata, ok := p.Metadata.(pkg.ApkMetadata)
-	// ignore packages without source indirection hints or where source name is identical to package name
-	if !ok || metadata.OriginPackage == "" || metadata.OriginPackage == p.Name {
-		return pkg.Package{}, errNoIndirectPackage
-	}
-
-	var indirectPackage pkg.Package
-	err := copier.Copy(&indirectPackage, p)
-	if err != nil {
-		return pkg.Package{}, fmt.Errorf("failed to copy package: %w", err)
-	}
-
-	// use the source package name
-	indirectPackage.Name = metadata.OriginPackage
-
-	// For each cpe, replace pkg name with origin and add to set
-	cpeStrings := strset.New()
-	for _, cpe := range indirectPackage.CPEs {
-		updatedCPEString := strings.ReplaceAll(cpe.BindToFmtString(), p.Name, indirectPackage.Name)
-		cpeStrings.Add(updatedCPEString)
-	}
-
-	// With each entry in set, convert string to CPE and update indirectPackage CPEs
-	var updatedCPEs []syftPkg.CPE
-	for _, cpeString := range cpeStrings.List() {
-		updatedCPE, _ := syftPkg.NewCPE(cpeString)
-		updatedCPEs = append(updatedCPEs, updatedCPE)
-	}
-	indirectPackage.CPEs = updatedCPEs
-
-	return indirectPackage, nil
-}

grype/matcher/apk/matcher_test.go

@@ -487,11 +487,15 @@ func TestDistroMatchBySourceIndirection(t *testing.T) {
 		t.Fatalf("failed to create a new distro: %+v", err)
 	}
 	p := pkg.Package{
-		ID:       pkg.ID(uuid.NewString()),
-		Name:     "musl-utils",
-		Version:  "1.3.2-r0",
-		Type:     syftPkg.ApkPkg,
-		Metadata: pkg.ApkMetadata{OriginPackage: "musl"},
+		ID:      pkg.ID(uuid.NewString()),
+		Name:    "musl-utils",
+		Version: "1.3.2-r0",
+		Type:    syftPkg.ApkPkg,
+		Upstreams: []pkg.UpstreamPackage{
+			{
+				Name: "musl",
+			},
+		},
 	}
 
 	vulnFound, err := vulnerability.NewVulnerability(secDbVuln)
@@ -567,7 +571,11 @@ func TestNVDMatchBySourceIndirection(t *testing.T) {
 			must(syftPkg.NewCPE("cpe:2.3:a:musl-utils:musl-utils:*:*:*:*:*:*:*:*")),
 			must(syftPkg.NewCPE("cpe:2.3:a:musl-utils:musl-utils:*:*:*:*:*:*:*:*")),
 		},
-		Metadata: pkg.ApkMetadata{OriginPackage: "musl"},
+		Upstreams: []pkg.UpstreamPackage{
+			{
+				Name: "musl",
+			},
+		},
 	}
 
 	vulnFound, err := vulnerability.NewVulnerability(nvdVuln)

grype/matcher/dpkg/matcher.go

@@ -9,7 +9,6 @@ import (
 	"github.com/anchore/grype/grype/search"
 	"github.com/anchore/grype/grype/vulnerability"
 	syftPkg "github.com/anchore/syft/syft/pkg"
-	"github.com/jinzhu/copier"
 )
 
 type Matcher struct {
@@ -26,7 +25,7 @@ func (m *Matcher) Type() match.MatcherType {
 func (m *Matcher) Match(store vulnerability.Provider, d *distro.Distro, p pkg.Package) ([]match.Match, error) {
 	matches := make([]match.Match, 0)
 
-	sourceMatches, err := m.matchBySourceIndirection(store, d, p)
+	sourceMatches, err := m.matchUpstreamPackages(store, d, p)
 	if err != nil {
 		return nil, fmt.Errorf("failed to match by source indirection: %w", err)
 	}
@@ -41,31 +40,15 @@ func (m *Matcher) Match(store vulnerability.Provider, d *distro.Distro, p pkg.Pa
 	return matches, nil
 }
 
-func (m *Matcher) matchBySourceIndirection(store vulnerability.ProviderByDistro, d *distro.Distro, p pkg.Package) ([]match.Match, error) {
-	metadata, ok := p.Metadata.(pkg.DpkgMetadata)
-	if !ok {
-		return nil, nil
-	}
-
-	// ignore packages without source indirection hints
-	if metadata.Source == "" {
-		return []match.Match{}, nil
-	}
+func (m *Matcher) matchUpstreamPackages(store vulnerability.ProviderByDistro, d *distro.Distro, p pkg.Package) ([]match.Match, error) {
+	var matches []match.Match
 
-	// use source package name for exact package name matching
-	var indirectPackage pkg.Package
-
-	err := copier.Copy(&indirectPackage, p)
-	if err != nil {
-		return nil, fmt.Errorf("failed to copy package: %w", err)
-	}
-
-	// use the source package name
-	indirectPackage.Name = metadata.Source
-
-	matches, err := search.ByPackageDistro(store, d, indirectPackage, m.Type())
-	if err != nil {
-		return nil, fmt.Errorf("failed to find vulnerabilities by dpkg source indirection: %w", err)
+	for _, indirectPackage := range pkg.UpstreamPackages(p) {
+		indirectMatches, err := search.ByPackageDistro(store, d, indirectPackage, m.Type())
+		if err != nil {
+			return nil, fmt.Errorf("failed to find vulnerabilities for dpkg upstream source package: %w", err)
+		}
+		matches = append(matches, indirectMatches...)
 	}
 
 	// we want to make certain that we are tracking the match based on the package from the SBOM (not the indirect package)

grype/matcher/dpkg/matcher_test.go

@@ -22,8 +22,10 @@ func TestMatcherDpkg_matchBySourceIndirection(t *testing.T) {
 		Name:    "neutron",
 		Version: "2014.1.3-6",
 		Type:    syftPkg.DebPkg,
-		Metadata: pkg.DpkgMetadata{
-			Source: "neutron-devel",
+		Upstreams: []pkg.UpstreamPackage{
+			{
+				Name: "neutron-devel",
+			},
 		},
 	}
 
@@ -33,7 +35,7 @@ func TestMatcherDpkg_matchBySourceIndirection(t *testing.T) {
 	}
 
 	store := newMockProvider()
-	actual, err := matcher.matchBySourceIndirection(store, d, p)
+	actual, err := matcher.matchUpstreamPackages(store, d, p)
 
 	assert.Len(t, actual, 2, "unexpected indirect matches count")