From ca032434b39ff106c5643bb5a84301bc7297a603 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 10 Feb 2022 13:46:38 -0500 Subject: [PATCH 1/6] Add pURL generation for java packages + fix NPM pURL generation (#812) * enhance pURL generation for java packages Signed-off-by: Alex Goodman * optionally split out npm namespaces for pURL generation Signed-off-by: Alex Goodman * nit updates Signed-off-by: Alex Goodman --- syft/pkg/cataloger/common/cpe/java.go | 6 +-- syft/pkg/cataloger/common/cpe/java_test.go | 2 +- syft/pkg/cataloger/java/archive_filename.go | 17 +------ syft/pkg/cataloger/java/archive_parser.go | 25 ++++++++++- .../pkg/cataloger/java/archive_parser_test.go | 4 ++ syft/pkg/cataloger/java/package_url.go | 25 +++++++++++ syft/pkg/cataloger/java/package_url_test.go | 45 +++++++++++++++++++ syft/pkg/java_metadata.go | 21 ++------- syft/pkg/java_metadata_test.go | 36 --------------- syft/pkg/language.go | 18 +++++--- syft/pkg/npm_package_json_metadata.go | 15 ++++++- syft/pkg/npm_package_json_metadata_test.go | 42 +++++++++++++++++ syft/pkg/url_test.go | 18 ++------ 13 files changed, 176 insertions(+), 98 deletions(-) create mode 100644 syft/pkg/cataloger/java/package_url.go create mode 100644 syft/pkg/cataloger/java/package_url_test.go create mode 100644 syft/pkg/npm_package_json_metadata_test.go diff --git a/syft/pkg/cataloger/common/cpe/java.go b/syft/pkg/cataloger/common/cpe/java.go index 1ca5d8f19de..be86549e7e5 100644 --- a/syft/pkg/cataloger/common/cpe/java.go +++ b/syft/pkg/cataloger/common/cpe/java.go @@ -40,11 +40,11 @@ var ( ) func candidateProductsForJava(p pkg.Package) []string { - return productsFromArtifactAndGroupIDs(artifactIDFromJavaPackage(p), groupIDsFromJavaPackage(p)) + return productsFromArtifactAndGroupIDs(artifactIDFromJavaPackage(p), GroupIDsFromJavaPackage(p)) } func candidateVendorsForJava(p pkg.Package) fieldCandidateSet { - gidVendors := vendorsFromGroupIDs(groupIDsFromJavaPackage(p)) + gidVendors := vendorsFromGroupIDs(GroupIDsFromJavaPackage(p)) nameVendors := vendorsFromJavaManifestNames(p) return newFieldCandidateSetFromSets(gidVendors, nameVendors) } @@ -173,7 +173,7 @@ func artifactIDFromJavaPackage(p pkg.Package) string { return artifactID } -func groupIDsFromJavaPackage(p pkg.Package) (groupIDs []string) { +func GroupIDsFromJavaPackage(p pkg.Package) (groupIDs []string) { metadata, ok := p.Metadata.(pkg.JavaMetadata) if !ok { return nil diff --git a/syft/pkg/cataloger/common/cpe/java_test.go b/syft/pkg/cataloger/common/cpe/java_test.go index a830f3c1de8..0f87e5b56fe 100644 --- a/syft/pkg/cataloger/common/cpe/java_test.go +++ b/syft/pkg/cataloger/common/cpe/java_test.go @@ -333,7 +333,7 @@ func Test_groupIDsFromJavaPackage(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - assert.ElementsMatch(t, test.expects, groupIDsFromJavaPackage(test.pkg)) + assert.ElementsMatch(t, test.expects, GroupIDsFromJavaPackage(test.pkg)) }) } } diff --git a/syft/pkg/cataloger/java/archive_filename.go b/syft/pkg/cataloger/java/archive_filename.go index 625f5bf736a..870e4ec7deb 100644 --- a/syft/pkg/cataloger/java/archive_filename.go +++ b/syft/pkg/cataloger/java/archive_filename.go @@ -55,28 +55,13 @@ type archiveFilename struct { version string } -// TODO: Remove this method once we're using Go 1.15+. -// -// Go 1.15 introduces a `SubexpIndex` method for the Regexp type that would let -// this code be made more elegant. Once we've reached 1.15, we should eliminate -// this function in favor of that method. -func subexpIndex(re *regexp.Regexp, name string) int { - for i, subexpName := range re.SubexpNames() { - if subexpName == name { - return i - } - } - - return -1 -} - func getSubexp(matches []string, subexpName string, re *regexp.Regexp, raw string) string { if len(matches) < 1 { log.Warnf("unexpectedly empty matches for archive '%s'", raw) return "" } - index := subexpIndex(re, subexpName) + index := re.SubexpIndex(subexpName) if index < 1 { log.Warnf("unexpected index of '%s' capture group for Java archive '%s'", subexpName, raw) return "" diff --git a/syft/pkg/cataloger/java/archive_parser.go b/syft/pkg/cataloger/java/archive_parser.go index 75dd3368356..b96f8f1827d 100644 --- a/syft/pkg/cataloger/java/archive_parser.go +++ b/syft/pkg/cataloger/java/archive_parser.go @@ -122,6 +122,13 @@ func (j *archiveParser) parse() ([]*pkg.Package, []artifact.Relationship, error) pkgs = append([]*pkg.Package{parentPkg}, pkgs...) } + // add pURLs to all packages found + // note: since package information may change after initial creation when parsing multiple locations within the + // jar, we wait until the conclusion of the parsing process before synthesizing pURLs. + for _, p := range pkgs { + addPURL(p) + } + return pkgs, relationships, nil } @@ -348,7 +355,7 @@ func newPackageFromMavenData(pomProperties pkg.PomProperties, pomProject *pkg.Po } if packageIdentitiesMatch(p, parentPkg) { - updatePackage(p, parentPkg) + updateParentPackage(p, parentPkg) return nil } @@ -379,7 +386,7 @@ func packageIdentitiesMatch(p pkg.Package, parentPkg *pkg.Package) bool { return false } -func updatePackage(p pkg.Package, parentPkg *pkg.Package) { +func updateParentPackage(p pkg.Package, parentPkg *pkg.Package) { // we've run across more information about our parent package, add this info to the parent package metadata // the pom properties is typically a better source of information for name and version than the manifest parentPkg.Name = p.Name @@ -401,3 +408,17 @@ func updatePackage(p pkg.Package, parentPkg *pkg.Package) { parentPkg.Metadata = parentMetadata } } + +func addPURL(p *pkg.Package) { + purl := packageURL(*p) + if purl == "" { + return + } + + metadata, ok := p.Metadata.(pkg.JavaMetadata) + if !ok { + return + } + metadata.PURL = purl + p.Metadata = metadata +} diff --git a/syft/pkg/cataloger/java/archive_parser_test.go b/syft/pkg/cataloger/java/archive_parser_test.go index ac1b881c01e..748bc10cce1 100644 --- a/syft/pkg/cataloger/java/archive_parser_test.go +++ b/syft/pkg/cataloger/java/archive_parser_test.go @@ -134,6 +134,7 @@ func TestParseJar(t *testing.T) { Version: "1.0-SNAPSHOT", Extra: map[string]string{}, }, + PURL: "pkg:maven/io.jenkins.plugins/example-jenkins-plugin@1.0-SNAPSHOT", }, }, }, @@ -154,6 +155,7 @@ func TestParseJar(t *testing.T) { "Manifest-Version": "1.0", }, }, + PURL: "pkg:maven/example-java-app-gradle/example-java-app-gradle@0.1.0", }, }, }, @@ -191,6 +193,7 @@ func TestParseJar(t *testing.T) { Version: "0.1.0", Extra: map[string]string{}, }, + PURL: "pkg:maven/org.anchore/example-java-app-maven@0.1.0", }, }, "joda-time": { @@ -219,6 +222,7 @@ func TestParseJar(t *testing.T) { Description: "Date and time library to replace JDK date handling", URL: "http://www.joda.org/joda-time/", }, + PURL: "pkg:maven/joda-time/joda-time@2.9.2", }, }, }, diff --git a/syft/pkg/cataloger/java/package_url.go b/syft/pkg/cataloger/java/package_url.go new file mode 100644 index 00000000000..9d5cccd3eb9 --- /dev/null +++ b/syft/pkg/cataloger/java/package_url.go @@ -0,0 +1,25 @@ +package java + +import ( + "github.com/anchore/packageurl-go" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/common/cpe" +) + +// PackageURL returns the PURL for the specific java package (see https://github.com/package-url/purl-spec) +func packageURL(p pkg.Package) string { + var groupID = p.Name + groupIDs := cpe.GroupIDsFromJavaPackage(p) + if len(groupIDs) > 0 { + groupID = groupIDs[0] + } + + pURL := packageurl.NewPackageURL( + packageurl.TypeMaven, // TODO: should we filter down by package types here? + groupID, + p.Name, + p.Version, + nil, // TODO: there are probably several qualifiers that can be specified here + "") + return pURL.ToString() +} diff --git a/syft/pkg/cataloger/java/package_url_test.go b/syft/pkg/cataloger/java/package_url_test.go new file mode 100644 index 00000000000..27cf46ef91a --- /dev/null +++ b/syft/pkg/cataloger/java/package_url_test.go @@ -0,0 +1,45 @@ +package java + +import ( + "github.com/anchore/syft/syft/pkg" + "github.com/stretchr/testify/assert" + "testing" +) + +func Test_packageURL(t *testing.T) { + tests := []struct { + pkg pkg.Package + expect string + }{ + { + pkg: pkg.Package{ + Name: "example-java-app-maven", + Version: "0.1.0", + Language: pkg.Java, + Type: pkg.JavaPkg, + MetadataType: pkg.JavaMetadataType, + Metadata: pkg.JavaMetadata{ + VirtualPath: "test-fixtures/java-builds/packages/example-java-app-maven-0.1.0.jar", + Manifest: &pkg.JavaManifest{ + Main: map[string]string{ + "Manifest-Version": "1.0", + }, + }, + PomProperties: &pkg.PomProperties{ + Path: "META-INF/maven/org.anchore/example-java-app-maven/pom.properties", + GroupID: "org.anchore", + ArtifactID: "example-java-app-maven", + Version: "0.1.0", + Extra: make(map[string]string), + }, + }, + }, + expect: "pkg:maven/org.anchore/example-java-app-maven@0.1.0", + }, + } + for _, tt := range tests { + t.Run(tt.expect, func(t *testing.T) { + assert.Equal(t, tt.expect, packageURL(tt.pkg)) + }) + } +} diff --git a/syft/pkg/java_metadata.go b/syft/pkg/java_metadata.go index 7fe919ae566..8581c218eac 100644 --- a/syft/pkg/java_metadata.go +++ b/syft/pkg/java_metadata.go @@ -5,13 +5,12 @@ import ( "github.com/anchore/syft/syft/linux" - "github.com/anchore/packageurl-go" "github.com/anchore/syft/internal" ) var _ urlIdentifier = (*JavaMetadata)(nil) -var JenkinsPluginPomPropertiesGroupIDs = []string{ +var jenkinsPluginPomPropertiesGroupIDs = []string{ "io.jenkins.plugins", "org.jenkins.plugins", "org.jenkins-ci.plugins", @@ -25,6 +24,7 @@ type JavaMetadata struct { Manifest *JavaManifest `mapstructure:"Manifest" json:"manifest,omitempty"` PomProperties *PomProperties `mapstructure:"PomProperties" json:"pomProperties,omitempty" cyclonedx:"-"` PomProject *PomProject `mapstructure:"PomProject" json:"pomProject,omitempty"` + PURL string `hash:"ignore" json:"-"` // pURLs and CPEs are ignored for package IDs Parent *Package `hash:"ignore" json:"-"` // note: the parent cannot be included in the minimal definition of uniqueness since this field is not reproducible in an encode-decode cycle (is lossy). } @@ -59,7 +59,7 @@ type PomParent struct { // PkgTypeIndicated returns the package Type indicated by the data contained in the PomProperties. func (p PomProperties) PkgTypeIndicated() Type { - if internal.HasAnyOfPrefixes(p.GroupID, JenkinsPluginPomPropertiesGroupIDs...) || strings.Contains(p.GroupID, ".jenkins.plugin") { + if internal.HasAnyOfPrefixes(p.GroupID, jenkinsPluginPomPropertiesGroupIDs...) || strings.Contains(p.GroupID, ".jenkins.plugin") { return JenkinsPluginPkg } @@ -74,18 +74,5 @@ type JavaManifest struct { // PackageURL returns the PURL for the specific Maven package (see https://github.com/package-url/purl-spec) func (m JavaMetadata) PackageURL(_ *linux.Release) string { - if m.PomProperties != nil { - pURL := packageurl.NewPackageURL( - packageurl.TypeMaven, - m.PomProperties.GroupID, - m.PomProperties.ArtifactID, - m.PomProperties.Version, - nil, // TODO: there are probably several qualifiers that can be specified here - "") - return pURL.ToString() - } - - // TODO: support non-maven artifacts - - return "" + return m.PURL } diff --git a/syft/pkg/java_metadata_test.go b/syft/pkg/java_metadata_test.go index 07a6ca97793..d538ee5a9d0 100644 --- a/syft/pkg/java_metadata_test.go +++ b/syft/pkg/java_metadata_test.go @@ -3,7 +3,6 @@ package pkg import ( "testing" - "github.com/sergi/go-diff/diffmatchpatch" "github.com/stretchr/testify/assert" ) @@ -110,38 +109,3 @@ func TestPomProperties_PkgTypeIndicated(t *testing.T) { }) } } - -func TestJavaMetadata_pURL(t *testing.T) { - tests := []struct { - metadata JavaMetadata - expected string - }{ - { - metadata: JavaMetadata{ - PomProperties: &PomProperties{ - Path: "p", - Name: "n", - GroupID: "g.id", - ArtifactID: "a", - Version: "v", - }, - }, - expected: "pkg:maven/g.id/a@v", - }, - { - metadata: JavaMetadata{}, - expected: "", - }, - } - - for _, test := range tests { - t.Run(test.expected, func(t *testing.T) { - actual := test.metadata.PackageURL(nil) - if actual != test.expected { - dmp := diffmatchpatch.New() - diffs := dmp.DiffMain(test.expected, actual, true) - t.Errorf("diff: %s", dmp.DiffPrettyText(diffs)) - } - }) - } -} diff --git a/syft/pkg/language.go b/syft/pkg/language.go index e0168f4759a..42e96b7ab43 100644 --- a/syft/pkg/language.go +++ b/syft/pkg/language.go @@ -1,6 +1,10 @@ package pkg -import "github.com/anchore/packageurl-go" +import ( + "strings" + + "github.com/anchore/packageurl-go" +) // Language represents a single programming language. type Language string @@ -43,16 +47,16 @@ func LanguageFromPURL(p string) Language { } func LanguageByName(name string) Language { - switch name { - case packageurl.TypeMaven, purlGradlePkgType: + switch strings.ToLower(name) { + case packageurl.TypeMaven, string(purlGradlePkgType), string(JavaPkg), string(Java): return Java - case packageurl.TypeComposer: + case packageurl.TypeComposer, string(PhpComposerPkg), string(PHP): return PHP - case packageurl.TypeGolang: + case packageurl.TypeGolang, string(GoModulePkg), string(Go): return Go - case packageurl.TypeNPM: + case packageurl.TypeNPM, string(JavaScript): return JavaScript - case packageurl.TypePyPi: + case packageurl.TypePyPi, string(Python): return Python case packageurl.TypeGem: return Ruby diff --git a/syft/pkg/npm_package_json_metadata.go b/syft/pkg/npm_package_json_metadata.go index 3246b5909a1..2f9a0180ab1 100644 --- a/syft/pkg/npm_package_json_metadata.go +++ b/syft/pkg/npm_package_json_metadata.go @@ -1,6 +1,8 @@ package pkg import ( + "strings" + "github.com/anchore/packageurl-go" "github.com/anchore/syft/syft/linux" ) @@ -21,10 +23,19 @@ type NpmPackageJSONMetadata struct { // PackageURL returns the PURL for the specific NPM package (see https://github.com/package-url/purl-spec) func (p NpmPackageJSONMetadata) PackageURL(_ *linux.Release) string { + var namespace string + name := p.Name + + fields := strings.SplitN(p.Name, "/", 2) + if len(fields) > 1 { + namespace = fields[0] + name = fields[1] + } + return packageurl.NewPackageURL( packageurl.TypeNPM, - "", - p.Name, + namespace, + name, p.Version, nil, "", diff --git a/syft/pkg/npm_package_json_metadata_test.go b/syft/pkg/npm_package_json_metadata_test.go new file mode 100644 index 00000000000..87b12960e42 --- /dev/null +++ b/syft/pkg/npm_package_json_metadata_test.go @@ -0,0 +1,42 @@ +package pkg + +import ( + "github.com/anchore/packageurl-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "testing" +) + +func TestNpmPackageJSONMetadata_PackageURL(t *testing.T) { + + tests := []struct { + name string + metadata NpmPackageJSONMetadata + expected string + }{ + { + name: "no namespace", + metadata: NpmPackageJSONMetadata{ + Name: "arborist", + Version: "2.6.2", + }, + expected: "pkg:npm/arborist@2.6.2", + }, + { + name: "split by namespace", + metadata: NpmPackageJSONMetadata{ + Name: "@npmcli/arborist", + Version: "2.6.2", + }, + expected: "pkg:npm/@npmcli/arborist@2.6.2", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := tt.metadata.PackageURL(nil) + assert.Equal(t, tt.expected, actual) + _, err := packageurl.FromString(actual) + require.NoError(t, err) + }) + } +} diff --git a/syft/pkg/url_test.go b/syft/pkg/url_test.go index 2abae9368d3..dad951b6ba9 100644 --- a/syft/pkg/url_test.go +++ b/syft/pkg/url_test.go @@ -141,13 +141,8 @@ func TestPackageURL(t *testing.T) { Version: "bad-v0.1.0", Type: JavaPkg, Metadata: JavaMetadata{ - PomProperties: &PomProperties{ - Path: "p", - Name: "n", - GroupID: "g.id", - ArtifactID: "a", - Version: "v", - }, + PomProperties: &PomProperties{}, + PURL: "pkg:maven/g.id/a@v", // assembled by the java cataloger }, }, @@ -160,13 +155,8 @@ func TestPackageURL(t *testing.T) { Version: "bad-v0.1.0", Type: JenkinsPluginPkg, Metadata: JavaMetadata{ - PomProperties: &PomProperties{ - Path: "p", - Name: "n", - GroupID: "g.id", - ArtifactID: "a", - Version: "v", - }, + PomProperties: &PomProperties{}, + PURL: "pkg:maven/g.id/a@v", // assembled by the java cataloger }, }, From 220f3a24fdf55ba379535983d32bd41ada76e4bf Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 10 Feb 2022 16:18:00 -0500 Subject: [PATCH 2/6] deduplicate SPDX tag-value package IDs (#813) Signed-off-by: Alex Goodman --- .../snapshot/TestSPDXTagValueDirectoryEncoder.golden | 10 +++++----- .../snapshot/TestSPDXTagValueImageEncoder.golden | 10 +++++----- internal/formats/spdx22tagvalue/to_format_model.go | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/internal/formats/spdx22tagvalue/test-fixtures/snapshot/TestSPDXTagValueDirectoryEncoder.golden b/internal/formats/spdx22tagvalue/test-fixtures/snapshot/TestSPDXTagValueDirectoryEncoder.golden index c2085810948..84a7de316a7 100644 --- a/internal/formats/spdx22tagvalue/test-fixtures/snapshot/TestSPDXTagValueDirectoryEncoder.golden +++ b/internal/formats/spdx22tagvalue/test-fixtures/snapshot/TestSPDXTagValueDirectoryEncoder.golden @@ -2,16 +2,16 @@ SPDXVersion: SPDX-2.2 DataLicense: CC0-1.0 SPDXID: SPDXRef-DOCUMENT DocumentName: /some/path -DocumentNamespace: https://anchore.com/syft/dir/some/path-22f5732a-cab0-4376-9b79-15e413049500 -LicenseListVersion: 3.15 +DocumentNamespace: https://anchore.com/syft/dir/some/path-4b90f56d-d596-4ad8-b6a5-17f7d801350d +LicenseListVersion: 3.16 Creator: Organization: Anchore, Inc Creator: Tool: syft-[not provided] -Created: 2021-12-01T15:08:43Z +Created: 2022-02-10T21:09:27Z ##### Package: package-2 PackageName: package-2 -SPDXID: SPDXRef-Package-deb-package-2 +SPDXID: SPDXRef-Package-deb-package-2-ad3d1c4abd84bf75 PackageVersion: 2.0.1 PackageDownloadLocation: NOASSERTION FilesAnalyzed: false @@ -24,7 +24,7 @@ ExternalRef: PACKAGE_MANAGER purl a-purl-2 ##### Package: package-1 PackageName: package-1 -SPDXID: SPDXRef-Package-python-package-1 +SPDXID: SPDXRef-Package-python-package-1-1d97af55efe9512f PackageVersion: 1.0.1 PackageDownloadLocation: NOASSERTION FilesAnalyzed: false diff --git a/internal/formats/spdx22tagvalue/test-fixtures/snapshot/TestSPDXTagValueImageEncoder.golden b/internal/formats/spdx22tagvalue/test-fixtures/snapshot/TestSPDXTagValueImageEncoder.golden index 7743514fd2a..f54713441c9 100644 --- a/internal/formats/spdx22tagvalue/test-fixtures/snapshot/TestSPDXTagValueImageEncoder.golden +++ b/internal/formats/spdx22tagvalue/test-fixtures/snapshot/TestSPDXTagValueImageEncoder.golden @@ -2,16 +2,16 @@ SPDXVersion: SPDX-2.2 DataLicense: CC0-1.0 SPDXID: SPDXRef-DOCUMENT DocumentName: user-image-input -DocumentNamespace: https://anchore.com/syft/image/user-image-input-ce4d4ae5-9d79-4f84-a410-361e394c2908 -LicenseListVersion: 3.15 +DocumentNamespace: https://anchore.com/syft/image/user-image-input-26a2def6-53d0-4504-b99a-a046832508ac +LicenseListVersion: 3.16 Creator: Organization: Anchore, Inc Creator: Tool: syft-[not provided] -Created: 2021-12-01T15:08:44Z +Created: 2022-02-10T21:09:27Z ##### Package: package-2 PackageName: package-2 -SPDXID: SPDXRef-Package-deb-package-2 +SPDXID: SPDXRef-Package-deb-package-2-73f796c846875b9e PackageVersion: 2.0.1 PackageDownloadLocation: NOASSERTION FilesAnalyzed: false @@ -24,7 +24,7 @@ ExternalRef: PACKAGE_MANAGER purl a-purl-2 ##### Package: package-1 PackageName: package-1 -SPDXID: SPDXRef-Package-python-package-1 +SPDXID: SPDXRef-Package-python-package-1-d9527e708c11f8b9 PackageVersion: 1.0.1 PackageDownloadLocation: NOASSERTION FilesAnalyzed: false diff --git a/internal/formats/spdx22tagvalue/to_format_model.go b/internal/formats/spdx22tagvalue/to_format_model.go index c240eeaa762..ef17dc54a04 100644 --- a/internal/formats/spdx22tagvalue/to_format_model.go +++ b/internal/formats/spdx22tagvalue/to_format_model.go @@ -94,9 +94,9 @@ func toFormatModel(s sbom.SBOM) (*spdx.Document2_2, error) { func toFormatPackages(catalog *pkg.Catalog) map[spdx.ElementID]*spdx.Package2_2 { results := make(map[spdx.ElementID]*spdx.Package2_2) - for p := range catalog.Enumerate() { + for _, p := range catalog.Sorted() { // name should be guaranteed to be unique, but semantically useful and stable - id := fmt.Sprintf("Package-%+v-%s", p.Type, p.Name) + id := fmt.Sprintf("Package-%+v-%s-%s", p.Type, p.Name, p.ID()) // If the Concluded License is not the same as the Declared License, a written explanation should be provided // in the Comments on License field (section 3.16). With respect to NOASSERTION, a written explanation in From e1e9ccb401198a48345dbf3f12c5fb7efb7fe645 Mon Sep 17 00:00:00 2001 From: Christopher Angelo Phillips <32073428+spiffcs@users.noreply.github.com> Date: Fri, 11 Feb 2022 13:36:52 -0500 Subject: [PATCH 3/6] update golang crypto library dependency (#815) * bump golang crypto to resolve CVE-2020-29652 Signed-off-by: Christopher Phillips * go mod tidy Signed-off-by: Christopher Phillips --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 340e2e66688..cb756068a5f 100644 --- a/go.mod +++ b/go.mod @@ -101,7 +101,7 @@ require ( github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect - golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect + golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect golang.org/x/sys v0.0.0-20211110154304-99a53858aa08 // indirect diff --git a/go.sum b/go.sum index a3ded79203f..d38fc844b03 100644 --- a/go.sum +++ b/go.sum @@ -896,8 +896,9 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 h1:XdAboW3BNMv9ocSCOk/u1MFioZGzCNkiJZ19v9Oe3Ig= +golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= From 51c6eb30f5cb9fe9af6bb55277ed4c728d0f6906 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Mon, 14 Feb 2022 20:40:51 -0500 Subject: [PATCH 4/6] bump stereoscope to include functional options (#823) Signed-off-by: Alex Goodman --- go.mod | 2 +- go.sum | 4 ++-- internal/formats/syftjson/decoder_test.go | 4 ++++ internal/formats/syftjson/to_format_model.go | 10 +++++++++- internal/formats/syftjson/to_format_model_test.go | 2 ++ syft/source/source.go | 9 +++++++-- 6 files changed, 25 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index cb756068a5f..6534436c2b6 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29 - github.com/anchore/stereoscope v0.0.0-20220209160132-2e595043fa19 + github.com/anchore/stereoscope v0.0.0-20220214165125-25ebd49a842b github.com/antihax/optional v1.0.0 github.com/bmatcuk/doublestar/v4 v4.0.2 github.com/docker/docker v20.10.12+incompatible diff --git a/go.sum b/go.sum index d38fc844b03..8839a06e6fb 100644 --- a/go.sum +++ b/go.sum @@ -111,8 +111,8 @@ github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZV github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E= github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29 h1:K9LfnxwhqvihqU0+MF325FNy7fsKV9EGaUxdfR4gnWk= github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29/go.mod h1:Oc1UkGaJwY6ND6vtAqPSlYrptKRJngHwkwB6W7l1uP0= -github.com/anchore/stereoscope v0.0.0-20220209160132-2e595043fa19 h1:INJWzjqSo4uF5NrYISnIfIpnmgV+nfYwbrL8nnmIz7g= -github.com/anchore/stereoscope v0.0.0-20220209160132-2e595043fa19/go.mod h1:QpDHHV2h1NNfu7klzU75XC8RvSlaPK6HHgi0dy8A6sk= +github.com/anchore/stereoscope v0.0.0-20220214165125-25ebd49a842b h1:PMMXpTEHVVLErrXQ6mH9ocLAQyvQu/LUhdstrhx7AC4= +github.com/anchore/stereoscope v0.0.0-20220214165125-25ebd49a842b/go.mod h1:QpDHHV2h1NNfu7klzU75XC8RvSlaPK6HHgi0dy8A6sk= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= diff --git a/internal/formats/syftjson/decoder_test.go b/internal/formats/syftjson/decoder_test.go index ea69ee2c7c9..3dc069fc2bd 100644 --- a/internal/formats/syftjson/decoder_test.go +++ b/internal/formats/syftjson/decoder_test.go @@ -21,6 +21,10 @@ func TestEncodeDecodeCycle(t *testing.T) { assert.NoError(t, err) for _, d := range deep.Equal(originalSBOM.Source, actualSBOM.Source) { + if strings.HasSuffix(d, " != []") { + // semantically the same + continue + } t.Errorf("metadata difference: %+v", d) } diff --git a/internal/formats/syftjson/to_format_model.go b/internal/formats/syftjson/to_format_model.go index 06cf5f4cdfa..7d4feee4af3 100644 --- a/internal/formats/syftjson/to_format_model.go +++ b/internal/formats/syftjson/to_format_model.go @@ -217,9 +217,17 @@ func toRelationshipModel(relationships []artifact.Relationship) []model.Relation func toSourceModel(src source.Metadata) (model.Source, error) { switch src.Scheme { case source.ImageScheme: + metadata := src.ImageMetadata + // ensure that empty collections are not shown as null + if metadata.RepoDigests == nil { + metadata.RepoDigests = []string{} + } + if metadata.Tags == nil { + metadata.Tags = []string{} + } return model.Source{ Type: "image", - Target: src.ImageMetadata, + Target: metadata, }, nil case source.DirectoryScheme: return model.Source{ diff --git a/internal/formats/syftjson/to_format_model_test.go b/internal/formats/syftjson/to_format_model_test.go index ec250dc709b..18d67936dc0 100644 --- a/internal/formats/syftjson/to_format_model_test.go +++ b/internal/formats/syftjson/to_format_model_test.go @@ -63,6 +63,8 @@ func Test_toSourceModel(t *testing.T) { ID: "id...", ManifestDigest: "digest...", MediaType: "type...", + RepoDigests: []string{}, + Tags: []string{}, }, }, }, diff --git a/syft/source/source.go b/syft/source/source.go index 91d754e65ae..810b97d93fe 100644 --- a/syft/source/source.go +++ b/syft/source/source.go @@ -90,7 +90,12 @@ func parseScheme(userInput string) string { func getImageWithRetryStrategy(userInput, location string, imageSource image.Source, registryOptions *image.RegistryOptions) (*image.Image, func(), error) { ctx := context.TODO() - img, err := stereoscope.GetImageFromSource(ctx, location, imageSource, registryOptions) + var opts []stereoscope.Option + if registryOptions != nil { + opts = append(opts, stereoscope.WithRegistryOptions(*registryOptions)) + } + + img, err := stereoscope.GetImageFromSource(ctx, location, imageSource, opts...) if err == nil { // Success on the first try! return img, stereoscope.Cleanup, nil @@ -120,7 +125,7 @@ func getImageWithRetryStrategy(userInput, location string, imageSource image.Sou // We need to determine the image source again, such that this determination // doesn't take scheme parsing into account. imageSource = image.DetermineImagePullSource(userInput) - img, err = stereoscope.GetImageFromSource(ctx, userInput, imageSource, registryOptions) + img, err = stereoscope.GetImageFromSource(ctx, userInput, imageSource, opts...) if err != nil { return nil, nil, err } From 52d2e62cdcc08967ead746820346766e12698488 Mon Sep 17 00:00:00 2001 From: Christopher Angelo Phillips <32073428+spiffcs@users.noreply.github.com> Date: Tue, 15 Feb 2022 12:51:02 -0500 Subject: [PATCH 5/6] remove duplicate manifest lines (#828) --- .goreleaser.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index cb90936819c..5b6031dc17f 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -124,17 +124,9 @@ dockers: docker_manifests: - name_template: anchore/syft:{{ .Tag }} image_templates: - - anchore/syft:{{ .Tag }}-amd64 - - anchore/syft:v{{ .Major }}-amd64 - anchore/syft:v{{ .Major }}.{{ .Minor }}-amd64 - - anchore/syft:{{ .Tag }}-arm64v8 - - anchore/syft:v{{ .Major }}-arm64v8 - anchore/syft:v{{ .Major }}.{{ .Minor }}-arm64v8 - name_template: anchore/syft:latest image_templates: - - anchore/syft:{{ .Tag }}-amd64 - - anchore/syft:v{{ .Major }}-amd64 - anchore/syft:v{{ .Major }}.{{ .Minor }}-amd64 - - anchore/syft:{{ .Tag }}-arm64v8 - - anchore/syft:v{{ .Major }}-arm64v8 - anchore/syft:v{{ .Major }}.{{ .Minor }}-arm64v8 From 55c7f3d1e7118a2cc46624a73e2e23dcbece775b Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 15 Feb 2022 17:23:11 -0500 Subject: [PATCH 6/6] Upgrade install.sh to support installations for previous versions (#830) --- .github/workflows/release.yaml | 2 ++ install.sh | 32 +++++++++++++++++++++++++------- test/install/Makefile | 28 ++++++++++++++++++++++++++-- 3 files changed, 53 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 97e331de915..aee64641343 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -142,10 +142,12 @@ jobs: AC_PASSWORD: ${{ secrets.ENG_CI_APPLE_ID_PASS }} - uses: anchore/sbom-action@v0 + continue-on-error: true with: artifact-name: sbom.spdx.json - uses: 8398a7/action-slack@v3 + continue-on-error: true with: status: ${{ job.status }} fields: repo,workflow,action,eventName diff --git a/install.sh b/install.sh index edadb457087..f118257ed68 100755 --- a/install.sh +++ b/install.sh @@ -6,6 +6,11 @@ PROJECT_NAME="syft" OWNER=anchore REPO="${PROJECT_NAME}" GITHUB_DOWNLOAD_PREFIX=https://github.com/${OWNER}/${REPO}/releases/download +INSTALL_SH_BASE_URL=https://raw.githubusercontent.com/${OWNER}/${PROJECT_NAME} +PROGRAM_ARGS=$@ + +# do not change the name of this parameter (this must always be backwards compatible) +DOWNLOAD_TAG_INSTALL_SCRIPT=${DOWNLOAD_TAG_INSTALL_SCRIPT:-true} # # usage [script-name] @@ -607,7 +612,10 @@ install_asset() ( main() ( # parse arguments + # note: never change default install directory (this must always be backwards compatible) install_dir=${install_dir:-./bin} + + # note: never change the program flags or arguments (this must always be backwards compatible) while getopts "b:dh?x" arg; do case "$arg" in b) install_dir="$OPTARG" ;; @@ -629,10 +637,10 @@ main() ( tag=$1 if [ -z "${tag}" ]; then - log_info "checking github for the current release tag" + log_debug "checking github for the current release tag" tag="" else - log_info "checking github for release tag='${tag}'" + log_debug "checking github for release tag='${tag}'" fi set -u @@ -644,20 +652,30 @@ main() ( return 1 fi - version=$(tag_to_version "${tag}") - - download_dir=$(mktemp -d) - trap 'rm -rf -- "$download_dir"' EXIT - # run the application + version=$(tag_to_version "${tag}") os=$(uname_os) arch=$(uname_arch) format=$(get_format_name "${os}" "${arch}" "tar.gz") binary=$(get_binary_name "${os}" "${arch}" "${PROJECT_NAME}") download_url="${GITHUB_DOWNLOAD_PREFIX}/${tag}" + # we always use the install.sh script that is associated with the tagged release. Why? the latest install.sh is not + # guaranteed to be able to install every version of the application. We use the DOWNLOAD_TAG_INSTALL_SCRIPT env var + # to indicate if we should continue processing with the existing script or to download the script from the given tag. + if [ "${DOWNLOAD_TAG_INSTALL_SCRIPT}" = "true" ]; then + export DOWNLOAD_TAG_INSTALL_SCRIPT=false + log_info "fetching release script for tag='${tag}'" + http_copy "${INSTALL_SH_BASE_URL}/${tag}/install.sh" "" | sh -s -- ${PROGRAM_ARGS} + exit $? + fi + log_info "using release tag='${tag}' version='${version}' os='${os}' arch='${arch}'" + + download_dir=$(mktemp -d) + trap 'rm -rf -- "$download_dir"' EXIT + log_debug "downloading files into ${download_dir}" download_and_install_asset "${download_url}" "${download_dir}" "${install_dir}" "${PROJECT_NAME}" "${os}" "${arch}" "${version}" "${format}" "${binary}" diff --git a/test/install/Makefile b/test/install/Makefile index 024bb44cba2..9e74cbe4b0c 100644 --- a/test/install/Makefile +++ b/test/install/Makefile @@ -13,6 +13,9 @@ UNIT=make unit-local # the line if there are breaking changes made that don't align with the latest release (but will be OK with the next # release) ACCEPTANCE_CMD=sh -c '../../install.sh -b /usr/local/bin && syft version' +# we also want to test against a previous release to ensure that install.sh defers execution to a former install.sh +PREVIOUS_RELEASE=v0.33.0 +ACCEPTANCE_PREVIOUS_RELEASE_CMD=sh -c "../../install.sh -b /usr/local/bin $(PREVIOUS_RELEASE) && syft version" # CI cache busting values; change these if you want CI to not use previous stored cache INSTALL_TEST_CACHE_BUSTER=894d8ca @@ -28,23 +31,37 @@ test: unit acceptance ci-test-mac: unit-local acceptance-local # note: do not add acceptance-local to this list +.PHONY: acceptance acceptance: acceptance-ubuntu-20.04 acceptance-alpine-3.6 acceptance-busybox-1.35 +.PHONY: unit unit: unit-ubuntu-20.04 +.PHONY: unit-local unit-local: $(call title,unit tests) @for f in $(shell ls *_test.sh); do echo "Running unit test suite '$${f}'"; bash $${f} || exit 1; done -acceptance-local: - $(acceptance) +.PHONY: acceptance-local +acceptance-local: acceptance-current-release-local acceptance-previous-release-local +.PHONY: acceptance-current-release-local +acceptance-current-release-local: + $(ACCEPTANCE_CMD) + +.PHONY: acceptance-previous-release-local +acceptance-previous-release-local: + $(ACCEPTANCE_PREVIOUS_RELEASE_CMD) + syft version | grep $(shell echo $(PREVIOUS_RELEASE)| tr -d "v") + +.PHONY: save save: ubuntu-20.04 alpine-3.6 busybox-1.35 @mkdir cache || true docker image save -o cache/ubuntu-env.tar $(UBUNTU_IMAGE) docker image save -o cache/alpine-env.tar $(ALPINE_IMAGE) docker image save -o cache/busybox-env.tar $(BUSYBOX_IMAGE) +.PHONY: load load: docker image load -i cache/ubuntu-env.tar docker image load -i cache/alpine-env.tar @@ -52,16 +69,19 @@ load: ## UBUNTU ####################################################### +.PHONY: acceptance-ubuntu-20.04 acceptance-ubuntu-20.04: ubuntu-20.04 $(call title,ubuntu:20.04 - acceptance) $(DOCKER_RUN) $(UBUNTU_IMAGE) \ $(ACCEPTANCE_CMD) +.PHONY: unit-ubuntu-20.04 unit-ubuntu-20.04: ubuntu-20.04 $(call title,ubuntu:20.04 - unit) $(DOCKER_RUN) $(UBUNTU_IMAGE) \ $(UNIT) +.PHONY: ubuntu-20.04 ubuntu-20.04: $(call title,ubuntu:20.04 - build environment) docker build -t $(UBUNTU_IMAGE) -f $(ENVS)/Dockerfile-ubuntu-20.04 . @@ -70,11 +90,13 @@ ubuntu-20.04: # note: unit tests cannot be run with sh (alpine dosn't have bash by default) +.PHONY: acceptance-alpine-3.6 acceptance-alpine-3.6: alpine-3.6 $(call title,alpine:3.6 - acceptance) $(DOCKER_RUN) $(ALPINE_IMAGE) \ $(ACCEPTANCE_CMD) +.PHONY: alpine-3.6 alpine-3.6: $(call title,alpine:3.6 - build environment) docker build -t $(ALPINE_IMAGE) -f $(ENVS)/Dockerfile-alpine-3.6 . @@ -85,12 +107,14 @@ alpine-3.6: # note: busybox by default will not have cacerts, so you will get TLS warnings (we want to test under these conditions) +.PHONY: acceptance-busybox-1.35 acceptance-busybox-1.35: busybox-1.35 $(call title,busybox-1.35 - acceptance) $(DOCKER_RUN) $(BUSYBOX_IMAGE) \ $(ACCEPTANCE_CMD) @echo "\n*** test note: you should see syft spit out a 'x509: certificate signed by unknown authority' error --this is expected ***" +.PHONY: busybox-1.35 busybox-1.35: $(call title,busybox-1.35 - build environment) docker pull $(BUSYBOX_IMAGE)