From 699a7a3577820d2c2715c63c1cd0b9e8f921c05d Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 21 Oct 2021 10:27:03 -0400 Subject: [PATCH 1/5] remove existing spdxjson presenter + helpers Signed-off-by: Alex Goodman --- internal/presenter/packages/spdx_helpers.go | 207 ------- .../presenter/packages/spdx_helpers_test.go | 527 ------------------ .../presenter/packages/spdx_json_presenter.go | 142 ----- .../packages/spdx_json_presenter_test.go | 42 -- 4 files changed, 918 deletions(-) delete mode 100644 internal/presenter/packages/spdx_helpers.go delete mode 100644 internal/presenter/packages/spdx_helpers_test.go delete mode 100644 internal/presenter/packages/spdx_json_presenter.go delete mode 100644 internal/presenter/packages/spdx_json_presenter_test.go diff --git a/internal/presenter/packages/spdx_helpers.go b/internal/presenter/packages/spdx_helpers.go deleted file mode 100644 index c175e0da9ff..00000000000 --- a/internal/presenter/packages/spdx_helpers.go +++ /dev/null @@ -1,207 +0,0 @@ -package packages - -import ( - "crypto/sha256" - "fmt" - "path/filepath" - "strings" - - "github.com/anchore/syft/internal/presenter/packages/model/spdx22" - "github.com/anchore/syft/internal/spdxlicense" - "github.com/anchore/syft/syft/pkg" -) - -func getSPDXExternalRefs(p *pkg.Package) (externalRefs []spdx22.ExternalRef) { - externalRefs = make([]spdx22.ExternalRef, 0) - for _, c := range p.CPEs { - externalRefs = append(externalRefs, spdx22.ExternalRef{ - ReferenceCategory: spdx22.SecurityReferenceCategory, - ReferenceLocator: c.BindToFmtString(), - ReferenceType: spdx22.Cpe23ExternalRefType, - }) - } - - if p.PURL != "" { - externalRefs = append(externalRefs, spdx22.ExternalRef{ - ReferenceCategory: spdx22.PackageManagerReferenceCategory, - ReferenceLocator: p.PURL, - ReferenceType: spdx22.PurlExternalRefType, - }) - } - return externalRefs -} - -func getSPDXFiles(packageSpdxID string, p *pkg.Package) (files []spdx22.File, fileIDs []string, relationships []spdx22.Relationship) { - files = make([]spdx22.File, 0) - fileIDs = make([]string, 0) - relationships = make([]spdx22.Relationship, 0) - - pkgFileOwner, ok := p.Metadata.(pkg.FileOwner) - if !ok { - return files, fileIDs, relationships - } - - for _, ownedFilePath := range pkgFileOwner.OwnedFiles() { - baseFileName := filepath.Base(ownedFilePath) - pathHash := sha256.Sum256([]byte(ownedFilePath)) - fileSpdxID := spdx22.ElementID(fmt.Sprintf("File-%s-%x", p.Name, pathHash)).String() - - fileIDs = append(fileIDs, fileSpdxID) - - files = append(files, spdx22.File{ - FileName: ownedFilePath, - Item: spdx22.Item{ - Element: spdx22.Element{ - SPDXID: fileSpdxID, - Name: baseFileName, - }, - }, - }) - - relationships = append(relationships, spdx22.Relationship{ - SpdxElementID: packageSpdxID, - RelationshipType: spdx22.ContainsRelationship, - RelatedSpdxElement: fileSpdxID, - }) - } - - return files, fileIDs, relationships -} - -func getSPDXLicense(p *pkg.Package) string { - // source: https://spdx.github.io/spdx-spec/3-package-information/#313-concluded-license - // The options to populate this field are limited to: - // A valid SPDX License Expression as defined in Appendix IV; - // NONE, if the SPDX file creator concludes there is no license available for this package; or - // NOASSERTION if: - // (i) the SPDX file creator has attempted to but cannot reach a reasonable objective determination; - // (ii) the SPDX file creator has made no attempt to determine this field; or - // (iii) the SPDX file creator has intentionally provided no information (no meaning should be implied by doing so). - - if len(p.Licenses) == 0 { - return "NONE" - } - - // take all licenses and assume an AND expression; for information about license expressions see https://spdx.github.io/spdx-spec/appendix-IV-SPDX-license-expressions/ - var parsedLicenses []string - for _, l := range p.Licenses { - if value, exists := spdxlicense.ID(l); exists { - parsedLicenses = append(parsedLicenses, value) - } - } - - if len(parsedLicenses) == 0 { - return "NOASSERTION" - } - - return strings.Join(parsedLicenses, " AND ") -} - -func noneIfEmpty(value string) string { - if strings.TrimSpace(value) == "" { - return "NONE" - } - return value -} - -func getSPDXDownloadLocation(p *pkg.Package) string { - // 3.7: Package Download Location - // Cardinality: mandatory, one - // NONE if there is no download location whatsoever. - // NOASSERTION if: - // (i) the SPDX file creator has attempted to but cannot reach a reasonable objective determination; - // (ii) the SPDX file creator has made no attempt to determine this field; or - // (iii) the SPDX file creator has intentionally provided no information (no meaning should be implied by doing so). - - switch metadata := p.Metadata.(type) { - case pkg.ApkMetadata: - return noneIfEmpty(metadata.URL) - case pkg.NpmPackageJSONMetadata: - return noneIfEmpty(metadata.URL) - default: - return "NOASSERTION" - } -} - -func getSPDXHomepage(p *pkg.Package) string { - switch metadata := p.Metadata.(type) { - case pkg.GemMetadata: - return metadata.Homepage - case pkg.NpmPackageJSONMetadata: - return metadata.Homepage - default: - return "" - } -} - -func getSPDXSourceInfo(p *pkg.Package) string { - answer := "" - switch p.Type { - case pkg.RpmPkg: - answer = "acquired package info from RPM DB" - case pkg.ApkPkg: - answer = "acquired package info from APK DB" - case pkg.DebPkg: - answer = "acquired package info from DPKG DB" - case pkg.NpmPkg: - answer = "acquired package info from installed node module manifest file" - case pkg.PythonPkg: - answer = "acquired package info from installed python package manifest file" - case pkg.JavaPkg, pkg.JenkinsPluginPkg: - answer = "acquired package info from installed java archive" - case pkg.GemPkg: - answer = "acquired package info from installed gem metadata file" - case pkg.GoModulePkg: - answer = "acquired package info from go module information" - case pkg.RustPkg: - answer = "acquired package info from rust cargo manifest" - default: - answer = "acquired package info from the following paths" - } - var paths []string - for _, l := range p.Locations { - paths = append(paths, l.RealPath) - } - - return answer + ": " + strings.Join(paths, ", ") -} - -func getSPDXOriginator(p *pkg.Package) string { - switch metadata := p.Metadata.(type) { - case pkg.ApkMetadata: - return metadata.Maintainer - case pkg.NpmPackageJSONMetadata: - return metadata.Author - case pkg.PythonPackageMetadata: - author := metadata.Author - if author == "" { - return metadata.AuthorEmail - } - if metadata.AuthorEmail != "" { - author += fmt.Sprintf(" <%s>", metadata.AuthorEmail) - } - return author - case pkg.GemMetadata: - if len(metadata.Authors) > 0 { - return metadata.Authors[0] - } - return "" - case pkg.RpmdbMetadata: - return metadata.Vendor - case pkg.DpkgMetadata: - return metadata.Maintainer - default: - return "" - } -} - -func getSPDXDescription(p *pkg.Package) string { - switch metadata := p.Metadata.(type) { - case pkg.ApkMetadata: - return metadata.Description - case pkg.NpmPackageJSONMetadata: - return metadata.Description - default: - return "" - } -} diff --git a/internal/presenter/packages/spdx_helpers_test.go b/internal/presenter/packages/spdx_helpers_test.go deleted file mode 100644 index 5002dc62147..00000000000 --- a/internal/presenter/packages/spdx_helpers_test.go +++ /dev/null @@ -1,527 +0,0 @@ -package packages - -import ( - "testing" - - "github.com/anchore/syft/syft/source" - - "github.com/stretchr/testify/assert" - - "github.com/anchore/syft/internal/presenter/packages/model/spdx22" - "github.com/anchore/syft/syft/pkg" -) - -func Test_getSPDXExternalRefs(t *testing.T) { - testCPE := pkg.MustCPE("cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*") - tests := []struct { - name string - input pkg.Package - expected []spdx22.ExternalRef - }{ - { - name: "cpe + purl", - input: pkg.Package{ - CPEs: []pkg.CPE{ - testCPE, - }, - PURL: "a-purl", - }, - expected: []spdx22.ExternalRef{ - { - ReferenceCategory: spdx22.SecurityReferenceCategory, - ReferenceLocator: testCPE.BindToFmtString(), - ReferenceType: spdx22.Cpe23ExternalRefType, - }, - { - ReferenceCategory: spdx22.PackageManagerReferenceCategory, - ReferenceLocator: "a-purl", - ReferenceType: spdx22.PurlExternalRefType, - }, - }, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - assert.ElementsMatch(t, test.expected, getSPDXExternalRefs(&test.input)) - }) - } -} - -func Test_getSPDXLicense(t *testing.T) { - tests := []struct { - name string - input pkg.Package - expected string - }{ - { - name: "no licenses", - input: pkg.Package{}, - expected: "NONE", - }, - { - name: "no SPDX licenses", - input: pkg.Package{ - Licenses: []string{ - "made-up", - }, - }, - expected: "NOASSERTION", - }, - { - name: "with SPDX license", - input: pkg.Package{ - Licenses: []string{ - "MIT", - }, - }, - expected: "MIT", - }, - { - name: "with SPDX license expression", - input: pkg.Package{ - Licenses: []string{ - "MIT", - "GPL-3.0", - }, - }, - expected: "MIT AND GPL-3.0", - }, - { - name: "cap insensitive", - input: pkg.Package{ - Licenses: []string{ - "gpl-3.0", - }, - }, - expected: "GPL-3.0", - }, - { - name: "debian to spdx conversion", - input: pkg.Package{ - Licenses: []string{ - "GPL-2", - }, - }, - expected: "GPL-2.0", - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.expected, getSPDXLicense(&test.input)) - }) - } -} - -func Test_noneIfEmpty(t *testing.T) { - tests := []struct { - name string - value string - expected string - }{ - { - name: "non-zero value", - value: "something", - expected: "something", - }, - { - name: "empty", - value: "", - expected: "NONE", - }, - { - name: "space", - value: " ", - expected: "NONE", - }, - { - name: "tab", - value: "\t", - expected: "NONE", - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.expected, noneIfEmpty(test.value)) - }) - } -} - -func Test_getSPDXDownloadLocation(t *testing.T) { - tests := []struct { - name string - input pkg.Package - expected string - }{ - { - name: "no metadata", - input: pkg.Package{}, - expected: "NOASSERTION", - }, - { - name: "from apk", - input: pkg.Package{ - Metadata: pkg.ApkMetadata{ - URL: "http://a-place.gov", - }, - }, - expected: "http://a-place.gov", - }, - { - name: "from npm", - input: pkg.Package{ - Metadata: pkg.NpmPackageJSONMetadata{ - URL: "http://a-place.gov", - }, - }, - expected: "http://a-place.gov", - }, - { - name: "empty", - input: pkg.Package{ - Metadata: pkg.NpmPackageJSONMetadata{ - URL: "", - }, - }, - expected: "NONE", - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.expected, getSPDXDownloadLocation(&test.input)) - }) - } -} - -func Test_getSPDXHomepage(t *testing.T) { - tests := []struct { - name string - input pkg.Package - expected string - }{ - { - // note: since this is an optional field, no value is preferred over NONE or NOASSERTION - name: "no metadata", - input: pkg.Package{}, - expected: "", - }, - { - name: "from gem", - input: pkg.Package{ - Metadata: pkg.GemMetadata{ - Homepage: "http://a-place.gov", - }, - }, - expected: "http://a-place.gov", - }, - { - name: "from npm", - input: pkg.Package{ - Metadata: pkg.NpmPackageJSONMetadata{ - Homepage: "http://a-place.gov", - }, - }, - expected: "http://a-place.gov", - }, - { - // note: since this is an optional field, no value is preferred over NONE or NOASSERTION - name: "empty", - input: pkg.Package{ - Metadata: pkg.NpmPackageJSONMetadata{ - Homepage: "", - }, - }, - expected: "", - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.expected, getSPDXHomepage(&test.input)) - }) - } -} - -func Test_getSPDXSourceInfo(t *testing.T) { - tests := []struct { - name string - input pkg.Package - expected []string - }{ - { - name: "locations are captured", - input: pkg.Package{ - // note: no type given - Locations: []source.Location{ - { - RealPath: "/a-place", - VirtualPath: "/b-place", - }, - { - RealPath: "/c-place", - VirtualPath: "/d-place", - }, - }, - }, - expected: []string{ - "from the following paths", - "/a-place", - "/c-place", - }, - }, - { - // note: no specific support for this - input: pkg.Package{ - Type: pkg.KbPkg, - }, - expected: []string{ - "from the following paths", - }, - }, - { - input: pkg.Package{ - Type: pkg.RpmPkg, - }, - expected: []string{ - "from RPM DB", - }, - }, - { - input: pkg.Package{ - Type: pkg.ApkPkg, - }, - expected: []string{ - "from APK DB", - }, - }, - { - input: pkg.Package{ - Type: pkg.DebPkg, - }, - expected: []string{ - "from DPKG DB", - }, - }, - { - input: pkg.Package{ - Type: pkg.NpmPkg, - }, - expected: []string{ - "from installed node module manifest file", - }, - }, - { - input: pkg.Package{ - Type: pkg.PythonPkg, - }, - expected: []string{ - "from installed python package manifest file", - }, - }, - { - input: pkg.Package{ - Type: pkg.JavaPkg, - }, - expected: []string{ - "from installed java archive", - }, - }, - { - input: pkg.Package{ - Type: pkg.JenkinsPluginPkg, - }, - expected: []string{ - "from installed java archive", - }, - }, - { - input: pkg.Package{ - Type: pkg.GemPkg, - }, - expected: []string{ - "from installed gem metadata file", - }, - }, - { - input: pkg.Package{ - Type: pkg.GoModulePkg, - }, - expected: []string{ - "from go module information", - }, - }, - { - input: pkg.Package{ - Type: pkg.RustPkg, - }, - expected: []string{ - "from rust cargo manifest", - }, - }, - } - var pkgTypes []pkg.Type - for _, test := range tests { - t.Run(test.name+" "+string(test.input.Type), func(t *testing.T) { - if test.input.Type != "" { - pkgTypes = append(pkgTypes, test.input.Type) - } - actual := getSPDXSourceInfo(&test.input) - for _, expected := range test.expected { - assert.Contains(t, actual, expected) - } - }) - } - assert.ElementsMatch(t, pkg.AllPkgs, pkgTypes, "missing one or more package types to test against (maybe a package type was added?)") -} - -func Test_getSPDXOriginator(t *testing.T) { - tests := []struct { - name string - input pkg.Package - expected string - }{ - { - // note: since this is an optional field, no value is preferred over NONE or NOASSERTION - name: "no metadata", - input: pkg.Package{}, - expected: "", - }, - { - name: "from gem", - input: pkg.Package{ - Metadata: pkg.GemMetadata{ - Authors: []string{ - "auth1", - "auth2", - }, - }, - }, - expected: "auth1", - }, - { - name: "from npm", - input: pkg.Package{ - Metadata: pkg.NpmPackageJSONMetadata{ - Author: "auth", - }, - }, - expected: "auth", - }, - { - name: "from apk", - input: pkg.Package{ - Metadata: pkg.ApkMetadata{ - Maintainer: "auth", - }, - }, - expected: "auth", - }, - { - name: "from python - just name", - input: pkg.Package{ - Metadata: pkg.PythonPackageMetadata{ - Author: "auth", - }, - }, - expected: "auth", - }, - { - name: "from python - just email", - input: pkg.Package{ - Metadata: pkg.PythonPackageMetadata{ - AuthorEmail: "auth@auth.gov", - }, - }, - expected: "auth@auth.gov", - }, - { - name: "from python - both name and email", - input: pkg.Package{ - Metadata: pkg.PythonPackageMetadata{ - Author: "auth", - AuthorEmail: "auth@auth.gov", - }, - }, - expected: "auth ", - }, - { - name: "from rpm", - input: pkg.Package{ - Metadata: pkg.RpmdbMetadata{ - Vendor: "auth", - }, - }, - expected: "auth", - }, - { - name: "from dpkg", - input: pkg.Package{ - Metadata: pkg.DpkgMetadata{ - Maintainer: "auth", - }, - }, - expected: "auth", - }, - { - // note: since this is an optional field, no value is preferred over NONE or NOASSERTION - name: "empty", - input: pkg.Package{ - Metadata: pkg.NpmPackageJSONMetadata{ - Author: "", - }, - }, - expected: "", - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.expected, getSPDXOriginator(&test.input)) - }) - } -} - -func Test_getSPDXDescription(t *testing.T) { - tests := []struct { - name string - input pkg.Package - expected string - }{ - { - // note: since this is an optional field, no value is preferred over NONE or NOASSERTION - name: "no metadata", - input: pkg.Package{}, - expected: "", - }, - { - name: "from apk", - input: pkg.Package{ - Metadata: pkg.ApkMetadata{ - Description: "a description!", - }, - }, - expected: "a description!", - }, - { - name: "from npm", - input: pkg.Package{ - Metadata: pkg.NpmPackageJSONMetadata{ - Description: "a description!", - }, - }, - expected: "a description!", - }, - { - // note: since this is an optional field, no value is preferred over NONE or NOASSERTION - name: "empty", - input: pkg.Package{ - Metadata: pkg.NpmPackageJSONMetadata{ - Homepage: "", - }, - }, - expected: "", - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.expected, getSPDXDescription(&test.input)) - }) - } -} diff --git a/internal/presenter/packages/spdx_json_presenter.go b/internal/presenter/packages/spdx_json_presenter.go deleted file mode 100644 index a73c3080923..00000000000 --- a/internal/presenter/packages/spdx_json_presenter.go +++ /dev/null @@ -1,142 +0,0 @@ -package packages - -import ( - "encoding/json" - "fmt" - "io" - "path" - "strings" - "time" - - "github.com/anchore/syft/internal" - "github.com/anchore/syft/internal/presenter/packages/model/spdx22" - "github.com/anchore/syft/internal/spdxlicense" - "github.com/anchore/syft/internal/version" - "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/source" - "github.com/google/uuid" -) - -const anchoreNamespace = "https://anchore.com/syft" - -// SPDXJsonPresenter is a SPDX presentation object for the syft results (see https://github.com/spdx/spdx-spec) -type SPDXJsonPresenter struct { - catalog *pkg.Catalog - srcMetadata source.Metadata -} - -// NewSPDXJSONPresenter creates a new JSON presenter object for the given cataloging results. -func NewSPDXJSONPresenter(catalog *pkg.Catalog, srcMetadata source.Metadata) *SPDXJsonPresenter { - return &SPDXJsonPresenter{ - catalog: catalog, - srcMetadata: srcMetadata, - } -} - -// Present the catalog results to the given writer. -func (pres *SPDXJsonPresenter) Present(output io.Writer) error { - doc := newSPDXJsonDocument(pres.catalog, pres.srcMetadata) - - enc := json.NewEncoder(output) - // prevent > and < from being escaped in the payload - enc.SetEscapeHTML(false) - enc.SetIndent("", " ") - return enc.Encode(&doc) -} - -// newSPDXJsonDocument creates and populates a new JSON document struct that follows the SPDX 2.2 spec from the given cataloging results. -func newSPDXJsonDocument(catalog *pkg.Catalog, srcMetadata source.Metadata) spdx22.Document { - uniqueID := uuid.Must(uuid.NewRandom()) - - var name, input, identifier string - switch srcMetadata.Scheme { - case source.ImageScheme: - name = cleanSPDXName(srcMetadata.ImageMetadata.UserInput) - input = "image" - case source.DirectoryScheme: - name = cleanSPDXName(srcMetadata.Path) - input = "dir" - } - - if name != "." { - identifier = path.Join(input, fmt.Sprintf("%s-%s", name, uniqueID.String())) - } else { - identifier = path.Join(input, uniqueID.String()) - } - - namespace := path.Join(anchoreNamespace, identifier) - packages, files, relationships := newSPDXJsonElements(catalog) - - return spdx22.Document{ - Element: spdx22.Element{ - SPDXID: spdx22.ElementID("DOCUMENT").String(), - Name: name, - }, - SPDXVersion: spdx22.Version, - CreationInfo: spdx22.CreationInfo{ - Created: time.Now().UTC(), - Creators: []string{ - // note: key-value format derived from the JSON example document examples: https://github.com/spdx/spdx-spec/blob/v2.2/examples/SPDXJSONExample-v2.2.spdx.json - "Organization: Anchore, Inc", - "Tool: " + internal.ApplicationName + "-" + version.FromBuild().Version, - }, - LicenseListVersion: spdxlicense.Version, - }, - DataLicense: "CC0-1.0", - DocumentNamespace: namespace, - Packages: packages, - Files: files, - Relationships: relationships, - } -} - -func newSPDXJsonElements(catalog *pkg.Catalog) ([]spdx22.Package, []spdx22.File, []spdx22.Relationship) { - packages := make([]spdx22.Package, 0) - relationships := make([]spdx22.Relationship, 0) - files := make([]spdx22.File, 0) - - for _, p := range catalog.Sorted() { - license := getSPDXLicense(p) - packageSpdxID := spdx22.ElementID(fmt.Sprintf("Package-%+v-%s-%s", p.Type, p.Name, p.Version)).String() - - packageFiles, fileIDs, packageFileRelationships := getSPDXFiles(packageSpdxID, p) - files = append(files, packageFiles...) - - relationships = append(relationships, packageFileRelationships...) - - // note: the license concluded and declared should be the same since we are collecting license information - // from the project data itself (the installed package files). - packages = append(packages, spdx22.Package{ - Description: getSPDXDescription(p), - DownloadLocation: getSPDXDownloadLocation(p), - ExternalRefs: getSPDXExternalRefs(p), - FilesAnalyzed: false, - HasFiles: fileIDs, - Homepage: getSPDXHomepage(p), - LicenseDeclared: license, // The Declared License is what the authors of a project believe govern the package - Originator: getSPDXOriginator(p), - SourceInfo: getSPDXSourceInfo(p), - VersionInfo: p.Version, - Item: spdx22.Item{ - LicenseConcluded: license, // The Concluded License field is the license the SPDX file creator believes governs the package - Element: spdx22.Element{ - SPDXID: packageSpdxID, - Name: p.Name, - }, - }, - }) - } - - return packages, files, relationships -} - -func cleanSPDXName(name string) string { - // remove # according to specification - name = strings.ReplaceAll(name, "#", "-") - - // remove : for url construction - name = strings.ReplaceAll(name, ":", "-") - - // clean relative pathing - return path.Clean(name) -} diff --git a/internal/presenter/packages/spdx_json_presenter_test.go b/internal/presenter/packages/spdx_json_presenter_test.go deleted file mode 100644 index 98d8377593d..00000000000 --- a/internal/presenter/packages/spdx_json_presenter_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package packages - -import ( - "flag" - "regexp" - "testing" - - "github.com/anchore/syft/internal/formats/common/testutils" -) - -var updateSpdxJson = flag.Bool("update-spdx-json", false, "update the *.golden files for spdx-json presenters") - -func TestSPDXJSONDirectoryPresenter(t *testing.T) { - catalog, metadata, _ := testutils.DirectoryInput(t) - testutils.AssertPresenterAgainstGoldenSnapshot(t, - NewSPDXJSONPresenter(catalog, metadata), - *updateSpdxJson, - spdxJsonRedactor, - ) -} - -func TestSPDXJSONImagePresenter(t *testing.T) { - testImage := "image-simple" - catalog, metadata, _ := testutils.ImageInput(t, testImage) - testutils.AssertPresenterAgainstGoldenImageSnapshot(t, - NewSPDXJSONPresenter(catalog, metadata), - testImage, - *updateSpdxJson, - spdxJsonRedactor, - ) -} - -func spdxJsonRedactor(s []byte) []byte { - // each SBOM reports the time it was generated, which is not useful during snapshot testing - s = regexp.MustCompile(`"created": .*`).ReplaceAll(s, []byte("redacted")) - - // each SBOM reports a unique documentNamespace when generated, this is not useful for snapshot testing - s = regexp.MustCompile(`"documentNamespace": .*`).ReplaceAll(s, []byte("redacted")) - - // the license list will be updated periodically, the value here should not be directly tested in snapshot tests - return regexp.MustCompile(`"licenseListVersion": .*`).ReplaceAll(s, []byte("redacted")) -} From 04b26935575167d82a41bd1a0b9eec8480556fe7 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 21 Oct 2021 10:27:19 -0400 Subject: [PATCH 2/5] add new spdx22json format Signed-off-by: Alex Goodman --- internal/formats/spdx22json/decoder.go | 24 +++ internal/formats/spdx22json/decoder_test.go | 52 +++++ internal/formats/spdx22json/encoder.go | 24 +++ internal/formats/spdx22json/encoder_test.go | 45 +++++ internal/formats/spdx22json/format.go | 12 ++ .../formats/spdx22json/model/annotation.go | 21 ++ internal/formats/spdx22json/model/checksum.go | 7 + .../formats/spdx22json/model/creation_info.go | 19 ++ internal/formats/spdx22json/model/document.go | 51 +++++ internal/formats/spdx22json/model/element.go | 12 ++ .../formats/spdx22json/model/element_id.go | 37 ++++ .../spdx22json/model/external_document_ref.go | 9 + .../formats/spdx22json/model/external_ref.go | 43 ++++ internal/formats/spdx22json/model/file.go | 41 ++++ .../model/has_extracted_licensing_info.go | 14 ++ internal/formats/spdx22json/model/item.go | 22 +++ internal/formats/spdx22json/model/package.go | 53 +++++ .../model/package_verification_code.go | 23 +++ .../formats/spdx22json/model/relationship.go | 183 ++++++++++++++++++ internal/formats/spdx22json/model/snippet.go | 32 +++ .../spdx22json/model/syft_distro_data.go | 7 + .../spdx22json/model/syft_package_data.go | 115 +++++++++++ internal/formats/spdx22json/model/version.go | 3 + .../test-fixtures/image-simple/Dockerfile | 4 + .../test-fixtures/image-simple/file-1.txt | 1 + .../test-fixtures/image-simple/file-2.txt | 1 + .../TestSPDXJSONDirectoryPresenter.golden | 148 ++++++++++++++ .../TestSPDXJSONImagePresenter.golden | 140 ++++++++++++++ .../stereoscope-fixture-image-simple.golden | Bin 0 -> 15360 bytes .../formats/spdx22json/to_format_model.go | 165 ++++++++++++++++ internal/formats/spdx22json/to_syft_model.go | 65 +++++++ internal/formats/spdx22json/validator.go | 26 +++ 32 files changed, 1399 insertions(+) create mode 100644 internal/formats/spdx22json/decoder.go create mode 100644 internal/formats/spdx22json/decoder_test.go create mode 100644 internal/formats/spdx22json/encoder.go create mode 100644 internal/formats/spdx22json/encoder_test.go create mode 100644 internal/formats/spdx22json/format.go create mode 100644 internal/formats/spdx22json/model/annotation.go create mode 100644 internal/formats/spdx22json/model/checksum.go create mode 100644 internal/formats/spdx22json/model/creation_info.go create mode 100644 internal/formats/spdx22json/model/document.go create mode 100644 internal/formats/spdx22json/model/element.go create mode 100644 internal/formats/spdx22json/model/element_id.go create mode 100644 internal/formats/spdx22json/model/external_document_ref.go create mode 100644 internal/formats/spdx22json/model/external_ref.go create mode 100644 internal/formats/spdx22json/model/file.go create mode 100644 internal/formats/spdx22json/model/has_extracted_licensing_info.go create mode 100644 internal/formats/spdx22json/model/item.go create mode 100644 internal/formats/spdx22json/model/package.go create mode 100644 internal/formats/spdx22json/model/package_verification_code.go create mode 100644 internal/formats/spdx22json/model/relationship.go create mode 100644 internal/formats/spdx22json/model/snippet.go create mode 100644 internal/formats/spdx22json/model/syft_distro_data.go create mode 100644 internal/formats/spdx22json/model/syft_package_data.go create mode 100644 internal/formats/spdx22json/model/version.go create mode 100644 internal/formats/spdx22json/test-fixtures/image-simple/Dockerfile create mode 100644 internal/formats/spdx22json/test-fixtures/image-simple/file-1.txt create mode 100644 internal/formats/spdx22json/test-fixtures/image-simple/file-2.txt create mode 100644 internal/formats/spdx22json/test-fixtures/snapshot/TestSPDXJSONDirectoryPresenter.golden create mode 100644 internal/formats/spdx22json/test-fixtures/snapshot/TestSPDXJSONImagePresenter.golden create mode 100644 internal/formats/spdx22json/test-fixtures/snapshot/stereoscope-fixture-image-simple.golden create mode 100644 internal/formats/spdx22json/to_format_model.go create mode 100644 internal/formats/spdx22json/to_syft_model.go create mode 100644 internal/formats/spdx22json/validator.go diff --git a/internal/formats/spdx22json/decoder.go b/internal/formats/spdx22json/decoder.go new file mode 100644 index 00000000000..4a011b8885d --- /dev/null +++ b/internal/formats/spdx22json/decoder.go @@ -0,0 +1,24 @@ +package spdx22json + +import ( + "encoding/json" + "fmt" + "io" + + "github.com/anchore/syft/internal/formats/spdx22json/model" + "github.com/anchore/syft/syft/distro" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" +) + +func decoder(reader io.Reader) (*pkg.Catalog, *source.Metadata, *distro.Distro, source.Scope, error) { + dec := json.NewDecoder(reader) + + var doc model.Document + err := dec.Decode(&doc) + if err != nil { + return nil, nil, nil, source.UnknownScope, fmt.Errorf("unable to decode spdx-json: %w", err) + } + + return toSyftModel(doc) +} diff --git a/internal/formats/spdx22json/decoder_test.go b/internal/formats/spdx22json/decoder_test.go new file mode 100644 index 00000000000..939a8408252 --- /dev/null +++ b/internal/formats/spdx22json/decoder_test.go @@ -0,0 +1,52 @@ +package spdx22json + +import ( + "bytes" + "strings" + "testing" + + "github.com/anchore/syft/syft/source" + + "github.com/anchore/syft/internal/formats/common/testutils" + "github.com/go-test/deep" + "github.com/stretchr/testify/assert" +) + +func TestEncodeDecodeCycle(t *testing.T) { + testImage := "image-simple" + originalCatalog, originalMetadata, _ := testutils.ImageInput(t, testImage) + + var buf bytes.Buffer + assert.NoError(t, encoder(&buf, originalCatalog, &originalMetadata, nil, source.SquashedScope)) + + actualCatalog, actualMetadata, _, _, err := decoder(bytes.NewReader(buf.Bytes())) + assert.NoError(t, err) + + for _, d := range deep.Equal(originalMetadata, *actualMetadata) { + t.Errorf("metadata difference: %+v", d) + } + + actualPackages := actualCatalog.Sorted() + for idx, p := range originalCatalog.Sorted() { + if !assert.Equal(t, p.Name, actualPackages[idx].Name) { + t.Errorf("different package at idx=%d: %s vs %s", idx, p.Name, actualPackages[idx].Name) + continue + } + + // ids will never be equal + p.ID = "" + actualPackages[idx].ID = "" + + for _, d := range deep.Equal(*p, *actualPackages[idx]) { + if strings.Contains(d, ".VirtualPath: ") { + // location.Virtual path is not exposed in the json output + continue + } + if strings.HasSuffix(d, " != []") { + // semantically the same + continue + } + t.Errorf("package difference (%s): %+v", p.Name, d) + } + } +} diff --git a/internal/formats/spdx22json/encoder.go b/internal/formats/spdx22json/encoder.go new file mode 100644 index 00000000000..4e42605abaa --- /dev/null +++ b/internal/formats/spdx22json/encoder.go @@ -0,0 +1,24 @@ +package spdx22json + +import ( + "encoding/json" + "io" + + "github.com/anchore/syft/syft/distro" + + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" +) + +const anchoreNamespace = "https://anchore.com/syft" + +func encoder(output io.Writer, catalog *pkg.Catalog, srcMetadata *source.Metadata, d *distro.Distro, scope source.Scope) error { + doc := toFormatModel(catalog, srcMetadata, d, scope) + + enc := json.NewEncoder(output) + // prevent > and < from being escaped in the payload + enc.SetEscapeHTML(false) + enc.SetIndent("", " ") + + return enc.Encode(&doc) +} diff --git a/internal/formats/spdx22json/encoder_test.go b/internal/formats/spdx22json/encoder_test.go new file mode 100644 index 00000000000..61239b0506d --- /dev/null +++ b/internal/formats/spdx22json/encoder_test.go @@ -0,0 +1,45 @@ +package spdx22json + +import ( + "flag" + "regexp" + "testing" + + "github.com/anchore/syft/syft/source" + + "github.com/anchore/syft/internal/formats/common/testutils" + "github.com/anchore/syft/syft/format" +) + +var updateSpdxJson = flag.Bool("update-spdx-json", false, "update the *.golden files for spdx-json presenters") + +func TestSPDXJSONDirectoryPresenter(t *testing.T) { + catalog, metadata, distro := testutils.DirectoryInput(t) + testutils.AssertPresenterAgainstGoldenSnapshot(t, + format.NewPresenter(encoder, catalog, &metadata, distro, source.UnknownScope), + *updateSpdxJson, + spdxJsonRedactor, + ) +} + +func TestSPDXJSONImagePresenter(t *testing.T) { + testImage := "image-simple" + catalog, metadata, distro := testutils.ImageInput(t, testImage) + testutils.AssertPresenterAgainstGoldenImageSnapshot(t, + format.NewPresenter(encoder, catalog, &metadata, distro, source.SquashedScope), + testImage, + *updateSpdxJson, + spdxJsonRedactor, + ) +} + +func spdxJsonRedactor(s []byte) []byte { + // each SBOM reports the time it was generated, which is not useful during snapshot testing + s = regexp.MustCompile(`"created": .*`).ReplaceAll(s, []byte("redacted")) + + // each SBOM reports a unique documentNamespace when generated, this is not useful for snapshot testing + s = regexp.MustCompile(`"documentNamespace": .*`).ReplaceAll(s, []byte("redacted")) + + // the license list will be updated periodically, the value here should not be directly tested in snapshot tests + return regexp.MustCompile(`"licenseListVersion": .*`).ReplaceAll(s, []byte("redacted")) +} diff --git a/internal/formats/spdx22json/format.go b/internal/formats/spdx22json/format.go new file mode 100644 index 00000000000..88b4ee64156 --- /dev/null +++ b/internal/formats/spdx22json/format.go @@ -0,0 +1,12 @@ +package spdx22json + +import "github.com/anchore/syft/syft/format" + +func Format() format.Format { + return format.NewFormat( + format.SPDXJSONOption, + encoder, + decoder, + validator, + ) +} diff --git a/internal/formats/spdx22json/model/annotation.go b/internal/formats/spdx22json/model/annotation.go new file mode 100644 index 00000000000..e924cb7c678 --- /dev/null +++ b/internal/formats/spdx22json/model/annotation.go @@ -0,0 +1,21 @@ +package model + +import "time" + +type AnnotationType string + +const ( + ReviewerAnnotationType AnnotationType = "REVIEWER" + OtherAnnotationType AnnotationType = "OTHER" +) + +type Annotation struct { + // Identify when the comment was made. This is to be specified according to the combined date and time in the + // UTC format, as specified in the ISO 8601 standard. + AnnotationDate time.Time `json:"annotationDate"` + // Type of the annotation + AnnotationType AnnotationType `json:"annotationType"` + // This field identifies the person, organization or tool that has commented on a file, package, or the entire document. + Annotator string `json:"annotator"` + Comment string `json:"comment"` +} diff --git a/internal/formats/spdx22json/model/checksum.go b/internal/formats/spdx22json/model/checksum.go new file mode 100644 index 00000000000..b995a95fb9f --- /dev/null +++ b/internal/formats/spdx22json/model/checksum.go @@ -0,0 +1,7 @@ +package model + +type Checksum struct { + // Identifies the algorithm used to produce the subject Checksum. One of: "SHA256", "SHA1", "SHA384", "MD2", "MD4", "SHA512", "MD6", "MD5", "SHA224" + Algorithm string `json:"algorithm"` + ChecksumValue string `json:"checksumValue"` +} diff --git a/internal/formats/spdx22json/model/creation_info.go b/internal/formats/spdx22json/model/creation_info.go new file mode 100644 index 00000000000..c7b545d98e1 --- /dev/null +++ b/internal/formats/spdx22json/model/creation_info.go @@ -0,0 +1,19 @@ +package model + +import "time" + +type CreationInfo struct { + Comment string `json:"comment,omitempty"` + // Identify when the SPDX file was originally created. The date is to be specified according to combined date and + // time in UTC format as specified in ISO 8601 standard. This field is distinct from the fields in section 8, + // which involves the addition of information during a subsequent review. + Created time.Time `json:"created"` + // Identify who (or what, in the case of a tool) created the SPDX file. If the SPDX file was created by an + // individual, indicate the person's name. If the SPDX file was created on behalf of a company or organization, + // indicate the entity name. If the SPDX file was created using a software tool, indicate the name and version + // for that tool. If multiple participants or tools were involved, use multiple instances of this field. Person + // name or organization name may be designated as “anonymous” if appropriate. + Creators []string `json:"creators"` + // An optional field for creators of the SPDX file to provide the version of the SPDX License List used when the SPDX file was created. + LicenseListVersion string `json:"licenseListVersion"` +} diff --git a/internal/formats/spdx22json/model/document.go b/internal/formats/spdx22json/model/document.go new file mode 100644 index 00000000000..c7e9b9965ae --- /dev/null +++ b/internal/formats/spdx22json/model/document.go @@ -0,0 +1,51 @@ +package model + +import "github.com/anchore/syft/syft/source" + +// derived from: +// - https://spdx.github.io/spdx-spec/appendix-III-RDF-data-model-implementation-and-identifier-syntax/ +// - https://github.com/spdx/spdx-spec/blob/v2.2/schemas/spdx-schema.json +// - https://github.com/spdx/spdx-spec/tree/v2.2/ontology + +type Document struct { + Element + SPDXVersion string `json:"spdxVersion"` + // One instance is required for each SPDX file produced. It provides the necessary information for forward + // and backward compatibility for processing tools. + CreationInfo CreationInfo `json:"creationInfo"` + // SyftSourceData contains information about what is being described in this SPDX document (e.g. a container image, a directory, etc) + SyftSourceData *source.Metadata `json:"syftSourceData,omitempty"` + // SyftDistroData contains information about the linux distribution discovered + SyftDistroData *SyftDistroData `json:"syftDistroData,omitempty"` + // 2.2: Data License; should be "CC0-1.0" + // Cardinality: mandatory, one + // License expression for dataLicense. Compliance with the SPDX specification includes populating the SPDX + // fields therein with data related to such fields (\"SPDX-Metadata\"). The SPDX specification contains numerous + // fields where an SPDX document creator may provide relevant explanatory text in SPDX-Metadata. Without + // opining on the lawfulness of \"database rights\" (in jurisdictions where applicable), such explanatory text + // is copyrightable subject matter in most Berne Convention countries. By using the SPDX specification, or any + // portion hereof, you hereby agree that any copyright rights (as determined by your jurisdiction) in any + // SPDX-Metadata, including without limitation explanatory text, shall be subject to the terms of the Creative + // Commons CC0 1.0 Universal license. For SPDX-Metadata not containing any copyright rights, you hereby agree + // and acknowledge that the SPDX-Metadata is provided to you \"as-is\" and without any representations or + // warranties of any kind concerning the SPDX-Metadata, express, implied, statutory or otherwise, including + // without limitation warranties of title, merchantability, fitness for a particular purpose, non-infringement, + // or the absence of latent or other defects, accuracy, or the presence or absence of errors, whether or not + // discoverable, all to the greatest extent permissible under applicable law. + DataLicense string `json:"dataLicense"` + // Information about an external SPDX document reference including the checksum. This allows for verification of the external references. + ExternalDocumentRefs []ExternalDocumentRef `json:"externalDocumentRefs,omitempty"` + // Indicates that a particular ExtractedLicensingInfo was defined in the subject SpdxDocument. + HasExtractedLicensingInfos []HasExtractedLicensingInfo `json:"hasExtractedLicensingInfos,omitempty"` + // note: found in example documents from SPDX, but not in the JSON schema. See https://spdx.github.io/spdx-spec/2-document-creation-information/#25-spdx-document-namespace + DocumentNamespace string `json:"documentNamespace"` + // note: found in example documents from SPDX, but not in the JSON schema + // DocumentDescribes []string `json:"documentDescribes"` + Packages []Package `json:"packages"` + // Files referenced in the SPDX document + Files []File `json:"files,omitempty"` + // Snippets referenced in the SPDX document + Snippets []Snippet `json:"snippets,omitempty"` + // Relationships referenced in the SPDX document + Relationships []Relationship `json:"relationships,omitempty"` +} diff --git a/internal/formats/spdx22json/model/element.go b/internal/formats/spdx22json/model/element.go new file mode 100644 index 00000000000..9c2a68bcbd0 --- /dev/null +++ b/internal/formats/spdx22json/model/element.go @@ -0,0 +1,12 @@ +package model + +type Element struct { + SPDXID string `json:"SPDXID"` + // Identify name of this SpdxElement. + Name string `json:"name"` + // Relationships referenced in the SPDX document + Relationships []Relationship `json:"relationships,omitempty"` + // Provide additional information about an SpdxElement. + Annotations []Annotation `json:"annotations,omitempty"` + Comment string `json:"comment,omitempty"` +} diff --git a/internal/formats/spdx22json/model/element_id.go b/internal/formats/spdx22json/model/element_id.go new file mode 100644 index 00000000000..50251fe37ce --- /dev/null +++ b/internal/formats/spdx22json/model/element_id.go @@ -0,0 +1,37 @@ +package model + +// ElementID represents the identifier string portion of an SPDX element +// identifier. DocElementID should be used for any attributes which can +// contain identifiers defined in a different SPDX document. +// ElementIDs should NOT contain the mandatory 'SPDXRef-' portion. +type ElementID string + +func (e ElementID) String() string { + return "SPDXRef-" + string(e) +} + +// DocElementID represents an SPDX element identifier that could be defined +// in a different SPDX document, and therefore could have a "DocumentRef-" +// portion, such as Relationship and Annotations. +// ElementID is used for attributes in which a "DocumentRef-" portion cannot +// appear, such as a Package or File definition (since it is necessarily +// being defined in the present document). +// DocumentRefID will be the empty string for elements defined in the +// present document. +// DocElementIDs should NOT contain the mandatory 'DocumentRef-' or +// 'SPDXRef-' portions. +type DocElementID struct { + DocumentRefID string + ElementRefID ElementID +} + +// RenderDocElementID takes a DocElementID and returns the string equivalent, +// with the SPDXRef- prefix (and, if applicable, the DocumentRef- prefix) +// reinserted. +func (d DocElementID) String() string { + prefix := "" + if d.DocumentRefID != "" { + prefix = "DocumentRef-" + d.DocumentRefID + ":" + } + return prefix + d.ElementRefID.String() +} diff --git a/internal/formats/spdx22json/model/external_document_ref.go b/internal/formats/spdx22json/model/external_document_ref.go new file mode 100644 index 00000000000..10e1d9ec3b1 --- /dev/null +++ b/internal/formats/spdx22json/model/external_document_ref.go @@ -0,0 +1,9 @@ +package model + +type ExternalDocumentRef struct { + // externalDocumentId is a string containing letters, numbers, ., - and/or + which uniquely identifies an external document within this document. + ExternalDocumentID string `json:"externalDocumentId"` + Checksum Checksum `json:"checksum"` + // SPDX ID for SpdxDocument. A propoerty containing an SPDX document. + SpdxDocument string `json:"spdxDocument"` +} diff --git a/internal/formats/spdx22json/model/external_ref.go b/internal/formats/spdx22json/model/external_ref.go new file mode 100644 index 00000000000..9dc49b3dd90 --- /dev/null +++ b/internal/formats/spdx22json/model/external_ref.go @@ -0,0 +1,43 @@ +package model + +type ReferenceCategory string + +const ( + SecurityReferenceCategory ReferenceCategory = "SECURITY" + PackageManagerReferenceCategory ReferenceCategory = "PACKAGE_MANAGER" + OtherReferenceCategory ReferenceCategory = "OTHER" +) + +// source: https://spdx.github.io/spdx-spec/appendix-VI-external-repository-identifiers/ + +type ExternalRefType string + +const ( + // see https://nvd.nist.gov/cpe + Cpe22ExternalRefType ExternalRefType = "cpe22Type" + // see https://nvd.nist.gov/cpe + Cpe23ExternalRefType ExternalRefType = "cpe23Type" + // see http://repo1.maven.org/maven2/ + MavenCentralExternalRefType ExternalRefType = "maven-central" + // see https://www.npmjs.com/ + NpmExternalRefType ExternalRefType = "npm" + // see https://www.nuget.org/ + NugetExternalRefType ExternalRefType = "nuget" + // see http://bower.io/ + BowerExternalRefType ExternalRefType = "bower" + // see https://github.com/package-url/purl-spec + PurlExternalRefType ExternalRefType = "purl" + // These point to objects present in the Software Heritage archive by the means of SoftWare Heritage persistent Identifiers (SWHID) + SwhExternalRefType ExternalRefType = "swh" +) + +type ExternalRef struct { + Comment string `json:"comment,omitempty"` + // Category for the external reference. + ReferenceCategory ReferenceCategory `json:"referenceCategory"` + // The unique string with no spaces necessary to access the package-specific information, metadata, or content + // within the target location. The format of the locator is subject to constraints defined by the . + ReferenceLocator string `json:"referenceLocator"` + // Type of the external reference. These are defined in an appendix in the SPDX specification. + ReferenceType ExternalRefType `json:"referenceType"` +} diff --git a/internal/formats/spdx22json/model/file.go b/internal/formats/spdx22json/model/file.go new file mode 100644 index 00000000000..970f369cc58 --- /dev/null +++ b/internal/formats/spdx22json/model/file.go @@ -0,0 +1,41 @@ +package model + +type FileType string + +const ( + DocumentationFileType FileType = "DOCUMENTATION" + ImageFileType FileType = "IMAGE" + VideoFileType FileType = "VIDEO" + ArchiveFileType FileType = "ARCHIVE" + SpdxFileType FileType = "SPDX" + ApplicationFileType FileType = "APPLICATION" + SourceFileType FileType = "SOURCE" + BinaryFileType FileType = "BINARY" + TextFileType FileType = "TEXT" + AudioFileType FileType = "AUDIO" + OtherFileType FileType = "OTHER" +) + +type File struct { + Item + // (At least one is required.) The checksum property provides a mechanism that can be used to verify that the + // contents of a File or Package have not changed. + Checksums []Checksum `json:"checksums,omitempty"` + // This field provides a place for the SPDX file creator to record file contributors. Contributors could include + // names of copyright holders and/or authors who may not be copyright holders yet contributed to the file content. + FileContributors []string `json:"fileContributors,omitempty"` + // Each element is a SPDX ID for a File. + FileDependencies []string `json:"fileDependencies,omitempty"` + // The name of the file relative to the root of the package. + FileName string `json:"fileName"` + // The type of the file + FileTypes []string `json:"fileTypes,omitempty"` + // This field provides a place for the SPDX file creator to record potential legal notices found in the file. + // This may or may not include copyright statements. + NoticeText string `json:"noticeText,omitempty"` + // Indicates the project in which the SpdxElement originated. Tools must preserve doap:homepage and doap:name + // properties and the URI (if one is known) of doap:Project resources that are values of this property. All other + // properties of doap:Projects are not directly supported by SPDX and may be dropped when translating to or + // from some SPDX formats(deprecated). + ArtifactOf []string `json:"artifactOf,omitempty"` +} diff --git a/internal/formats/spdx22json/model/has_extracted_licensing_info.go b/internal/formats/spdx22json/model/has_extracted_licensing_info.go new file mode 100644 index 00000000000..8c0748073b3 --- /dev/null +++ b/internal/formats/spdx22json/model/has_extracted_licensing_info.go @@ -0,0 +1,14 @@ +package model + +type HasExtractedLicensingInfo struct { + // Verbatim license or licensing notice text that was discovered. + ExtractedText string `json:"extractedText"` + // A human readable short form license identifier for a license. The license ID is iether on the standard license + // oist or the form \"LicenseRef-\"[idString] where [idString] is a unique string containing letters, + // numbers, \".\", \"-\" or \"+\". + LicenseID string `json:"licenseId"` + Comment string `json:"comment,omitempty"` + // Identify name of this SpdxElement. + Name string `json:"name,omitempty"` + SeeAlsos []string `json:"seeAlsos,omitempty"` +} diff --git a/internal/formats/spdx22json/model/item.go b/internal/formats/spdx22json/model/item.go new file mode 100644 index 00000000000..178fbf8421e --- /dev/null +++ b/internal/formats/spdx22json/model/item.go @@ -0,0 +1,22 @@ +package model + +type Item struct { + Element + // The licenseComments property allows the preparer of the SPDX document to describe why the licensing in + // spdx:licenseConcluded was chosen. + LicenseComments string `json:"licenseComments,omitempty"` + LicenseConcluded string `json:"licenseConcluded"` + // The licensing information that was discovered directly within the package. There will be an instance of this + // property for each distinct value of alllicenseInfoInFile properties of all files contained in the package. + LicenseInfoFromFiles []string `json:"licenseInfoFromFiles,omitempty"` + // Licensing information that was discovered directly in the subject file. This is also considered a declared license for the file. + LicenseInfoInFiles []string `json:"licenseInfoInFiles,omitempty"` + // The text of copyright declarations recited in the Package or File. + CopyrightText string `json:"copyrightText,omitempty"` + // This field provides a place for the SPDX data creator to record acknowledgements that may be required to be + // communicated in some contexts. This is not meant to include the actual complete license text (see + // licenseConculded and licenseDeclared), and may or may not include copyright notices (see also copyrightText). + // The SPDX data creator may use this field to record other acknowledgements, such as particular clauses from + // license texts, which may be necessary or desirable to reproduce. + AttributionTexts []string `json:"attributionTexts,omitempty"` +} diff --git a/internal/formats/spdx22json/model/package.go b/internal/formats/spdx22json/model/package.go new file mode 100644 index 00000000000..5d303574f71 --- /dev/null +++ b/internal/formats/spdx22json/model/package.go @@ -0,0 +1,53 @@ +package model + +type Package struct { + Item + // The checksum property provides a mechanism that can be used to verify that the contents of a File or + // Package have not changed. + Checksums []Checksum `json:"checksums,omitempty"` + // Provides a detailed description of the package. + Description string `json:"description,omitempty"` + // The URI at which this package is available for download. Private (i.e., not publicly reachable) URIs are + // acceptable as values of this property. The values http://spdx.org/rdf/terms#none and http://spdx.org/rdf/terms#noassertion + // may be used to specify that the package is not downloadable or that no attempt was made to determine its + // download location, respectively. + DownloadLocation string `json:"downloadLocation,omitempty"` + // An External Reference allows a Package to reference an external source of additional information, metadata, + // enumerations, asset identifiers, or downloadable content believed to be relevant to the Package. + ExternalRefs []ExternalRef `json:"externalRefs,omitempty"` + // Indicates whether the file content of this package has been available for or subjected to analysis when + // creating the SPDX document. If false indicates packages that represent metadata or URI references to a + // project, product, artifact, distribution or a component. If set to false, the package must not contain any files + FilesAnalyzed bool `json:"filesAnalyzed"` + // Indicates that a particular file belongs to a package (elements are SPDX ID for a File). + HasFiles []string `json:"hasFiles,omitempty"` + // Provide a place for the SPDX file creator to record a web site that serves as the package's home page. + // This link can also be used to reference further information about the package referenced by the SPDX file creator. + Homepage string `json:"homepage,omitempty"` + // List the licenses that have been declared by the authors of the package. Any license information that does not + // originate from the package authors, e.g. license information from a third party repository, should not be included in this field. + LicenseDeclared string `json:"licenseDeclared"` + // The name and, optionally, contact information of the person or organization that originally created the package. + // Values of this property must conform to the agent and tool syntax. + Originator string `json:"originator,omitempty"` + // The base name of the package file name. For example, zlib-1.2.5.tar.gz. + PackageFileName string `json:"packageFileName,omitempty"` + // A manifest based verification code (the algorithm is defined in section 4.7 of the full specification) of the + // SPDX Item. This allows consumers of this data and/or database to determine if an SPDX item they have in hand + // is identical to the SPDX item from which the data was produced. This algorithm works even if the SPDX document + // is included in the SPDX item. + PackageVerificationCode *PackageVerificationCode `json:"packageVerificationCode,omitempty"` + // Allows the producer(s) of the SPDX document to describe how the package was acquired and/or changed from the original source. + SourceInfo string `json:"sourceInfo,omitempty"` + // Provides a short description of the package. + Summary string `json:"summary,omitempty"` + // The name and, optionally, contact information of the person or organization who was the immediate supplier + // of this package to the recipient. The supplier may be different than originator when the software has been + // repackaged. Values of this property must conform to the agent and tool syntax. + Supplier string `json:"supplier,omitempty"` + // Provides an indication of the version of the package that is described by this SpdxDocument. + VersionInfo string `json:"versionInfo,omitempty"` + // SyftPackageData provides a spot to add syft-specific data that is not available in-spec or is not easily decodable from + // other SPDX fields. + SyftPackageData *SyftPackageData `json:"syftPackageData,omitempty"` +} diff --git a/internal/formats/spdx22json/model/package_verification_code.go b/internal/formats/spdx22json/model/package_verification_code.go new file mode 100644 index 00000000000..508c9169ed0 --- /dev/null +++ b/internal/formats/spdx22json/model/package_verification_code.go @@ -0,0 +1,23 @@ +package model + +// Why are there two package identifier fields Package Checksum and Package Verification? +// Although the values of the two fields Package Checksum and Package Verification are similar, they each serve a +// different purpose. The Package Checksum provides a unique identifier of a software package which is computed by +// taking the SHA1 of the entire software package file. This enables one to quickly determine if two different copies +// of a package are the same. One disadvantage of this approach is that one cannot add an SPDX data file into the +// original package without changing the Package Checksum value. Alternatively, the Package Verification field enables +// the inclusion of an SPDX file. It enables one to quickly verify if one or more of the original package files has +// changed. The Package Verification field is a unique identifier that is based on SHAing only the original package +// files (e.g., excluding the SPDX file). This allows one to add an SPDX file to the original package without changing +// this unique identifier. +// source: https://wiki.spdx.org/view/SPDX_FAQ +type PackageVerificationCode struct { + // "A file that was excluded when calculating the package verification code. This is usually a file containing + // SPDX data regarding the package. If a package contains more than one SPDX file all SPDX files must be excluded + // from the package verification code. If this is not done it would be impossible to correctly calculate the + // verification codes in both files. + PackageVerificationCodeExcludedFiles []string `json:"packageVerificationCodeExcludedFiles"` + + // The actual package verification code as a hex encoded value. + PackageVerificationCodeValue string `json:"packageVerificationCodeValue"` +} diff --git a/internal/formats/spdx22json/model/relationship.go b/internal/formats/spdx22json/model/relationship.go new file mode 100644 index 00000000000..51c52233b14 --- /dev/null +++ b/internal/formats/spdx22json/model/relationship.go @@ -0,0 +1,183 @@ +package model + +type Relationship struct { + // Id to which the SPDX element is related + SpdxElementID string `json:"spdxElementId"` + // Describes the type of relationship between two SPDX elements. + RelationshipType RelationshipType `json:"relationshipType"` + // SPDX ID for SpdxElement. A related SpdxElement. + RelatedSpdxElement string `json:"relatedSpdxElement"` + Comment string `json:"comment,omitempty"` +} + +// source: https://spdx.github.io/spdx-spec/7-relationships-between-SPDX-elements/ +type RelationshipType string + +const ( + // DescribedByRelationship is to be used when SPDXRef-A is described by SPDXREF-Document. + // Example: The package 'WildFly' is described by SPDX document WildFly.spdx. + DescribedByRelationship RelationshipType = "DESCRIBED_BY" + + // ContainsRelationship is to be used when SPDXRef-A contains SPDXRef-B. + // Example: An ARCHIVE file bar.tgz contains a SOURCE file foo.c. + ContainsRelationship RelationshipType = "CONTAINS" + + // ContainedByRelationship is to be used when SPDXRef-A is contained by SPDXRef-B. + // Example: A SOURCE file foo.c is contained by ARCHIVE file bar.tgz + ContainedByRelationship RelationshipType = "CONTAINED_BY" + + // DependsOnRelationship is to be used when SPDXRef-A depends on SPDXRef-B. + // Example: Package A depends on the presence of package B in order to build and run + DependsOnRelationship RelationshipType = "DEPENDS_ON" + + // DependencyOfRelationship is to be used when SPDXRef-A is dependency of SPDXRef-B. + // Example: A is explicitly stated as a dependency of B in a machine-readable file. Use when a package manager does not define scopes. + DependencyOfRelationship RelationshipType = "DEPENDENCY_OF" + + // DependencyManifestOfRelationship is to be used when SPDXRef-A is a manifest file that lists a set of dependencies for SPDXRef-B. + // Example: A file package.json is the dependency manifest of a package foo. Note that only one manifest should be used to define the same dependency graph. + DependencyManifestOfRelationship RelationshipType = "DEPENDENCY_MANIFEST_OF" + + // BuildDependencyOfRelationship is to be used when SPDXRef-A is a build dependency of SPDXRef-B. + // Example: A is in the compile scope of B in a Maven project. + BuildDependencyOfRelationship RelationshipType = "BUILD_DEPENDENCY_OF" + + // DevDependencyOfRelationship is to be used when SPDXRef-A is a development dependency of SPDXRef-B. + // Example: A is in the devDependencies scope of B in a Maven project. + DevDependencyOfRelationship RelationshipType = "DEV_DEPENDENCY_OF" + + // OptionalDependencyOfRelationship is to be used when SPDXRef-A is an optional dependency of SPDXRef-B. + // Example: Use when building the code will proceed even if a dependency cannot be found, fails to install, or is only installed on a specific platform. For example, A is in the optionalDependencies scope of npm project B. + OptionalDependencyOfRelationship RelationshipType = "OPTIONAL_DEPENDENCY_OF" + + // ProvidedDependencyOfRelationship is to be used when SPDXRef-A is a to be provided dependency of SPDXRef-B. + // Example: A is in the provided scope of B in a Maven project, indicating that the project expects it to be provided, for instance, by the container or JDK. + ProvidedDependencyOfRelationship RelationshipType = "PROVIDED_DEPENDENCY_OF" + + // TestDependencyOfRelationship is to be used when SPDXRef-A is a test dependency of SPDXRef-B. + // Example: A is in the test scope of B in a Maven project. + TestDependencyOfRelationship RelationshipType = "TEST_DEPENDENCY_OF" + + // RuntimeDependencyOfRelationship is to be used when SPDXRef-A is a dependency required for the execution of SPDXRef-B. + // Example: A is in the runtime scope of B in a Maven project. + RuntimeDependencyOfRelationship RelationshipType = "RUNTIME_DEPENDENCY_OF" + + // ExampleOfRelationship is to be used when SPDXRef-A is an example of SPDXRef-B. + // Example: The file or snippet that illustrates how to use an application or library. + ExampleOfRelationship RelationshipType = "EXAMPLE_OF" + + // GeneratesRelationship is to be used when SPDXRef-A generates SPDXRef-B. + // Example: A SOURCE file makefile.mk generates a BINARY file a.out + GeneratesRelationship RelationshipType = "GENERATES" + + // GeneratedFromRelationship is to be used when SPDXRef-A was generated from SPDXRef-B. + // Example: A BINARY file a.out has been generated from a SOURCE file makefile.mk. A BINARY file foolib.a is generated from a SOURCE file bar.c. + GeneratedFromRelationship RelationshipType = "GENERATED_FROM" + + // AncestorOfRelationship is to be used when SPDXRef-A is an ancestor (same lineage but pre-dates) SPDXRef-B. + // Example: A SOURCE file makefile.mk is a version of the original ancestor SOURCE file 'makefile2.mk' + AncestorOfRelationship RelationshipType = "ANCESTOR_OF" + + // DescendantOfRelationship is to be used when SPDXRef-A is a descendant of (same lineage but postdates) SPDXRef-B. + // Example: A SOURCE file makefile2.mk is a descendant of the original SOURCE file 'makefile.mk' + DescendantOfRelationship RelationshipType = "DESCENDANT_OF" + + // VariantOfRelationship is to be used when SPDXRef-A is a variant of (same lineage but not clear which came first) SPDXRef-B. + // Example: A SOURCE file makefile2.mk is a variant of SOURCE file makefile.mk if they differ by some edit, but there is no way to tell which came first (no reliable date information). + VariantOfRelationship RelationshipType = "VARIANT_OF" + + // DistributionArtifactRelationship is to be used when distributing SPDXRef-A requires that SPDXRef-B also be distributed. + // Example: A BINARY file foo.o requires that the ARCHIVE file bar-sources.tgz be made available on distribution. + DistributionArtifactRelationship RelationshipType = "DISTRIBUTION_ARTIFACT" + + // PatchForRelationship is to be used when SPDXRef-A is a patch file for (to be applied to) SPDXRef-B. + // Example: A SOURCE file foo.diff is a patch file for SOURCE file foo.c. + PatchForRelationship RelationshipType = "PATCH_FOR" + + // PatchAppliedRelationship is to be used when SPDXRef-A is a patch file that has been applied to SPDXRef-B. + // Example: A SOURCE file foo.diff is a patch file that has been applied to SOURCE file 'foo-patched.c'. + PatchAppliedRelationship RelationshipType = "PATCH_APPLIED" + + // CopyOfRelationship is to be used when SPDXRef-A is an exact copy of SPDXRef-B. + // Example: A BINARY file alib.a is an exact copy of BINARY file a2lib.a. + CopyOfRelationship RelationshipType = "COPY_OF" + + // FileAddedRelationship is to be used when SPDXRef-A is a file that was added to SPDXRef-B. + // Example: A SOURCE file foo.c has been added to package ARCHIVE bar.tgz. + FileAddedRelationship RelationshipType = "FILE_ADDED" + + // FileDeletedRelationship is to be used when SPDXRef-A is a file that was deleted from SPDXRef-B. + // Example: A SOURCE file foo.diff has been deleted from package ARCHIVE bar.tgz. + FileDeletedRelationship RelationshipType = "FILE_DELETED" + + // FileModifiedRelationship is to be used when SPDXRef-A is a file that was modified from SPDXRef-B. + // Example: A SOURCE file foo.c has been modified from SOURCE file foo.orig.c. + FileModifiedRelationship RelationshipType = "FILE_MODIFIED" + + // ExpandedFromArchiveRelationship is to be used when SPDXRef-A is expanded from the archive SPDXRef-B. + // Example: A SOURCE file foo.c, has been expanded from the archive ARCHIVE file xyz.tgz. + ExpandedFromArchiveRelationship RelationshipType = "EXPANDED_FROM_ARCHIVE" + + // DynamicLinkRelationship is to be used when SPDXRef-A dynamically links to SPDXRef-B. + // Example: An APPLICATION file 'myapp' dynamically links to BINARY file zlib.so. + DynamicLinkRelationship RelationshipType = "DYNAMIC_LINK" + + // StaticLinkRelationship is to be used when SPDXRef-A statically links to SPDXRef-B. + // Example: An APPLICATION file 'myapp' statically links to BINARY zlib.a. + StaticLinkRelationship RelationshipType = "STATIC_LINK" + + // DataFileOfRelationship is to be used when SPDXRef-A is a data file used in SPDXRef-B. + // Example: An IMAGE file 'kitty.jpg' is a data file of an APPLICATION 'hellokitty'. + DataFileOfRelationship RelationshipType = "DATA_FILE_OF" + + // TestCaseOfRelationship is to be used when SPDXRef-A is a test case used in testing SPDXRef-B. + // Example: A SOURCE file testMyCode.java is a unit test file used to test an APPLICATION MyPackage. + TestCaseOfRelationship RelationshipType = "TEST_CASE_OF" + + // BuildToolOfRelationship is to be used when SPDXRef-A is used to build SPDXRef-B. + // Example: A SOURCE file makefile.mk is used to build an APPLICATION 'zlib'. + BuildToolOfRelationship RelationshipType = "BUILD_TOOL_OF" + + // DevToolOfRelationship is to be used when SPDXRef-A is used as a development tool for SPDXRef-B. + // Example: Any tool used for development such as a code debugger. + DevToolOfRelationship RelationshipType = "DEV_TOOL_OF" + + // TestOfRelationship is to be used when SPDXRef-A is used for testing SPDXRef-B. + // Example: Generic relationship for cases where it's clear that something is used for testing but unclear whether it's TEST_CASE_OF or TEST_TOOL_OF. + TestOfRelationship RelationshipType = "TEST_OF" + + // TestToolOfRelationship is to be used when SPDXRef-A is used as a test tool for SPDXRef-B. + // Example: Any tool used to test the code such as ESlint. + TestToolOfRelationship RelationshipType = "TEST_TOOL_OF" + + // DocumentationOfRelationship is to be used when SPDXRef-A provides documentation of SPDXRef-B. + // Example: A DOCUMENTATION file readme.txt documents the APPLICATION 'zlib'. + DocumentationOfRelationship RelationshipType = "DOCUMENTATION_OF" + + // OptionalComponentOfRelationship is to be used when SPDXRef-A is an optional component of SPDXRef-B. + // Example: A SOURCE file fool.c (which is in the contributors directory) may or may not be included in the build of APPLICATION 'atthebar'. + OptionalComponentOfRelationship RelationshipType = "OPTIONAL_COMPONENT_OF" + + // MetafileOfRelationship is to be used when SPDXRef-A is a metafile of SPDXRef-B. + // Example: A SOURCE file pom.xml is a metafile of the APPLICATION 'Apache Xerces'. + MetafileOfRelationship RelationshipType = "METAFILE_OF" + + // PackageOfRelationship is to be used when SPDXRef-A is used as a package as part of SPDXRef-B. + // Example: A Linux distribution contains an APPLICATION package gawk as part of the distribution MyLinuxDistro. + PackageOfRelationship RelationshipType = "PACKAGE_OF" + + // AmendsRelationship is to be used when (current) SPDXRef-DOCUMENT amends the SPDX information in SPDXRef-B. + // Example: (Current) SPDX document A version 2 contains a correction to a previous version of the SPDX document A version 1. Note the reserved identifier SPDXRef-DOCUMENT for the current document is required. + AmendsRelationship RelationshipType = "AMENDS" + + // PrerequisiteForRelationship is to be used when SPDXRef-A is a prerequisite for SPDXRef-B. + // Example: A library bar.dll is a prerequisite or dependency for APPLICATION foo.exe + PrerequisiteForRelationship RelationshipType = "PREREQUISITE_FOR" + + // HasPrerequisiteRelationship is to be used when SPDXRef-A has as a prerequisite SPDXRef-B. + // Example: An APPLICATION foo.exe has prerequisite or dependency on bar.dll + HasPrerequisiteRelationship RelationshipType = "HAS_PREREQUISITE" + + // OtherRelationship is to be used for a relationship which has not been defined in the formal SPDX specification. A description of the relationship should be included in the Relationship comments field. + OtherRelationship RelationshipType = "OTHER" +) diff --git a/internal/formats/spdx22json/model/snippet.go b/internal/formats/spdx22json/model/snippet.go new file mode 100644 index 00000000000..0d39e5dca39 --- /dev/null +++ b/internal/formats/spdx22json/model/snippet.go @@ -0,0 +1,32 @@ +package model + +type StartPointer struct { + Offset int `json:"offset,omitempty"` + LineNumber int `json:"lineNumber,omitempty"` + // SPDX ID for File + Reference string `json:"reference"` +} + +type EndPointer struct { + Offset int `json:"offset,omitempty"` + LineNumber int `json:"lineNumber,omitempty"` + // SPDX ID for File + Reference string `json:"reference"` +} + +type Range struct { + StartPointer StartPointer `json:"startPointer"` + EndPointer EndPointer `json:"endPointer"` +} + +type Snippet struct { + Item + // Licensing information that was discovered directly in the subject snippet. This is also considered a declared + // license for the snippet. (elements are license expressions) + LicenseInfoInSnippets []string `json:"licenseInfoInSnippets"` + // SPDX ID for File. File containing the SPDX element (e.g. the file contaning a snippet). + SnippetFromFile string `json:"snippetFromFile"` + // (At least 1 range is required). This field defines the byte range in the original host file (in X.2) that the + // snippet information applies to. + Ranges []Range `json:"ranges"` +} diff --git a/internal/formats/spdx22json/model/syft_distro_data.go b/internal/formats/spdx22json/model/syft_distro_data.go new file mode 100644 index 00000000000..8f985bf2661 --- /dev/null +++ b/internal/formats/spdx22json/model/syft_distro_data.go @@ -0,0 +1,7 @@ +package model + +type SyftDistroData struct { + Name string `json:"name"` // Name of the Linux distribution + Version string `json:"version"` // Version of the Linux distribution (major or major.minor version) + IDLike string `json:"idLike"` // the ID_LIKE field found within the /etc/os-release file +} diff --git a/internal/formats/spdx22json/model/syft_package_data.go b/internal/formats/spdx22json/model/syft_package_data.go new file mode 100644 index 00000000000..92d99509cf1 --- /dev/null +++ b/internal/formats/spdx22json/model/syft_package_data.go @@ -0,0 +1,115 @@ +package model + +import ( + "encoding/json" + "fmt" + + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" +) + +type SyftPackageData struct { + SyftPackageBasicData + SyftPackageCustomData +} + +type SyftPackageCustomData struct { + MetadataType pkg.MetadataType `json:"metadataType,omitempty"` + Metadata interface{} `json:"metadata,omitempty"` +} + +type SyftPackageBasicData struct { + PackageType pkg.Type `json:"type,omitempty"` + FoundBy string `json:"foundBy,omitempty"` + Locations []source.Location `json:"locations,omitempty"` + Licenses []string `json:"licenses,omitempty"` + Language pkg.Language `json:"language,omitempty"` +} + +// syftPackageMetadataUnpacker is all values needed from Package to disambiguate ambiguous fields during json unmarshaling. +type syftPackageMetadataUnpacker struct { + MetadataType pkg.MetadataType `json:"metadataType"` + Metadata json.RawMessage `json:"metadata"` +} + +func (p *syftPackageMetadataUnpacker) String() string { + return fmt.Sprintf("metadataType: %s, metadata: %s", p.MetadataType, string(p.Metadata)) +} + +// UnmarshalJSON is a custom unmarshaller for handling basic values and values with ambiguous types. +// nolint:funlen +func (p *SyftPackageData) UnmarshalJSON(b []byte) error { + var basic SyftPackageBasicData + if err := json.Unmarshal(b, &basic); err != nil { + return err + } + p.SyftPackageBasicData = basic + + var unpacker syftPackageMetadataUnpacker + if err := json.Unmarshal(b, &unpacker); err != nil { + log.Warnf("failed to unmarshall into syftPackageMetadataUnpacker: %v", err) + return err + } + + p.MetadataType = unpacker.MetadataType + + switch p.MetadataType { + case pkg.ApkMetadataType: + var payload pkg.ApkMetadata + if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { + return err + } + p.Metadata = payload + case pkg.RpmdbMetadataType: + var payload pkg.RpmdbMetadata + if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { + return err + } + p.Metadata = payload + case pkg.DpkgMetadataType: + var payload pkg.DpkgMetadata + if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { + return err + } + p.Metadata = payload + case pkg.JavaMetadataType: + var payload pkg.JavaMetadata + if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { + return err + } + p.Metadata = payload + case pkg.RustCargoPackageMetadataType: + var payload pkg.CargoPackageMetadata + if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { + return err + } + p.Metadata = payload + case pkg.GemMetadataType: + var payload pkg.GemMetadata + if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { + return err + } + p.Metadata = payload + case pkg.KbPackageMetadataType: + var payload pkg.KbPackageMetadata + if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { + return err + } + p.Metadata = payload + case pkg.PythonPackageMetadataType: + var payload pkg.PythonPackageMetadata + if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { + return err + } + p.Metadata = payload + case pkg.NpmPackageJSONMetadataType: + var payload pkg.NpmPackageJSONMetadata + if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { + return err + } + p.Metadata = payload + } + + return nil +} diff --git a/internal/formats/spdx22json/model/version.go b/internal/formats/spdx22json/model/version.go new file mode 100644 index 00000000000..8f105cbfa71 --- /dev/null +++ b/internal/formats/spdx22json/model/version.go @@ -0,0 +1,3 @@ +package model + +const Version = "SPDX-2.2" diff --git a/internal/formats/spdx22json/test-fixtures/image-simple/Dockerfile b/internal/formats/spdx22json/test-fixtures/image-simple/Dockerfile new file mode 100644 index 00000000000..79cfa759e35 --- /dev/null +++ b/internal/formats/spdx22json/test-fixtures/image-simple/Dockerfile @@ -0,0 +1,4 @@ +# Note: changes to this file will result in updating several test values. Consider making a new image fixture instead of editing this one. +FROM scratch +ADD file-1.txt /somefile-1.txt +ADD file-2.txt /somefile-2.txt diff --git a/internal/formats/spdx22json/test-fixtures/image-simple/file-1.txt b/internal/formats/spdx22json/test-fixtures/image-simple/file-1.txt new file mode 100644 index 00000000000..985d3408e98 --- /dev/null +++ b/internal/formats/spdx22json/test-fixtures/image-simple/file-1.txt @@ -0,0 +1 @@ +this file has contents \ No newline at end of file diff --git a/internal/formats/spdx22json/test-fixtures/image-simple/file-2.txt b/internal/formats/spdx22json/test-fixtures/image-simple/file-2.txt new file mode 100644 index 00000000000..396d08bbc72 --- /dev/null +++ b/internal/formats/spdx22json/test-fixtures/image-simple/file-2.txt @@ -0,0 +1 @@ +file-2 contents! \ No newline at end of file diff --git a/internal/formats/spdx22json/test-fixtures/snapshot/TestSPDXJSONDirectoryPresenter.golden b/internal/formats/spdx22json/test-fixtures/snapshot/TestSPDXJSONDirectoryPresenter.golden new file mode 100644 index 00000000000..33ae5998927 --- /dev/null +++ b/internal/formats/spdx22json/test-fixtures/snapshot/TestSPDXJSONDirectoryPresenter.golden @@ -0,0 +1,148 @@ +{ + "SPDXID": "SPDXRef-DOCUMENT", + "name": "/some/path", + "spdxVersion": "SPDX-2.2", + "creationInfo": { + "created": "2021-10-21T14:23:51.664982Z", + "creators": [ + "Organization: Anchore, Inc", + "Tool: syft-[not provided]" + ], + "licenseListVersion": "3.14" + }, + "syftSourceData": { + "Scheme": "DirectoryScheme", + "ImageMetadata": { + "userInput": "", + "imageID": "", + "manifestDigest": "", + "mediaType": "", + "tags": null, + "imageSize": 0, + "layers": null, + "manifest": null, + "config": null, + "repoDigests": null + }, + "Path": "/some/path" + }, + "syftDistroData": { + "name": "debian", + "version": "1.2.3", + "idLike": "like!" + }, + "dataLicense": "CC0-1.0", + "documentNamespace": "https:/anchore.com/syft/dir/some/path-df32db3e-c495-4f3b-9304-a84e1cc22fc7", + "packages": [ + { + "SPDXID": "SPDXRef-Package-python-package-1-1.0.1", + "name": "package-1", + "licenseConcluded": "MIT", + "downloadLocation": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:*:some:package:2:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + }, + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "a-purl-2", + "referenceType": "purl" + } + ], + "filesAnalyzed": false, + "hasFiles": [ + "SPDXRef-File-package-1-efae7fecc76ca25da40f79d7ef5b8933510434914835832c7976f3e866aa756a" + ], + "licenseDeclared": "MIT", + "sourceInfo": "acquired package info from installed python package manifest file: /some/path/pkg1", + "versionInfo": "1.0.1", + "syftPackageData": { + "type": "python", + "foundBy": "the-cataloger-1", + "locations": [ + { + "path": "/some/path/pkg1" + } + ], + "licenses": [ + "MIT" + ], + "language": "python", + "metadataType": "PythonPackageMetadata", + "metadata": { + "name": "package-1", + "version": "1.0.1", + "license": "", + "author": "", + "authorEmail": "", + "platform": "", + "files": [ + { + "path": "/some/path/pkg1/dependencies/foo" + } + ], + "sitePackagesRootPath": "" + } + } + }, + { + "SPDXID": "SPDXRef-Package-deb-package-2-2.0.1", + "name": "package-2", + "licenseConcluded": "NONE", + "downloadLocation": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:*:some:package:2:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + }, + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "a-purl-2", + "referenceType": "purl" + } + ], + "filesAnalyzed": false, + "licenseDeclared": "NONE", + "sourceInfo": "acquired package info from DPKG DB: /some/path/pkg1", + "versionInfo": "2.0.1", + "syftPackageData": { + "type": "deb", + "foundBy": "the-cataloger-2", + "locations": [ + { + "path": "/some/path/pkg1" + } + ], + "metadataType": "DpkgMetadata", + "metadata": { + "package": "package-2", + "source": "", + "version": "2.0.1", + "sourceVersion": "", + "architecture": "", + "maintainer": "", + "installedSize": 0, + "files": null + } + } + } + ], + "files": [ + { + "SPDXID": "SPDXRef-File-package-1-efae7fecc76ca25da40f79d7ef5b8933510434914835832c7976f3e866aa756a", + "name": "foo", + "licenseConcluded": "", + "fileName": "/some/path/pkg1/dependencies/foo" + } + ], + "relationships": [ + { + "spdxElementId": "SPDXRef-Package-python-package-1-1.0.1", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-File-package-1-efae7fecc76ca25da40f79d7ef5b8933510434914835832c7976f3e866aa756a" + } + ] +} diff --git a/internal/formats/spdx22json/test-fixtures/snapshot/TestSPDXJSONImagePresenter.golden b/internal/formats/spdx22json/test-fixtures/snapshot/TestSPDXJSONImagePresenter.golden new file mode 100644 index 00000000000..1e7b1f71df2 --- /dev/null +++ b/internal/formats/spdx22json/test-fixtures/snapshot/TestSPDXJSONImagePresenter.golden @@ -0,0 +1,140 @@ +{ + "SPDXID": "SPDXRef-DOCUMENT", + "name": "user-image-input", + "spdxVersion": "SPDX-2.2", + "creationInfo": { + "created": "2021-10-21T14:23:51.669903Z", + "creators": [ + "Organization: Anchore, Inc", + "Tool: syft-[not provided]" + ], + "licenseListVersion": "3.14" + }, + "syftSourceData": { + "Scheme": "ImageScheme", + "ImageMetadata": { + "userInput": "user-image-input", + "imageID": "sha256:2480160b55bec40c44d3b145c7b2c1c47160db8575c3dcae086d76b9370ae7ca", + "manifestDigest": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368", + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "tags": [ + "stereoscope-fixture-image-simple:85066c51088bdd274f7a89e99e00490f666c49e72ffc955707cd6e18f0e22c5b" + ], + "imageSize": 38, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "digest": "sha256:fb6beecb75b39f4bb813dbf177e501edd5ddb3e69bb45cedeb78c676ee1b7a59", + "size": 22 + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "digest": "sha256:319b588ce64253a87b533c8ed01cf0025e0eac98e7b516e12532957e1244fdec", + "size": 16 + } + ], + "manifest": "eyJzY2hlbWFWZXJzaW9uIjoyLCJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmRpc3RyaWJ1dGlvbi5tYW5pZmVzdC52Mitqc29uIiwiY29uZmlnIjp7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuY29udGFpbmVyLmltYWdlLnYxK2pzb24iLCJzaXplIjo2NjcsImRpZ2VzdCI6InNoYTI1NjoyNDgwMTYwYjU1YmVjNDBjNDRkM2IxNDVjN2IyYzFjNDcxNjBkYjg1NzVjM2RjYWUwODZkNzZiOTM3MGFlN2NhIn0sImxheWVycyI6W3sibWVkaWFUeXBlIjoiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsInNpemUiOjIwNDgsImRpZ2VzdCI6InNoYTI1NjpmYjZiZWVjYjc1YjM5ZjRiYjgxM2RiZjE3N2U1MDFlZGQ1ZGRiM2U2OWJiNDVjZWRlYjc4YzY3NmVlMWI3YTU5In0seyJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwic2l6ZSI6MjA0OCwiZGlnZXN0Ijoic2hhMjU2OjMxOWI1ODhjZTY0MjUzYTg3YjUzM2M4ZWQwMWNmMDAyNWUwZWFjOThlN2I1MTZlMTI1MzI5NTdlMTI0NGZkZWMifV19", + "config": "eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZpZyI6eyJFbnYiOlsiUEFUSD0vdXNyL2xvY2FsL3NiaW46L3Vzci9sb2NhbC9iaW46L3Vzci9zYmluOi91c3IvYmluOi9zYmluOi9iaW4iXSwiV29ya2luZ0RpciI6Ii8iLCJPbkJ1aWxkIjpudWxsfSwiY3JlYXRlZCI6IjIwMjEtMTAtMDRUMTE6NDA6MDAuNjM4Mzk0NVoiLCJoaXN0b3J5IjpbeyJjcmVhdGVkIjoiMjAyMS0xMC0wNFQxMTo0MDowMC41OTA3MzE2WiIsImNyZWF0ZWRfYnkiOiJBREQgZmlsZS0xLnR4dCAvc29tZWZpbGUtMS50eHQgIyBidWlsZGtpdCIsImNvbW1lbnQiOiJidWlsZGtpdC5kb2NrZXJmaWxlLnYwIn0seyJjcmVhdGVkIjoiMjAyMS0xMC0wNFQxMTo0MDowMC42MzgzOTQ1WiIsImNyZWF0ZWRfYnkiOiJBREQgZmlsZS0yLnR4dCAvc29tZWZpbGUtMi50eHQgIyBidWlsZGtpdCIsImNvbW1lbnQiOiJidWlsZGtpdC5kb2NrZXJmaWxlLnYwIn1dLCJvcyI6ImxpbnV4Iiwicm9vdGZzIjp7InR5cGUiOiJsYXllcnMiLCJkaWZmX2lkcyI6WyJzaGEyNTY6ZmI2YmVlY2I3NWIzOWY0YmI4MTNkYmYxNzdlNTAxZWRkNWRkYjNlNjliYjQ1Y2VkZWI3OGM2NzZlZTFiN2E1OSIsInNoYTI1NjozMTliNTg4Y2U2NDI1M2E4N2I1MzNjOGVkMDFjZjAwMjVlMGVhYzk4ZTdiNTE2ZTEyNTMyOTU3ZTEyNDRmZGVjIl19fQ==", + "repoDigests": [] + }, + "Path": "" + }, + "syftDistroData": { + "name": "debian", + "version": "1.2.3", + "idLike": "like!" + }, + "dataLicense": "CC0-1.0", + "documentNamespace": "https:/anchore.com/syft/image/user-image-input-a5c175cf-bfd5-47fb-9604-f6106415d5bb", + "packages": [ + { + "SPDXID": "SPDXRef-Package-python-package-1-1.0.1", + "name": "package-1", + "licenseConcluded": "MIT", + "downloadLocation": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:*:some:package:1:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + }, + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "a-purl-1", + "referenceType": "purl" + } + ], + "filesAnalyzed": false, + "licenseDeclared": "MIT", + "sourceInfo": "acquired package info from installed python package manifest file: /somefile-1.txt", + "versionInfo": "1.0.1", + "syftPackageData": { + "type": "python", + "foundBy": "the-cataloger-1", + "locations": [ + { + "path": "/somefile-1.txt", + "layerID": "sha256:fb6beecb75b39f4bb813dbf177e501edd5ddb3e69bb45cedeb78c676ee1b7a59" + } + ], + "licenses": [ + "MIT" + ], + "language": "python", + "metadataType": "PythonPackageMetadata", + "metadata": { + "name": "package-1", + "version": "1.0.1", + "license": "", + "author": "", + "authorEmail": "", + "platform": "", + "sitePackagesRootPath": "" + } + } + }, + { + "SPDXID": "SPDXRef-Package-deb-package-2-2.0.1", + "name": "package-2", + "licenseConcluded": "NONE", + "downloadLocation": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:*:some:package:2:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + }, + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "a-purl-2", + "referenceType": "purl" + } + ], + "filesAnalyzed": false, + "licenseDeclared": "NONE", + "sourceInfo": "acquired package info from DPKG DB: /somefile-2.txt", + "versionInfo": "2.0.1", + "syftPackageData": { + "type": "deb", + "foundBy": "the-cataloger-2", + "locations": [ + { + "path": "/somefile-2.txt", + "layerID": "sha256:319b588ce64253a87b533c8ed01cf0025e0eac98e7b516e12532957e1244fdec" + } + ], + "metadataType": "DpkgMetadata", + "metadata": { + "package": "package-2", + "source": "", + "version": "2.0.1", + "sourceVersion": "", + "architecture": "", + "maintainer": "", + "installedSize": 0, + "files": null + } + } + } + ] +} diff --git a/internal/formats/spdx22json/test-fixtures/snapshot/stereoscope-fixture-image-simple.golden b/internal/formats/spdx22json/test-fixtures/snapshot/stereoscope-fixture-image-simple.golden new file mode 100644 index 0000000000000000000000000000000000000000..c1b1d2b797ecd34a5276a1aa2fb18c5b0a58c732 GIT binary patch literal 15360 zcmeHO+iu%N5Y=ZaKc z4r#BxuDkuJ7{OBcm#%FzyUOYr8lw^9HIDjR+4F_Fg0dOV`$VGKgH38I>Hgnpur)R2 zl)HY_XM^MD!m4A0dKhDcAS?&*f(4T;Kn>ke#l4gzVN` ze(?Rjwf#RFzkC1s- z9>CiEk8VKl|Mzk`OrxxEw%gtY==qIJx$uy+*ED8?qX(EcDL{I387Ngsb4dh?x$wg9 z*h*rg6@ZQ(w@kA*(GHyphO!?TNJz`YJ>fe4rt;(!W2awfa(sRRC68dzXajVE@y-n>o(^op5E2ZJDVs7 z4hRGU0s;YnXN7=m8LaJu|D{?6{&zpONo)J>g#RTf@W1=9f8c-jgY$EprS<=wcn6J zY-{-4@PCvC{%=1g#vT7ROM=?XoSXT-=K0?$|A*;W&6n z=@JS7fq+0jARzERL||m{EQP9mOdkK&{`^hnzm$9>7qof)M_D-kzZUPOjKl2tmy1l1 zh;8Ft5R3%h!MHdz7xT>W8cHZCSB&8>&90l53=Xp<>E*D>M<2Vo8*8l9D*K-ZL#y zZQ=xy1TbwlA*sSvk$`4t>Jlk6(avMSc$^Yo%t_n4Rq$q>(~gPk+Y4v#cWf^=0@K;g zetL>o_SUyVTU8mpNs&Rdt4TMle|Dc6KFbBi^{-S||GP5Biri^bZCIhryCba^=@v2! WZ*QOH3R;0P6aoSPfq=kwgTOxo35itz literal 0 HcmV?d00001 diff --git a/internal/formats/spdx22json/to_format_model.go b/internal/formats/spdx22json/to_format_model.go new file mode 100644 index 00000000000..3a1b98da300 --- /dev/null +++ b/internal/formats/spdx22json/to_format_model.go @@ -0,0 +1,165 @@ +package spdx22json + +import ( + "fmt" + "path" + "strings" + "time" + + "github.com/anchore/syft/syft/distro" + + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/formats/common/spdxhelpers" + "github.com/anchore/syft/internal/formats/spdx22json/model" + "github.com/anchore/syft/internal/spdxlicense" + "github.com/anchore/syft/internal/version" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" + "github.com/google/uuid" +) + +// toFormatModel creates and populates a new JSON document struct that follows the SPDX 2.2 spec from the given cataloging results. +func toFormatModel(catalog *pkg.Catalog, srcMetadata *source.Metadata, d *distro.Distro, _ source.Scope) model.Document { + name := documentName(srcMetadata) + packages, files, relationships := extractFromCatalog(catalog) + + return model.Document{ + Element: model.Element{ + SPDXID: model.ElementID("DOCUMENT").String(), + Name: name, + }, + SPDXVersion: model.Version, + CreationInfo: model.CreationInfo{ + Created: time.Now().UTC(), + Creators: []string{ + // note: key-value format derived from the JSON example document examples: https://github.com/spdx/spdx-spec/blob/v2.2/examples/SPDXJSONExample-v2.2.spdx.json + "Organization: Anchore, Inc", + "Tool: " + internal.ApplicationName + "-" + version.FromBuild().Version, + }, + LicenseListVersion: spdxlicense.Version, + }, + DataLicense: "CC0-1.0", + DocumentNamespace: documentNamespace(name, srcMetadata), + Packages: packages, + Files: files, + Relationships: relationships, + // TODO: add scope + SyftSourceData: srcMetadata, + SyftDistroData: toSyftDistroData(d), + } +} + +func toSyftDistroData(d *distro.Distro) *model.SyftDistroData { + if d == nil { + return nil + } + return &model.SyftDistroData{ + Name: d.Name(), + Version: d.FullVersion(), + IDLike: d.IDLike, + } +} + +func documentName(srcMetadata *source.Metadata) string { + if srcMetadata != nil { + switch srcMetadata.Scheme { + case source.ImageScheme: + return cleanSPDXName(srcMetadata.ImageMetadata.UserInput) + case source.DirectoryScheme: + return cleanSPDXName(srcMetadata.Path) + } + } + // TODO: is this alright? + return uuid.Must(uuid.NewRandom()).String() +} + +func documentNamespace(name string, srcMetadata *source.Metadata) string { + input := "unknown-source-type" + if srcMetadata != nil { + switch srcMetadata.Scheme { + case source.ImageScheme: + input = "image" + case source.DirectoryScheme: + input = "dir" + } + } + + uniqueID := uuid.Must(uuid.NewRandom()) + identifier := path.Join(input, uniqueID.String()) + if name != "." { + identifier = path.Join(input, fmt.Sprintf("%s-%s", name, uniqueID.String())) + } + + return path.Join(anchoreNamespace, identifier) +} + +func extractFromCatalog(catalog *pkg.Catalog) ([]model.Package, []model.File, []model.Relationship) { + packages := make([]model.Package, 0) + relationships := make([]model.Relationship, 0) + files := make([]model.File, 0) + + for _, p := range catalog.Sorted() { + license := spdxhelpers.License(p) + packageSpdxID := model.ElementID(fmt.Sprintf("Package-%+v-%s-%s", p.Type, p.Name, p.Version)).String() + + packageFiles, fileIDs, packageFileRelationships := spdxhelpers.Files(packageSpdxID, p) + files = append(files, packageFiles...) + + relationships = append(relationships, packageFileRelationships...) + + // note: the license concluded and declared should be the same since we are collecting license information + // from the project data itself (the installed package files). + packages = append(packages, model.Package{ + Description: spdxhelpers.Description(p), + DownloadLocation: spdxhelpers.DownloadLocation(p), + ExternalRefs: spdxhelpers.ExternalRefs(p), + FilesAnalyzed: false, + HasFiles: fileIDs, + Homepage: spdxhelpers.Homepage(p), + LicenseDeclared: license, // The Declared License is what the authors of a project believe govern the package + Originator: spdxhelpers.Originator(p), + SourceInfo: spdxhelpers.SourceInfo(p), + VersionInfo: p.Version, + Item: model.Item{ + LicenseConcluded: license, // The Concluded License field is the license the SPDX file creator believes governs the package + Element: model.Element{ + SPDXID: packageSpdxID, + Name: p.Name, + }, + }, + SyftPackageData: toSyftPackageData(p), + }) + } + + return packages, files, relationships +} + +func toSyftPackageData(p *pkg.Package) *model.SyftPackageData { + if p == nil { + return nil + } + return &model.SyftPackageData{ + SyftPackageBasicData: model.SyftPackageBasicData{ + PackageType: p.Type, + FoundBy: p.FoundBy, + Locations: p.Locations, + Language: p.Language, + Licenses: p.Licenses, + }, + SyftPackageCustomData: model.SyftPackageCustomData{ + MetadataType: p.MetadataType, + Metadata: p.Metadata, + }, + } +} + +func cleanSPDXName(name string) string { + // remove # according to specification + name = strings.ReplaceAll(name, "#", "-") + + // remove : for url construction + name = strings.ReplaceAll(name, ":", "-") + + // clean relative pathing + return path.Clean(name) +} diff --git a/internal/formats/spdx22json/to_syft_model.go b/internal/formats/spdx22json/to_syft_model.go new file mode 100644 index 00000000000..f9a98763ef1 --- /dev/null +++ b/internal/formats/spdx22json/to_syft_model.go @@ -0,0 +1,65 @@ +package spdx22json + +import ( + "github.com/anchore/syft/internal/formats/common/spdxhelpers" + "github.com/anchore/syft/internal/formats/spdx22json/model" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/distro" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" +) + +func toSyftModel(doc model.Document) (*pkg.Catalog, *source.Metadata, *distro.Distro, source.Scope, error) { + d, err := toSyftDistro(doc.SyftDistroData) + if err != nil { + log.Warnf("unable to parse distro info=%+v: %+v", d, err) + d = nil + } + + // TODO: add scope parsing + return toSyftCatalog(doc.Packages), doc.SyftSourceData, d, source.UnknownScope, nil +} + +func toSyftCatalog(pkgs []model.Package) *pkg.Catalog { + catalog := pkg.NewCatalog() + for _, p := range pkgs { + catalog.Add(toSyftPackage(p)) + } + return catalog +} + +func toSyftPackage(p model.Package) pkg.Package { + syftPkg := pkg.Package{ + Name: p.Name, + Version: p.VersionInfo, + CPEs: spdxhelpers.ExtractCPEs(p.ExternalRefs), + PURL: spdxhelpers.ExtractPURL(p.ExternalRefs), + } + + if extra := p.SyftPackageData; extra != nil { + syftPkg.Type = extra.PackageType + syftPkg.FoundBy = extra.FoundBy + syftPkg.Locations = extra.Locations + syftPkg.Language = extra.Language + syftPkg.Licenses = extra.Licenses + syftPkg.MetadataType = extra.MetadataType + syftPkg.Metadata = extra.Metadata + } + + // if syftPkg.Type == "" && syftPkg.PURL != "" { + // // TODO: extract package type from purl --this is useful for ingesting from tools other than syft and is important for grype + // } + + return syftPkg +} + +func toSyftDistro(d *model.SyftDistroData) (*distro.Distro, error) { + if d == nil { + return nil, nil + } + newDistro, err := distro.NewDistro(distro.Type(d.Name), d.Version, d.IDLike) + if err != nil { + return nil, err + } + return &newDistro, nil +} diff --git a/internal/formats/spdx22json/validator.go b/internal/formats/spdx22json/validator.go new file mode 100644 index 00000000000..5bcbc6f182f --- /dev/null +++ b/internal/formats/spdx22json/validator.go @@ -0,0 +1,26 @@ +package spdx22json + +import ( + "encoding/json" + "fmt" + "io" +) + +func validator(reader io.Reader) error { + type Document struct { + SPDXID string `json:"SPDXID"` + } + + dec := json.NewDecoder(reader) + + var doc Document + err := dec.Decode(&doc) + if err != nil { + return fmt.Errorf("unable to decode: %w", err) + } + + if doc.SPDXID != "" { + return nil + } + return fmt.Errorf("could not extract document SPDXID") +} From fc63ba01ed2118f7b15b48e3168fc37e1baa97cd Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 21 Oct 2021 10:27:35 -0400 Subject: [PATCH 3/5] add common sdpxhelpers (migrated) Signed-off-by: Alex Goodman --- .../formats/common/spdxhelpers/description.go | 14 ++ .../common/spdxhelpers/description_test.go | 56 +++++++ .../common/spdxhelpers/download_location.go | 22 +++ .../spdxhelpers/download_location_test.go | 54 +++++++ .../common/spdxhelpers/external_refs.go | 50 +++++++ .../common/spdxhelpers/external_refs_test.go | 45 ++++++ internal/formats/common/spdxhelpers/files.go | 47 ++++++ .../formats/common/spdxhelpers/homepage.go | 14 ++ .../common/spdxhelpers/homepage_test.go | 56 +++++++ .../formats/common/spdxhelpers/license.go | 37 +++++ .../common/spdxhelpers/license_test.go | 73 +++++++++ .../common/spdxhelpers/none_if_empty.go | 12 ++ .../common/spdxhelpers/none_if_empty_test.go | 41 +++++ .../common/spdxhelpers/originator_test.go | 114 ++++++++++++++ .../formats/common/spdxhelpers/origintor.go | 36 +++++ .../formats/common/spdxhelpers/source_info.go | 39 +++++ .../common/spdxhelpers/source_info_test.go | 141 ++++++++++++++++++ 17 files changed, 851 insertions(+) create mode 100644 internal/formats/common/spdxhelpers/description.go create mode 100644 internal/formats/common/spdxhelpers/description_test.go create mode 100644 internal/formats/common/spdxhelpers/download_location.go create mode 100644 internal/formats/common/spdxhelpers/download_location_test.go create mode 100644 internal/formats/common/spdxhelpers/external_refs.go create mode 100644 internal/formats/common/spdxhelpers/external_refs_test.go create mode 100644 internal/formats/common/spdxhelpers/files.go create mode 100644 internal/formats/common/spdxhelpers/homepage.go create mode 100644 internal/formats/common/spdxhelpers/homepage_test.go create mode 100644 internal/formats/common/spdxhelpers/license.go create mode 100644 internal/formats/common/spdxhelpers/license_test.go create mode 100644 internal/formats/common/spdxhelpers/none_if_empty.go create mode 100644 internal/formats/common/spdxhelpers/none_if_empty_test.go create mode 100644 internal/formats/common/spdxhelpers/originator_test.go create mode 100644 internal/formats/common/spdxhelpers/origintor.go create mode 100644 internal/formats/common/spdxhelpers/source_info.go create mode 100644 internal/formats/common/spdxhelpers/source_info_test.go diff --git a/internal/formats/common/spdxhelpers/description.go b/internal/formats/common/spdxhelpers/description.go new file mode 100644 index 00000000000..8b8783a26ea --- /dev/null +++ b/internal/formats/common/spdxhelpers/description.go @@ -0,0 +1,14 @@ +package spdxhelpers + +import "github.com/anchore/syft/syft/pkg" + +func Description(p *pkg.Package) string { + switch metadata := p.Metadata.(type) { + case pkg.ApkMetadata: + return metadata.Description + case pkg.NpmPackageJSONMetadata: + return metadata.Description + default: + return "" + } +} diff --git a/internal/formats/common/spdxhelpers/description_test.go b/internal/formats/common/spdxhelpers/description_test.go new file mode 100644 index 00000000000..77bc424edd1 --- /dev/null +++ b/internal/formats/common/spdxhelpers/description_test.go @@ -0,0 +1,56 @@ +package spdxhelpers + +import ( + "testing" + + "github.com/anchore/syft/syft/pkg" + "github.com/stretchr/testify/assert" +) + +func Test_Description(t *testing.T) { + tests := []struct { + name string + input pkg.Package + expected string + }{ + { + // note: since this is an optional field, no value is preferred over NONE or NOASSERTION + name: "no metadata", + input: pkg.Package{}, + expected: "", + }, + { + name: "from apk", + input: pkg.Package{ + Metadata: pkg.ApkMetadata{ + Description: "a description!", + }, + }, + expected: "a description!", + }, + { + name: "from npm", + input: pkg.Package{ + Metadata: pkg.NpmPackageJSONMetadata{ + Description: "a description!", + }, + }, + expected: "a description!", + }, + { + // note: since this is an optional field, no value is preferred over NONE or NOASSERTION + name: "empty", + input: pkg.Package{ + Metadata: pkg.NpmPackageJSONMetadata{ + Homepage: "", + }, + }, + expected: "", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.expected, Description(&test.input)) + }) + } +} diff --git a/internal/formats/common/spdxhelpers/download_location.go b/internal/formats/common/spdxhelpers/download_location.go new file mode 100644 index 00000000000..fea6b9d908b --- /dev/null +++ b/internal/formats/common/spdxhelpers/download_location.go @@ -0,0 +1,22 @@ +package spdxhelpers + +import "github.com/anchore/syft/syft/pkg" + +func DownloadLocation(p *pkg.Package) string { + // 3.7: Package Download Location + // Cardinality: mandatory, one + // NONE if there is no download location whatsoever. + // NOASSERTION if: + // (i) the SPDX file creator has attempted to but cannot reach a reasonable objective determination; + // (ii) the SPDX file creator has made no attempt to determine this field; or + // (iii) the SPDX file creator has intentionally provided no information (no meaning should be implied by doing so). + + switch metadata := p.Metadata.(type) { + case pkg.ApkMetadata: + return NoneIfEmpty(metadata.URL) + case pkg.NpmPackageJSONMetadata: + return NoneIfEmpty(metadata.URL) + default: + return "NOASSERTION" + } +} diff --git a/internal/formats/common/spdxhelpers/download_location_test.go b/internal/formats/common/spdxhelpers/download_location_test.go new file mode 100644 index 00000000000..3636c7c7707 --- /dev/null +++ b/internal/formats/common/spdxhelpers/download_location_test.go @@ -0,0 +1,54 @@ +package spdxhelpers + +import ( + "testing" + + "github.com/anchore/syft/syft/pkg" + "github.com/stretchr/testify/assert" +) + +func Test_DownloadLocation(t *testing.T) { + tests := []struct { + name string + input pkg.Package + expected string + }{ + { + name: "no metadata", + input: pkg.Package{}, + expected: "NOASSERTION", + }, + { + name: "from apk", + input: pkg.Package{ + Metadata: pkg.ApkMetadata{ + URL: "http://a-place.gov", + }, + }, + expected: "http://a-place.gov", + }, + { + name: "from npm", + input: pkg.Package{ + Metadata: pkg.NpmPackageJSONMetadata{ + URL: "http://a-place.gov", + }, + }, + expected: "http://a-place.gov", + }, + { + name: "empty", + input: pkg.Package{ + Metadata: pkg.NpmPackageJSONMetadata{ + URL: "", + }, + }, + expected: "NONE", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.expected, DownloadLocation(&test.input)) + }) + } +} diff --git a/internal/formats/common/spdxhelpers/external_refs.go b/internal/formats/common/spdxhelpers/external_refs.go new file mode 100644 index 00000000000..dc2ea273514 --- /dev/null +++ b/internal/formats/common/spdxhelpers/external_refs.go @@ -0,0 +1,50 @@ +package spdxhelpers + +import ( + "github.com/anchore/syft/internal/formats/spdx22json/model" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/pkg" +) + +func ExternalRefs(p *pkg.Package) (externalRefs []model.ExternalRef) { + externalRefs = make([]model.ExternalRef, 0) + for _, c := range p.CPEs { + externalRefs = append(externalRefs, model.ExternalRef{ + ReferenceCategory: model.SecurityReferenceCategory, + ReferenceLocator: c.BindToFmtString(), + ReferenceType: model.Cpe23ExternalRefType, + }) + } + + if p.PURL != "" { + externalRefs = append(externalRefs, model.ExternalRef{ + ReferenceCategory: model.PackageManagerReferenceCategory, + ReferenceLocator: p.PURL, + ReferenceType: model.PurlExternalRefType, + }) + } + return externalRefs +} + +func ExtractPURL(refs []model.ExternalRef) string { + for _, r := range refs { + if r.ReferenceType == model.PurlExternalRefType { + return r.ReferenceLocator + } + } + return "" +} + +func ExtractCPEs(refs []model.ExternalRef) (cpes []pkg.CPE) { + for _, r := range refs { + if r.ReferenceType == model.Cpe23ExternalRefType { + cpe, err := pkg.NewCPE(r.ReferenceLocator) + if err != nil { + log.Warnf("unable to extract SPDX CPE=%q: %+v", r.ReferenceLocator, err) + continue + } + cpes = append(cpes, cpe) + } + } + return cpes +} diff --git a/internal/formats/common/spdxhelpers/external_refs_test.go b/internal/formats/common/spdxhelpers/external_refs_test.go new file mode 100644 index 00000000000..8f0860577c7 --- /dev/null +++ b/internal/formats/common/spdxhelpers/external_refs_test.go @@ -0,0 +1,45 @@ +package spdxhelpers + +import ( + "testing" + + "github.com/anchore/syft/internal/formats/spdx22json/model" + "github.com/anchore/syft/syft/pkg" + "github.com/stretchr/testify/assert" +) + +func Test_ExternalRefs(t *testing.T) { + testCPE := pkg.MustCPE("cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*") + tests := []struct { + name string + input pkg.Package + expected []model.ExternalRef + }{ + { + name: "cpe + purl", + input: pkg.Package{ + CPEs: []pkg.CPE{ + testCPE, + }, + PURL: "a-purl", + }, + expected: []model.ExternalRef{ + { + ReferenceCategory: model.SecurityReferenceCategory, + ReferenceLocator: testCPE.BindToFmtString(), + ReferenceType: model.Cpe23ExternalRefType, + }, + { + ReferenceCategory: model.PackageManagerReferenceCategory, + ReferenceLocator: "a-purl", + ReferenceType: model.PurlExternalRefType, + }, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.ElementsMatch(t, test.expected, ExternalRefs(&test.input)) + }) + } +} diff --git a/internal/formats/common/spdxhelpers/files.go b/internal/formats/common/spdxhelpers/files.go new file mode 100644 index 00000000000..0e05e321cda --- /dev/null +++ b/internal/formats/common/spdxhelpers/files.go @@ -0,0 +1,47 @@ +package spdxhelpers + +import ( + "crypto/sha256" + "fmt" + "path/filepath" + + "github.com/anchore/syft/internal/formats/spdx22json/model" + "github.com/anchore/syft/syft/pkg" +) + +func Files(packageSpdxID string, p *pkg.Package) (files []model.File, fileIDs []string, relationships []model.Relationship) { + files = make([]model.File, 0) + fileIDs = make([]string, 0) + relationships = make([]model.Relationship, 0) + + pkgFileOwner, ok := p.Metadata.(pkg.FileOwner) + if !ok { + return files, fileIDs, relationships + } + + for _, ownedFilePath := range pkgFileOwner.OwnedFiles() { + baseFileName := filepath.Base(ownedFilePath) + pathHash := sha256.Sum256([]byte(ownedFilePath)) + fileSpdxID := model.ElementID(fmt.Sprintf("File-%s-%x", p.Name, pathHash)).String() + + fileIDs = append(fileIDs, fileSpdxID) + + files = append(files, model.File{ + FileName: ownedFilePath, + Item: model.Item{ + Element: model.Element{ + SPDXID: fileSpdxID, + Name: baseFileName, + }, + }, + }) + + relationships = append(relationships, model.Relationship{ + SpdxElementID: packageSpdxID, + RelationshipType: model.ContainsRelationship, + RelatedSpdxElement: fileSpdxID, + }) + } + + return files, fileIDs, relationships +} diff --git a/internal/formats/common/spdxhelpers/homepage.go b/internal/formats/common/spdxhelpers/homepage.go new file mode 100644 index 00000000000..8ea1ad75337 --- /dev/null +++ b/internal/formats/common/spdxhelpers/homepage.go @@ -0,0 +1,14 @@ +package spdxhelpers + +import "github.com/anchore/syft/syft/pkg" + +func Homepage(p *pkg.Package) string { + switch metadata := p.Metadata.(type) { + case pkg.GemMetadata: + return metadata.Homepage + case pkg.NpmPackageJSONMetadata: + return metadata.Homepage + default: + return "" + } +} diff --git a/internal/formats/common/spdxhelpers/homepage_test.go b/internal/formats/common/spdxhelpers/homepage_test.go new file mode 100644 index 00000000000..781873f7a26 --- /dev/null +++ b/internal/formats/common/spdxhelpers/homepage_test.go @@ -0,0 +1,56 @@ +package spdxhelpers + +import ( + "testing" + + "github.com/anchore/syft/syft/pkg" + "github.com/stretchr/testify/assert" +) + +func Test_Homepage(t *testing.T) { + tests := []struct { + name string + input pkg.Package + expected string + }{ + { + // note: since this is an optional field, no value is preferred over NONE or NOASSERTION + name: "no metadata", + input: pkg.Package{}, + expected: "", + }, + { + name: "from gem", + input: pkg.Package{ + Metadata: pkg.GemMetadata{ + Homepage: "http://a-place.gov", + }, + }, + expected: "http://a-place.gov", + }, + { + name: "from npm", + input: pkg.Package{ + Metadata: pkg.NpmPackageJSONMetadata{ + Homepage: "http://a-place.gov", + }, + }, + expected: "http://a-place.gov", + }, + { + // note: since this is an optional field, no value is preferred over NONE or NOASSERTION + name: "empty", + input: pkg.Package{ + Metadata: pkg.NpmPackageJSONMetadata{ + Homepage: "", + }, + }, + expected: "", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.expected, Homepage(&test.input)) + }) + } +} diff --git a/internal/formats/common/spdxhelpers/license.go b/internal/formats/common/spdxhelpers/license.go new file mode 100644 index 00000000000..f2e4307ad7e --- /dev/null +++ b/internal/formats/common/spdxhelpers/license.go @@ -0,0 +1,37 @@ +package spdxhelpers + +import ( + "strings" + + "github.com/anchore/syft/internal/spdxlicense" + "github.com/anchore/syft/syft/pkg" +) + +func License(p *pkg.Package) string { + // source: https://spdx.github.io/spdx-spec/3-package-information/#313-concluded-license + // The options to populate this field are limited to: + // A valid SPDX License Expression as defined in Appendix IV; + // NONE, if the SPDX file creator concludes there is no license available for this package; or + // NOASSERTION if: + // (i) the SPDX file creator has attempted to but cannot reach a reasonable objective determination; + // (ii) the SPDX file creator has made no attempt to determine this field; or + // (iii) the SPDX file creator has intentionally provided no information (no meaning should be implied by doing so). + + if len(p.Licenses) == 0 { + return "NONE" + } + + // take all licenses and assume an AND expression; for information about license expressions see https://spdx.github.io/spdx-spec/appendix-IV-SPDX-license-expressions/ + var parsedLicenses []string + for _, l := range p.Licenses { + if value, exists := spdxlicense.ID(l); exists { + parsedLicenses = append(parsedLicenses, value) + } + } + + if len(parsedLicenses) == 0 { + return "NOASSERTION" + } + + return strings.Join(parsedLicenses, " AND ") +} diff --git a/internal/formats/common/spdxhelpers/license_test.go b/internal/formats/common/spdxhelpers/license_test.go new file mode 100644 index 00000000000..c4762ee1873 --- /dev/null +++ b/internal/formats/common/spdxhelpers/license_test.go @@ -0,0 +1,73 @@ +package spdxhelpers + +import ( + "testing" + + "github.com/anchore/syft/syft/pkg" + "github.com/stretchr/testify/assert" +) + +func Test_License(t *testing.T) { + tests := []struct { + name string + input pkg.Package + expected string + }{ + { + name: "no licenses", + input: pkg.Package{}, + expected: "NONE", + }, + { + name: "no SPDX licenses", + input: pkg.Package{ + Licenses: []string{ + "made-up", + }, + }, + expected: "NOASSERTION", + }, + { + name: "with SPDX license", + input: pkg.Package{ + Licenses: []string{ + "MIT", + }, + }, + expected: "MIT", + }, + { + name: "with SPDX license expression", + input: pkg.Package{ + Licenses: []string{ + "MIT", + "GPL-3.0", + }, + }, + expected: "MIT AND GPL-3.0", + }, + { + name: "cap insensitive", + input: pkg.Package{ + Licenses: []string{ + "gpl-3.0", + }, + }, + expected: "GPL-3.0", + }, + { + name: "debian to spdx conversion", + input: pkg.Package{ + Licenses: []string{ + "GPL-2", + }, + }, + expected: "GPL-2.0", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.expected, License(&test.input)) + }) + } +} diff --git a/internal/formats/common/spdxhelpers/none_if_empty.go b/internal/formats/common/spdxhelpers/none_if_empty.go new file mode 100644 index 00000000000..fa97a0d7e59 --- /dev/null +++ b/internal/formats/common/spdxhelpers/none_if_empty.go @@ -0,0 +1,12 @@ +package spdxhelpers + +import ( + "strings" +) + +func NoneIfEmpty(value string) string { + if strings.TrimSpace(value) == "" { + return "NONE" + } + return value +} diff --git a/internal/formats/common/spdxhelpers/none_if_empty_test.go b/internal/formats/common/spdxhelpers/none_if_empty_test.go new file mode 100644 index 00000000000..4c447f1220a --- /dev/null +++ b/internal/formats/common/spdxhelpers/none_if_empty_test.go @@ -0,0 +1,41 @@ +package spdxhelpers + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_noneIfEmpty(t *testing.T) { + tests := []struct { + name string + value string + expected string + }{ + { + name: "non-zero value", + value: "something", + expected: "something", + }, + { + name: "empty", + value: "", + expected: "NONE", + }, + { + name: "space", + value: " ", + expected: "NONE", + }, + { + name: "tab", + value: "\t", + expected: "NONE", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.expected, NoneIfEmpty(test.value)) + }) + } +} diff --git a/internal/formats/common/spdxhelpers/originator_test.go b/internal/formats/common/spdxhelpers/originator_test.go new file mode 100644 index 00000000000..7e3ec04ed92 --- /dev/null +++ b/internal/formats/common/spdxhelpers/originator_test.go @@ -0,0 +1,114 @@ +package spdxhelpers + +import ( + "testing" + + "github.com/anchore/syft/syft/pkg" + "github.com/stretchr/testify/assert" +) + +func Test_Originator(t *testing.T) { + tests := []struct { + name string + input pkg.Package + expected string + }{ + { + // note: since this is an optional field, no value is preferred over NONE or NOASSERTION + name: "no metadata", + input: pkg.Package{}, + expected: "", + }, + { + name: "from gem", + input: pkg.Package{ + Metadata: pkg.GemMetadata{ + Authors: []string{ + "auth1", + "auth2", + }, + }, + }, + expected: "auth1", + }, + { + name: "from npm", + input: pkg.Package{ + Metadata: pkg.NpmPackageJSONMetadata{ + Author: "auth", + }, + }, + expected: "auth", + }, + { + name: "from apk", + input: pkg.Package{ + Metadata: pkg.ApkMetadata{ + Maintainer: "auth", + }, + }, + expected: "auth", + }, + { + name: "from python - just name", + input: pkg.Package{ + Metadata: pkg.PythonPackageMetadata{ + Author: "auth", + }, + }, + expected: "auth", + }, + { + name: "from python - just email", + input: pkg.Package{ + Metadata: pkg.PythonPackageMetadata{ + AuthorEmail: "auth@auth.gov", + }, + }, + expected: "auth@auth.gov", + }, + { + name: "from python - both name and email", + input: pkg.Package{ + Metadata: pkg.PythonPackageMetadata{ + Author: "auth", + AuthorEmail: "auth@auth.gov", + }, + }, + expected: "auth ", + }, + { + name: "from rpm", + input: pkg.Package{ + Metadata: pkg.RpmdbMetadata{ + Vendor: "auth", + }, + }, + expected: "auth", + }, + { + name: "from dpkg", + input: pkg.Package{ + Metadata: pkg.DpkgMetadata{ + Maintainer: "auth", + }, + }, + expected: "auth", + }, + { + // note: since this is an optional field, no value is preferred over NONE or NOASSERTION + name: "empty", + input: pkg.Package{ + Metadata: pkg.NpmPackageJSONMetadata{ + Author: "", + }, + }, + expected: "", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.expected, Originator(&test.input)) + }) + } +} diff --git a/internal/formats/common/spdxhelpers/origintor.go b/internal/formats/common/spdxhelpers/origintor.go new file mode 100644 index 00000000000..f9ecbe2be7d --- /dev/null +++ b/internal/formats/common/spdxhelpers/origintor.go @@ -0,0 +1,36 @@ +package spdxhelpers + +import ( + "fmt" + + "github.com/anchore/syft/syft/pkg" +) + +func Originator(p *pkg.Package) string { + switch metadata := p.Metadata.(type) { + case pkg.ApkMetadata: + return metadata.Maintainer + case pkg.NpmPackageJSONMetadata: + return metadata.Author + case pkg.PythonPackageMetadata: + author := metadata.Author + if author == "" { + return metadata.AuthorEmail + } + if metadata.AuthorEmail != "" { + author += fmt.Sprintf(" <%s>", metadata.AuthorEmail) + } + return author + case pkg.GemMetadata: + if len(metadata.Authors) > 0 { + return metadata.Authors[0] + } + return "" + case pkg.RpmdbMetadata: + return metadata.Vendor + case pkg.DpkgMetadata: + return metadata.Maintainer + default: + return "" + } +} diff --git a/internal/formats/common/spdxhelpers/source_info.go b/internal/formats/common/spdxhelpers/source_info.go new file mode 100644 index 00000000000..7c77696f0c3 --- /dev/null +++ b/internal/formats/common/spdxhelpers/source_info.go @@ -0,0 +1,39 @@ +package spdxhelpers + +import ( + "strings" + + "github.com/anchore/syft/syft/pkg" +) + +func SourceInfo(p *pkg.Package) string { + answer := "" + switch p.Type { + case pkg.RpmPkg: + answer = "acquired package info from RPM DB" + case pkg.ApkPkg: + answer = "acquired package info from APK DB" + case pkg.DebPkg: + answer = "acquired package info from DPKG DB" + case pkg.NpmPkg: + answer = "acquired package info from installed node module manifest file" + case pkg.PythonPkg: + answer = "acquired package info from installed python package manifest file" + case pkg.JavaPkg, pkg.JenkinsPluginPkg: + answer = "acquired package info from installed java archive" + case pkg.GemPkg: + answer = "acquired package info from installed gem metadata file" + case pkg.GoModulePkg: + answer = "acquired package info from go module information" + case pkg.RustPkg: + answer = "acquired package info from rust cargo manifest" + default: + answer = "acquired package info from the following paths" + } + var paths []string + for _, l := range p.Locations { + paths = append(paths, l.RealPath) + } + + return answer + ": " + strings.Join(paths, ", ") +} diff --git a/internal/formats/common/spdxhelpers/source_info_test.go b/internal/formats/common/spdxhelpers/source_info_test.go new file mode 100644 index 00000000000..0faec15484c --- /dev/null +++ b/internal/formats/common/spdxhelpers/source_info_test.go @@ -0,0 +1,141 @@ +package spdxhelpers + +import ( + "testing" + + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" + "github.com/stretchr/testify/assert" +) + +func Test_SourceInfo(t *testing.T) { + tests := []struct { + name string + input pkg.Package + expected []string + }{ + { + name: "locations are captured", + input: pkg.Package{ + // note: no type given + Locations: []source.Location{ + { + RealPath: "/a-place", + VirtualPath: "/b-place", + }, + { + RealPath: "/c-place", + VirtualPath: "/d-place", + }, + }, + }, + expected: []string{ + "from the following paths", + "/a-place", + "/c-place", + }, + }, + { + // note: no specific support for this + input: pkg.Package{ + Type: pkg.KbPkg, + }, + expected: []string{ + "from the following paths", + }, + }, + { + input: pkg.Package{ + Type: pkg.RpmPkg, + }, + expected: []string{ + "from RPM DB", + }, + }, + { + input: pkg.Package{ + Type: pkg.ApkPkg, + }, + expected: []string{ + "from APK DB", + }, + }, + { + input: pkg.Package{ + Type: pkg.DebPkg, + }, + expected: []string{ + "from DPKG DB", + }, + }, + { + input: pkg.Package{ + Type: pkg.NpmPkg, + }, + expected: []string{ + "from installed node module manifest file", + }, + }, + { + input: pkg.Package{ + Type: pkg.PythonPkg, + }, + expected: []string{ + "from installed python package manifest file", + }, + }, + { + input: pkg.Package{ + Type: pkg.JavaPkg, + }, + expected: []string{ + "from installed java archive", + }, + }, + { + input: pkg.Package{ + Type: pkg.JenkinsPluginPkg, + }, + expected: []string{ + "from installed java archive", + }, + }, + { + input: pkg.Package{ + Type: pkg.GemPkg, + }, + expected: []string{ + "from installed gem metadata file", + }, + }, + { + input: pkg.Package{ + Type: pkg.GoModulePkg, + }, + expected: []string{ + "from go module information", + }, + }, + { + input: pkg.Package{ + Type: pkg.RustPkg, + }, + expected: []string{ + "from rust cargo manifest", + }, + }, + } + var pkgTypes []pkg.Type + for _, test := range tests { + t.Run(test.name+" "+string(test.input.Type), func(t *testing.T) { + if test.input.Type != "" { + pkgTypes = append(pkgTypes, test.input.Type) + } + actual := SourceInfo(&test.input) + for _, expected := range test.expected { + assert.Contains(t, actual, expected) + } + }) + } + assert.ElementsMatch(t, pkg.AllPkgs, pkgTypes, "missing one or more package types to test against (maybe a package type was added?)") +} From 4821cbbacc67b1ebb38f8a4c5ab290768f481870 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 21 Oct 2021 10:28:01 -0400 Subject: [PATCH 4/5] use new common spdx helpers Signed-off-by: Alex Goodman --- internal/presenter/packages/spdx_tag_value_presenter.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/presenter/packages/spdx_tag_value_presenter.go b/internal/presenter/packages/spdx_tag_value_presenter.go index 5f66fdf4aa9..19974ecb8f4 100644 --- a/internal/presenter/packages/spdx_tag_value_presenter.go +++ b/internal/presenter/packages/spdx_tag_value_presenter.go @@ -5,6 +5,8 @@ import ( "io" "time" + "github.com/anchore/syft/internal/formats/common/spdxhelpers" + "github.com/anchore/syft/internal/spdxlicense" "github.com/anchore/syft/internal" @@ -114,7 +116,7 @@ func (pres *SPDXTagValuePresenter) packages() map[spdx.ElementID]*spdx.Package2_ // If the Concluded License is not the same as the Declared License, a written explanation should be provided // in the Comments on License field (section 3.16). With respect to NOASSERTION, a written explanation in // the Comments on License field (section 3.16) is preferred. - license := getSPDXLicense(p) + license := spdxhelpers.License(p) results[spdx.ElementID(id)] = &spdx.Package2_2{ @@ -274,7 +276,7 @@ func (pres *SPDXTagValuePresenter) packages() map[spdx.ElementID]*spdx.Package2_ } func formatSPDXExternalRefs(p *pkg.Package) (refs []*spdx.PackageExternalReference2_2) { - for _, ref := range getSPDXExternalRefs(p) { + for _, ref := range spdxhelpers.ExternalRefs(p) { refs = append(refs, &spdx.PackageExternalReference2_2{ Category: string(ref.ReferenceCategory), RefType: string(ref.ReferenceType), From f369476c2c0f8a8b168aa214aa8c5cfde197dbd4 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 21 Oct 2021 10:28:13 -0400 Subject: [PATCH 5/5] wire up new spdx22json format object Signed-off-by: Alex Goodman --- internal/formats/formats.go | 3 +++ syft/presenter/packages/presenter.go | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/formats/formats.go b/internal/formats/formats.go index 5c82bd6533a..8da8edf6760 100644 --- a/internal/formats/formats.go +++ b/internal/formats/formats.go @@ -3,6 +3,8 @@ package formats import ( "bytes" + "github.com/anchore/syft/internal/formats/spdx22json" + "github.com/anchore/syft/internal/formats/syftjson" "github.com/anchore/syft/syft/format" ) @@ -11,6 +13,7 @@ import ( func All() []format.Format { return []format.Format{ syftjson.Format(), + spdx22json.Format(), } } diff --git a/syft/presenter/packages/presenter.go b/syft/presenter/packages/presenter.go index e6dcc10779b..22cb400a00f 100644 --- a/syft/presenter/packages/presenter.go +++ b/syft/presenter/packages/presenter.go @@ -22,8 +22,6 @@ func Presenter(option format.Option, config PresenterConfig) presenter.Presenter return packages.NewCycloneDxPresenter(config.Catalog, config.SourceMetadata) case format.SPDXTagValueOption: return packages.NewSPDXTagValuePresenter(config.Catalog, config.SourceMetadata) - case format.SPDXJSONOption: - return packages.NewSPDXJSONPresenter(config.Catalog, config.SourceMetadata) default: // TODO: the final state is that all other cases would be replaced by formats.ByOption (wed remove this function entirely) f := formats.ByOption(option)