From af0aa6a198b69f0196d903b345416e9bbded255f Mon Sep 17 00:00:00 2001 From: Daniel Mikusa Date: Tue, 16 Nov 2021 12:23:27 -0500 Subject: [PATCH] Adds convenience methods for using the Syft CLI to create SBoM information - Supports running syft against a directory & generating one or more output formats - Automatically converts syft generated CycloneDX XML to JSON, which is what buildpacks require - This is working around a couple features not present in syft at the moment, such as support for multiple output formats or conversion, and being able to output directly to CycloneDX JSON. When those features are added, we can trim this code back accordingly. The function signatures should not need to change for that. Signed-off-by: Daniel Mikusa --- buildpack.go | 2 + go.mod | 1 + go.sum | 8 +- layer.go | 6 ++ sherpa/init_test.go | 1 + sherpa/mocks/bom_scanner.go | 25 +++++ sherpa/sbom.go | 154 ++++++++++++++++++++++++++++ sherpa/sbom_test.go | 196 ++++++++++++++++++++++++++++++++++++ 8 files changed, 392 insertions(+), 1 deletion(-) create mode 100644 sherpa/mocks/bom_scanner.go create mode 100644 sherpa/sbom.go create mode 100644 sherpa/sbom_test.go diff --git a/buildpack.go b/buildpack.go index 124fb80..f3cd09b 100644 --- a/buildpack.go +++ b/buildpack.go @@ -85,6 +85,8 @@ type BuildpackDependency struct { } // AsBOMEntry renders a bill of materials entry describing the dependency. +// +// Deprecated: as of Buildpacks RFC 95, use `sherpa.BOMScanner` instead func (b BuildpackDependency) AsBOMEntry() libcnb.BOMEntry { return libcnb.BOMEntry{ Name: b.ID, diff --git a/go.mod b/go.mod index feef1c6..cd0efb3 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/paketo-buildpacks/libpak go 1.15 require ( + github.com/CycloneDX/cyclonedx-go v0.4.0 github.com/Masterminds/semver/v3 v3.1.1 github.com/buildpacks/libcnb v1.24.0 github.com/creack/pty v1.1.17 diff --git a/go.sum b/go.sum index 5c0b3fd..3acf3cd 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,11 @@ github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw= github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/CycloneDX/cyclonedx-go v0.4.0 h1:Wz4QZ9B4RXGWIWTypVLEOVJgOdFfy5mcS5PGNzUkZxU= +github.com/CycloneDX/cyclonedx-go v0.4.0/go.mod h1:rmRcf//gT7PIzovatusbWi377xqCg1FS4jyST0GH20E= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/bradleyjkemp/cupaloy/v2 v2.6.0 h1:knToPYa2xtfg42U3I6punFEjaGFKWQRXJwj0JTv4mTs= +github.com/bradleyjkemp/cupaloy/v2 v2.6.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= github.com/buildpacks/libcnb v1.24.0 h1:jVpydlJPygweUBk4ac3WGT2X1NGeunH17eyn9tUqZuU= github.com/buildpacks/libcnb v1.24.0/go.mod h1:wIXTSW6ybtX9XIICQQqPnIUxx6t1bSZT7iIOKbEzRH0= github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI= @@ -57,9 +61,11 @@ github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8= github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4QnFHkOM= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= diff --git a/layer.go b/layer.go index 1967ec0..3d8a2a4 100644 --- a/layer.go +++ b/layer.go @@ -128,6 +128,9 @@ type DependencyLayerContributor struct { } // NewDependencyLayer returns a new DependencyLayerContributor for the given BuildpackDependency and a BOMEntry describing the layer contents. +// +// Deprecated: this method uses `libcnb.BOMEntry` which has been deprecated upstream, a future version will drop +// support for `libcnb.BOMEntry` which will change this method signature func NewDependencyLayer(dependency BuildpackDependency, cache DependencyCache, types libcnb.LayerTypes) (DependencyLayerContributor, libcnb.BOMEntry) { c := DependencyLayerContributor{ Dependency: dependency, @@ -197,6 +200,9 @@ type HelperLayerContributor struct { } // NewHelperLayer returns a new HelperLayerContributor and a BOMEntry describing the layer contents. +// +// Deprecated: this method uses `libcnb.BOMEntry` which has been deprecated upstream, a future version will drop +// support for `libcnb.BOMEntry` which will change this method signature func NewHelperLayer(buildpack libcnb.Buildpack, names ...string) (HelperLayerContributor, libcnb.BOMEntry) { c := HelperLayerContributor{ Path: filepath.Join(buildpack.Path, "bin", "helper"), diff --git a/sherpa/init_test.go b/sherpa/init_test.go index f524e5d..a853d48 100644 --- a/sherpa/init_test.go +++ b/sherpa/init_test.go @@ -29,6 +29,7 @@ func TestUnit(t *testing.T) { suite("EnvVar", testEnvVar) suite("FileListing", testFileListing) suite("NodeJS", testNodeJS) + suite("SBOM", testSBOM) suite("Sherpa", testSherpa) suite.Run(t) } diff --git a/sherpa/mocks/bom_scanner.go b/sherpa/mocks/bom_scanner.go new file mode 100644 index 0000000..33b0bbf --- /dev/null +++ b/sherpa/mocks/bom_scanner.go @@ -0,0 +1,25 @@ +// Code generated by mockery v1.0.0. DO NOT EDIT. + +package mocks + +import ( + libcnb "github.com/buildpacks/libcnb" + mock "github.com/stretchr/testify/mock" +) + +// BOMScanner is an autogenerated mock type for the BOMScanner type +type BOMScanner struct { + mock.Mock +} + +// Scan provides a mock function with given fields: location, scanDir, formats +func (_m *BOMScanner) Scan(location libcnb.BOMLocation, scanDir string, formats ...libcnb.BOMFormat) { + _va := make([]interface{}, len(formats)) + for _i := range formats { + _va[_i] = formats[_i] + } + var _ca []interface{} + _ca = append(_ca, location, scanDir) + _ca = append(_ca, _va...) + _m.Called(_ca...) +} diff --git a/sherpa/sbom.go b/sherpa/sbom.go new file mode 100644 index 0000000..59d1721 --- /dev/null +++ b/sherpa/sbom.go @@ -0,0 +1,154 @@ +package sherpa + +import ( + "fmt" + "io" + "os" + + "github.com/CycloneDX/cyclonedx-go" + "github.com/buildpacks/libcnb" + "github.com/paketo-buildpacks/libpak/bard" + "github.com/paketo-buildpacks/libpak/effect" +) + +//go:generate mockery -name BOMScanner -case=underscore + +type BOMScanner interface { + Scan(location libcnb.BOMLocation, scanDir string, formats ...libcnb.BOMFormat) +} + +type SyftCLIBOMScanner struct { + Layer libcnb.Layer + Executor effect.Executor + Logger bard.Logger +} + +// Scan will use syft CLI to scan the scanDir and write it's output to the location in the given formats +func (b SyftCLIBOMScanner) Scan(location libcnb.BOMLocation, scanDir string, formats ...libcnb.BOMFormat) error { + // syft doesn't presently support outputting multiple formats at once + // to workaround this we are running syft multiple times + // when syft supports multiple output formats or conversion between formats, this method should change + for _, format := range formats { + if err := b.runSyft(location, scanDir, format); err != nil { + return fmt.Errorf("unable to run syft\n%w", err) + } + + if format == libcnb.CycloneDXJSON { + // syft doesn't presently support cyclonedx JSON output and we need to convert + // until https://github.com/anchore/syft/issues/631 is addressed + if err := b.ConvertCycloneDXXMLtoJSON(location, false); err != nil { + return fmt.Errorf("unable convert XML to JSON\n%w", err) + } + } + } + + return nil +} + +// ConvertCycloneDXXMLtoJSON reads input CycloneDX XML, converts to JSON and overwrites the XML optionally keeping a backup copy of the xml +func (b SyftCLIBOMScanner) ConvertCycloneDXXMLtoJSON(location libcnb.BOMLocation, backup bool) error { + inputPath := b.Layer.BOMPath(location, libcnb.CycloneDXJSON) + + if backup { + if err := b.backupXMLFile(inputPath); err != nil { + return fmt.Errorf("unable to backup file\n%w", err) + } + } + + bom, err := b.readXMLBOM(inputPath) + if err != nil { + return fmt.Errorf("unable to backup file\n%w", err) + } + + if err := b.writeJSONBOM(inputPath, bom); err != nil { + return fmt.Errorf("unable to backup file\n%w", err) + } + + return nil +} + +func (b SyftCLIBOMScanner) writeJSONBOM(outputPath string, bom cyclonedx.BOM) error { + outputFile, err := os.Create(outputPath) + if err != nil { + return fmt.Errorf("unable to create BOM file %s\n%w", outputPath, err) + } + defer outputFile.Close() + + decoder := cyclonedx.NewBOMEncoder(outputFile, cyclonedx.BOMFileFormatJSON) + if err = decoder.Encode(&bom); err != nil { + return fmt.Errorf("unable to decode BOM\n%w", err) + } + + return nil +} + +func (b SyftCLIBOMScanner) readXMLBOM(inputPath string) (cyclonedx.BOM, error) { + inputFile, err := os.Open(inputPath) + if err != nil { + return cyclonedx.BOM{}, fmt.Errorf("unable to read file to convert %s\n%w", inputPath, err) + } + defer inputFile.Close() + + var bom cyclonedx.BOM + decoder := cyclonedx.NewBOMDecoder(inputFile, cyclonedx.BOMFileFormatXML) + if err = decoder.Decode(&bom); err != nil { + return cyclonedx.BOM{}, fmt.Errorf("unable to decode BOM\n%w", err) + } + + return bom, nil +} + +func (b SyftCLIBOMScanner) backupXMLFile(inputPath string) error { + backupPath := fmt.Sprintf("%s.bak", inputPath) + outputFile, err := os.Create(backupPath) + if err != nil { + return fmt.Errorf("unable to create backup file %s\n%w", backupPath, err) + } + defer outputFile.Close() + + inputFile, err := os.Open(inputPath) + if err != nil { + return fmt.Errorf("unable to read file for backup %s\n%w", inputPath, err) + } + defer inputFile.Close() + + _, err = io.Copy(outputFile, inputFile) + return err +} + +func (b SyftCLIBOMScanner) runSyft(location libcnb.BOMLocation, scanDir string, format libcnb.BOMFormat) error { + bomOutputPath := b.Layer.BOMPath(location, format) + writer, err := os.Create(bomOutputPath) + if err != nil { + return fmt.Errorf("unable to open output BOM file %s\n%w", bomOutputPath, err) + } + defer writer.Close() + + err = b.Executor.Execute(effect.Execution{ + Command: "syft", + Args: []string{"packges", "-o", BOMFormatToSyftOutputFormat(format), fmt.Sprintf("dir:%s", scanDir)}, + Stdout: writer, + Stderr: b.Logger.TerminalErrorWriter(), + }) + if err != nil { + return fmt.Errorf("unable to run syft on directory %s\n%w", scanDir, err) + } + + return nil +} + +// BOMFormatToSyftOutputFormat converts a libcnb.BOMFormat to the syft matching syft output format string +func BOMFormatToSyftOutputFormat(format libcnb.BOMFormat) string { + var formatRaw string + + switch format { + case libcnb.CycloneDXJSON: + formatRaw = "cyclonedx" + case libcnb.SPDXJSON: + formatRaw = "spdx-json" + case libcnb.SyftJSON: + formatRaw = "json" + } + + return formatRaw +} diff --git a/sherpa/sbom_test.go b/sherpa/sbom_test.go new file mode 100644 index 0000000..7d3b8dd --- /dev/null +++ b/sherpa/sbom_test.go @@ -0,0 +1,196 @@ +package sherpa_test + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/buildpacks/libcnb" + . "github.com/onsi/gomega" + "github.com/paketo-buildpacks/libpak/bard" + "github.com/paketo-buildpacks/libpak/effect" + "github.com/paketo-buildpacks/libpak/effect/mocks" + "github.com/paketo-buildpacks/libpak/sherpa" + "github.com/sclevine/spec" + "github.com/stretchr/testify/mock" +) + +func testSBOM(t *testing.T, context spec.G, it spec.S) { + var ( + Expect = NewWithT(t).Expect + + testRoot string + layer libcnb.Layer + executor mocks.Executor + ) + + it.Before(func() { + var err error + + executor = mocks.Executor{} + + testRoot, err = ioutil.TempDir("", "test-root") + Expect(err).NotTo(HaveOccurred()) + + layer.Path = filepath.Join(testRoot, "layer") + Expect(os.MkdirAll(layer.Path, 0755)).To(Succeed()) + }) + + it.After(func() { + Expect(os.RemoveAll(testRoot)).To(Succeed()) + }) + + context("syft", func() { + it("runs syft once to generate JSON", func() { + format := libcnb.SyftJSON + outputPath := layer.BOMPath(libcnb.BuildBOM, format) + + executor.On("Execute", mock.MatchedBy(func(e effect.Execution) bool { + return e.Command == "syft" && + len(e.Args) == 4 && + e.Args[2] == "json" && + e.Args[3] == "dir:something" + })).Run(func(args mock.Arguments) { + Expect(ioutil.WriteFile(outputPath, []byte("succeed"), 0644)).To(Succeed()) + }).Return(nil) + + scanner := sherpa.SyftCLIBOMScanner{ + Layer: layer, + Executor: &executor, + Logger: bard.NewLogger(io.Discard), + } + + Expect(scanner.Scan(libcnb.BuildBOM, "something", format)).To(Succeed()) + + result, err := ioutil.ReadFile(outputPath) + Expect(err).ToNot(HaveOccurred()) + Expect(string(result)).To(Equal("succeed")) + }) + + it("runs syft twice, once per format", func() { + outputPaths := map[libcnb.BOMFormat]string{ + libcnb.SPDXJSON: layer.BOMPath(libcnb.BuildBOM, libcnb.SPDXJSON), + libcnb.SyftJSON: layer.BOMPath(libcnb.BuildBOM, libcnb.SyftJSON), + } + + for format, outputPath := range outputPaths { + executor.On("Execute", mock.MatchedBy(func(e effect.Execution) bool { + return e.Command == "syft" && + len(e.Args) == 4 && + e.Args[2] == sherpa.BOMFormatToSyftOutputFormat(format) && + e.Args[3] == "dir:something" + })).Run(func(args mock.Arguments) { + Expect(ioutil.WriteFile(outputPath, []byte("succeed"), 0644)).To(Succeed()) + }).Return(nil) + + scanner := sherpa.SyftCLIBOMScanner{ + Layer: layer, + Executor: &executor, + Logger: bard.NewLogger(io.Discard), + } + + Expect(scanner.Scan(libcnb.BuildBOM, "something", format)).To(Succeed()) + + result, err := ioutil.ReadFile(outputPath) + Expect(err).ToNot(HaveOccurred()) + Expect(string(result)).To(Equal("succeed")) + } + }) + + it("converts between cyclonedx XML and JSON", func() { + outputPath := layer.BOMPath(libcnb.BuildBOM, libcnb.CycloneDXJSON) + Expect(ioutil.WriteFile(outputPath, []byte(` + + + 2021-11-15T16:15:46-05:00 + + + anchore + syft + 0.29.0 + + + + . + + + + + + github.com/BurntSushi/toml + v0.4.1 + pkg:golang/github.com/BurntSushi/toml@v0.4.1 + + +`), 0644)) + + scanner := sherpa.SyftCLIBOMScanner{ + Layer: layer, + Executor: &executor, + Logger: bard.NewLogger(io.Discard), + } + + Expect(scanner.ConvertCycloneDXXMLtoJSON(libcnb.BuildBOM, false)).To(Succeed()) + + Expect(outputPath).To(BeARegularFile()) + Expect(fmt.Sprintf("%s.bak", outputPath)).ToNot(BeARegularFile()) + + input, err := ioutil.ReadFile(outputPath) + Expect(err).ToNot(HaveOccurred()) + Expect(string(input)).To(ContainSubstring(`{"type":"library","name":"github.com/BurntSushi/toml","version":"v0.4.1","purl":"pkg:golang/github.com/BurntSushi/toml@v0.4.1"}`)) + }) + + it("converts between cyclonedx XML and JSON with backup", func() { + outputPath := layer.BOMPath(libcnb.BuildBOM, libcnb.CycloneDXJSON) + Expect(ioutil.WriteFile(outputPath, []byte(` + + + 2021-11-15T16:15:46-05:00 + + + anchore + syft + 0.29.0 + + + + . + + + + + + github.com/BurntSushi/toml + v0.4.1 + pkg:golang/github.com/BurntSushi/toml@v0.4.1 + + +`), 0644)) + + scanner := sherpa.SyftCLIBOMScanner{ + Layer: layer, + Executor: &executor, + Logger: bard.NewLogger(io.Discard), + } + + Expect(scanner.ConvertCycloneDXXMLtoJSON(libcnb.BuildBOM, true)).To(Succeed()) + + Expect(outputPath).To(BeARegularFile()) + + input, err := ioutil.ReadFile(outputPath) + Expect(err).ToNot(HaveOccurred()) + Expect(string(input)).To(ContainSubstring(`{"type":"library","name":"github.com/BurntSushi/toml","version":"v0.4.1","purl":"pkg:golang/github.com/BurntSushi/toml@v0.4.1"}`)) + + outputPath = fmt.Sprintf("%s.bak", outputPath) + Expect(outputPath).To(BeARegularFile()) + + input, err = ioutil.ReadFile(outputPath) + Expect(err).ToNot(HaveOccurred()) + Expect(string(input)).To(ContainSubstring(``)) + }) + }) + +}