Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Introduce a single SBOM document #606

Merged
merged 16 commits into from
Nov 5, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ SUCCESS := $(BOLD)$(GREEN)
COVERAGE_THRESHOLD := 62
# CI cache busting values; change these if you want CI to not use previous stored cache
INTEGRATION_CACHE_BUSTER="88738d2f"
CLI_CACHE_BUSTER="789bacdf"
CLI_CACHE_BUSTER="9a2c03cf"
BOOTSTRAP_CACHE="c7afb99ad"

## Build variables
Expand Down
16 changes: 12 additions & 4 deletions cmd/packages.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/format"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
"github.com/pkg/profile"
"github.com/spf13/cobra"
Expand Down Expand Up @@ -260,21 +261,29 @@ func packagesExecWorker(userInput string) <-chan error {
}

if appConfig.Anchore.Host != "" {
if err := runPackageSbomUpload(src, src.Metadata, catalog, d, appConfig.Package.Cataloger.ScopeOpt); err != nil {
if err := runPackageSbomUpload(src, src.Metadata, catalog, d); err != nil {
errs <- err
return
}
}

sbomResult := sbom.SBOM{
Artifacts: sbom.Artifacts{
PackageCatalog: catalog,
Distro: d,
},
Source: src.Metadata,
}

bus.Publish(partybus.Event{
Type: event.PresenterReady,
Value: f.Presenter(catalog, &src.Metadata, d, appConfig.Package.Cataloger.ScopeOpt),
Value: f.Presenter(sbomResult),
})
}()
return errs
}

func runPackageSbomUpload(src *source.Source, s source.Metadata, catalog *pkg.Catalog, d *distro.Distro, scope source.Scope) error {
func runPackageSbomUpload(src *source.Source, s source.Metadata, catalog *pkg.Catalog, d *distro.Distro) error {
wagoodman marked this conversation as resolved.
Show resolved Hide resolved
log.Infof("uploading results to %s", appConfig.Anchore.Host)

if src.Metadata.Scheme != source.ImageScheme {
Expand Down Expand Up @@ -314,7 +323,6 @@ func runPackageSbomUpload(src *source.Source, s source.Metadata, catalog *pkg.Ca
Distro: d,
Dockerfile: dockerfileContents,
OverwriteExistingUpload: appConfig.Anchore.OverwriteExistingImage,
Scope: scope,
Timeout: appConfig.Anchore.ImportTimeout,
}

Expand Down
11 changes: 6 additions & 5 deletions cmd/power_user.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"fmt"
"sync"

"github.com/anchore/syft/syft/sbom"

"github.com/anchore/stereoscope"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/bus"
Expand Down Expand Up @@ -107,17 +109,16 @@ func powerUserExecWorker(userInput string) <-chan error {
}
defer cleanup()

analysisResults := poweruser.JSONDocumentConfig{
SourceMetadata: src.Metadata,
ApplicationConfig: *appConfig,
analysisResults := sbom.SBOM{
Source: src.Metadata,
}

wg := &sync.WaitGroup{}
for _, task := range tasks {
wg.Add(1)
go func(task powerUserTask) {
defer wg.Done()
if err = task(&analysisResults, src); err != nil {
if err = task(&analysisResults.Artifacts, src); err != nil {
errs <- err
return
}
Expand All @@ -128,7 +129,7 @@ func powerUserExecWorker(userInput string) <-chan error {

bus.Publish(partybus.Event{
Type: event.PresenterReady,
Value: poweruser.NewJSONPresenter(analysisResults),
Value: poweruser.NewJSONPresenter(analysisResults, *appConfig),
})
}()
return errs
Expand Down
17 changes: 9 additions & 8 deletions cmd/power_user_tasks.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,14 @@ import (
"crypto"
"fmt"

"github.com/anchore/syft/internal/presenter/poweruser"
"github.com/anchore/syft/syft/sbom"

"github.com/anchore/syft/syft"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/source"
)

type powerUserTask func(*poweruser.JSONDocumentConfig, *source.Source) error
type powerUserTask func(*sbom.Artifacts, *source.Source) error

func powerUserTasks() ([]powerUserTask, error) {
var tasks []powerUserTask
Expand Down Expand Up @@ -42,7 +43,7 @@ func catalogPackagesTask() (powerUserTask, error) {
return nil, nil
}

task := func(results *poweruser.JSONDocumentConfig, src *source.Source) error {
task := func(results *sbom.Artifacts, src *source.Source) error {
packageCatalog, theDistro, err := syft.CatalogPackages(src, appConfig.Package.Cataloger.ScopeOpt)
if err != nil {
return err
Expand All @@ -64,7 +65,7 @@ func catalogFileMetadataTask() (powerUserTask, error) {

metadataCataloger := file.NewMetadataCataloger()

task := func(results *poweruser.JSONDocumentConfig, src *source.Source) error {
task := func(results *sbom.Artifacts, src *source.Source) error {
resolver, err := src.FileResolver(appConfig.FileMetadata.Cataloger.ScopeOpt)
if err != nil {
return err
Expand Down Expand Up @@ -110,7 +111,7 @@ func catalogFileDigestsTask() (powerUserTask, error) {
return nil, err
}

task := func(results *poweruser.JSONDocumentConfig, src *source.Source) error {
task := func(results *sbom.Artifacts, src *source.Source) error {
resolver, err := src.FileResolver(appConfig.FileMetadata.Cataloger.ScopeOpt)
if err != nil {
return err
Expand Down Expand Up @@ -142,7 +143,7 @@ func catalogSecretsTask() (powerUserTask, error) {
return nil, err
}

task := func(results *poweruser.JSONDocumentConfig, src *source.Source) error {
task := func(results *sbom.Artifacts, src *source.Source) error {
resolver, err := src.FileResolver(appConfig.Secrets.Cataloger.ScopeOpt)
if err != nil {
return err
Expand Down Expand Up @@ -170,7 +171,7 @@ func catalogFileClassificationsTask() (powerUserTask, error) {
return nil, err
}

task := func(results *poweruser.JSONDocumentConfig, src *source.Source) error {
task := func(results *sbom.Artifacts, src *source.Source) error {
resolver, err := src.FileResolver(appConfig.FileClassification.Cataloger.ScopeOpt)
if err != nil {
return err
Expand All @@ -197,7 +198,7 @@ func catalogContentsTask() (powerUserTask, error) {
return nil, err
}

task := func(results *poweruser.JSONDocumentConfig, src *source.Source) error {
task := func(results *sbom.Artifacts, src *source.Source) error {
resolver, err := src.FileResolver(appConfig.FileContents.Cataloger.ScopeOpt)
if err != nil {
return err
Expand Down
3 changes: 1 addition & 2 deletions internal/anchore/import.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ type ImportConfig struct {
Distro *distro.Distro
Dockerfile []byte
OverwriteExistingUpload bool
Scope source.Scope
Timeout uint
}

Expand Down Expand Up @@ -74,7 +73,7 @@ func (c *Client) Import(ctx context.Context, cfg ImportConfig) error {
prog.N++
sessionID := startOperation.Uuid

packageDigest, err := importPackageSBOM(authedCtx, c.client.ImportsApi, sessionID, cfg.SourceMetadata, cfg.Catalog, cfg.Distro, cfg.Scope, stage)
packageDigest, err := importPackageSBOM(authedCtx, c.client.ImportsApi, sessionID, cfg.SourceMetadata, cfg.Catalog, cfg.Distro, stage)
if err != nil {
return fmt.Errorf("failed to import Package SBOM: %w", err)
}
Expand Down
19 changes: 15 additions & 4 deletions internal/anchore/import_package_sbom.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import (
"fmt"
"net/http"

"github.com/anchore/syft/syft/sbom"

"github.com/anchore/syft/internal/formats/syftjson"

"github.com/wagoodman/go-progress"
Expand All @@ -24,10 +26,19 @@ type packageSBOMImportAPI interface {
ImportImagePackages(context.Context, string, external.ImagePackageManifest) (external.ImageImportContentResponse, *http.Response, error)
}

func packageSbomModel(s source.Metadata, catalog *pkg.Catalog, d *distro.Distro, scope source.Scope) (*external.ImagePackageManifest, error) {
func packageSbomModel(srcMetadata source.Metadata, catalog *pkg.Catalog, d *distro.Distro) (*external.ImagePackageManifest, error) {
var buf bytes.Buffer

err := syftjson.Format().Presenter(catalog, &s, d, scope).Present(&buf)
// TODO: once the top-level API is refactored and SBOMs are the unit of work, then this function will be passed an SBOM and there would be no more need to create an SBOM object here.
s := sbom.SBOM{
Artifacts: sbom.Artifacts{
PackageCatalog: catalog,
Distro: d,
},
Source: srcMetadata,
}

err := syftjson.Format().Presenter(s).Present(&buf)
if err != nil {
return nil, fmt.Errorf("unable to serialize results: %w", err)
}
Expand All @@ -41,11 +52,11 @@ func packageSbomModel(s source.Metadata, catalog *pkg.Catalog, d *distro.Distro,
return &model, nil
}

func importPackageSBOM(ctx context.Context, api packageSBOMImportAPI, sessionID string, s source.Metadata, catalog *pkg.Catalog, d *distro.Distro, scope source.Scope, stage *progress.Stage) (string, error) {
func importPackageSBOM(ctx context.Context, api packageSBOMImportAPI, sessionID string, s source.Metadata, catalog *pkg.Catalog, d *distro.Distro, stage *progress.Stage) (string, error) {
log.Debug("importing package SBOM")
stage.Current = "package SBOM"

model, err := packageSbomModel(s, catalog, d, scope)
model, err := packageSbomModel(s, catalog, d)
if err != nil {
return "", fmt.Errorf("unable to create PackageSBOM model: %w", err)
}
Expand Down
18 changes: 14 additions & 4 deletions internal/anchore/import_package_sbom_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import (
"strings"
"testing"

"github.com/anchore/syft/syft/sbom"

"github.com/anchore/client-go/pkg/external"
"github.com/anchore/syft/internal/formats/syftjson"
syftjsonModel "github.com/anchore/syft/internal/formats/syftjson/model"
Expand Down Expand Up @@ -72,7 +74,7 @@ func TestPackageSbomToModel(t *testing.T) {

c := pkg.NewCatalog(p)

model, err := packageSbomModel(m, c, &d, source.AllLayersScope)
model, err := packageSbomModel(m, c, &d)
if err != nil {
t.Fatalf("unable to generate model from source material: %+v", err)
}
Expand All @@ -84,8 +86,16 @@ func TestPackageSbomToModel(t *testing.T) {
t.Fatalf("unable to marshal model: %+v", err)
}

s := sbom.SBOM{
Artifacts: sbom.Artifacts{
PackageCatalog: c,
Distro: &d,
},
Source: m,
}

var buf bytes.Buffer
pres := syftjson.Format().Presenter(c, &m, &d, source.AllLayersScope)
pres := syftjson.Format().Presenter(s)
if err := pres.Present(&buf); err != nil {
t.Fatalf("unable to get expected json: %+v", err)
}
Expand Down Expand Up @@ -187,7 +197,7 @@ func TestPackageSbomImport(t *testing.T) {

d, _ := distro.NewDistro(distro.CentOS, "8.0", "")

theModel, err := packageSbomModel(m, catalog, &d, source.AllLayersScope)
theModel, err := packageSbomModel(m, catalog, &d)
if err != nil {
t.Fatalf("could not get sbom model: %+v", err)
}
Expand Down Expand Up @@ -226,7 +236,7 @@ func TestPackageSbomImport(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {

digest, err := importPackageSBOM(context.TODO(), test.api, sessionID, m, catalog, &d, source.AllLayersScope, &progress.Stage{})
digest, err := importPackageSBOM(context.TODO(), test.api, sessionID, m, catalog, &d, &progress.Stage{})

// validate error handling
if err != nil && !test.expectsError {
Expand Down
21 changes: 17 additions & 4 deletions internal/formats/common/testutils/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
"github.com/sergi/go-diff/diffmatchpatch"
"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -90,7 +91,7 @@ func AssertPresenterAgainstGoldenSnapshot(t *testing.T, pres presenter.Presenter
}
}

func ImageInput(t testing.TB, testImage string, options ...ImageOption) (*pkg.Catalog, source.Metadata, *distro.Distro) {
func ImageInput(t testing.TB, testImage string, options ...ImageOption) sbom.SBOM {
t.Helper()
catalog := pkg.NewCatalog()
var cfg imageCfg
Expand All @@ -117,7 +118,13 @@ func ImageInput(t testing.TB, testImage string, options ...ImageOption) (*pkg.Ca
dist, err := distro.NewDistro(distro.Debian, "1.2.3", "like!")
assert.NoError(t, err)

return catalog, src.Metadata, &dist
return sbom.SBOM{
Artifacts: sbom.Artifacts{
PackageCatalog: catalog,
Distro: &dist,
},
Source: src.Metadata,
}
}

func populateImageCatalog(catalog *pkg.Catalog, img *image.Image) {
Expand Down Expand Up @@ -167,7 +174,7 @@ func populateImageCatalog(catalog *pkg.Catalog, img *image.Image) {
})
}

func DirectoryInput(t testing.TB) (*pkg.Catalog, source.Metadata, *distro.Distro) {
func DirectoryInput(t testing.TB) sbom.SBOM {
catalog := newDirectoryCatalog()

dist, err := distro.NewDistro(distro.Debian, "1.2.3", "like!")
Expand All @@ -176,7 +183,13 @@ func DirectoryInput(t testing.TB) (*pkg.Catalog, source.Metadata, *distro.Distro
src, err := source.NewFromDirectory("/some/path")
assert.NoError(t, err)

return catalog, src.Metadata, &dist
return sbom.SBOM{
Artifacts: sbom.Artifacts{
PackageCatalog: catalog,
Distro: &dist,
},
Source: src.Metadata,
}
}

func newDirectoryCatalog() *pkg.Catalog {
Expand Down
9 changes: 3 additions & 6 deletions internal/formats/cyclonedx12xml/encoder.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,10 @@ import (
"encoding/xml"
"io"

"github.com/anchore/syft/syft/distro"

"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
"github.com/anchore/syft/syft/sbom"
)

func encoder(output io.Writer, catalog *pkg.Catalog, srcMetadata *source.Metadata, d *distro.Distro, scope source.Scope) error {
func encoder(output io.Writer, s sbom.SBOM) error {
enc := xml.NewEncoder(output)
enc.Indent("", " ")

Expand All @@ -19,7 +16,7 @@ func encoder(output io.Writer, catalog *pkg.Catalog, srcMetadata *source.Metadat
return err
}

err = enc.Encode(toFormatModel(catalog, srcMetadata, d, scope))
err = enc.Encode(toFormatModel(s))
if err != nil {
return err
}
Expand Down
8 changes: 2 additions & 6 deletions internal/formats/cyclonedx12xml/encoder_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,27 +5,23 @@ import (
"regexp"
"testing"

"github.com/anchore/syft/syft/source"

"github.com/anchore/syft/internal/formats/common/testutils"
)

var updateCycloneDx = flag.Bool("update-cyclonedx", false, "update the *.golden files for cyclone-dx presenters")

func TestCycloneDxDirectoryPresenter(t *testing.T) {
catalog, metadata, _ := testutils.DirectoryInput(t)
testutils.AssertPresenterAgainstGoldenSnapshot(t,
Format().Presenter(catalog, &metadata, nil, source.SquashedScope),
Format().Presenter(testutils.DirectoryInput(t)),
*updateCycloneDx,
cycloneDxRedactor,
)
}

func TestCycloneDxImagePresenter(t *testing.T) {
testImage := "image-simple"
catalog, metadata, _ := testutils.ImageInput(t, testImage)
testutils.AssertPresenterAgainstGoldenImageSnapshot(t,
Format().Presenter(catalog, &metadata, nil, source.SquashedScope),
Format().Presenter(testutils.ImageInput(t, testImage)),
testImage,
*updateCycloneDx,
cycloneDxRedactor,
Expand Down