Skip to content

Commit

Permalink
fix: dedupliate cataloging binary artifacts
Browse files Browse the repository at this point in the history
  • Loading branch information
@wagoodman
wagoodman committed Jan 20, 2022
1 parent ae58345 commit 1ff7067
Show file tree
Hide file tree
Showing 4 changed files with 600 additions and 93 deletions.
1 change: 1 addition & 0 deletions go.mod
View file
Expand Up @@ -26,6 +26,7 @@ require (
github.com/mitchellh/go-homedir v1.1.0
github.com/muesli/mango v0.0.0-20220118122812-f367188b892e
github.com/muesli/roff v0.1.0
github.com/scylladb/go-set v1.0.2
github.com/slack-go/slack v0.10.1
github.com/spf13/cobra v1.3.0
github.com/stretchr/testify v1.7.0
Expand Down
4 changes: 4 additions & 0 deletions go.sum
View file
Expand Up @@ -332,6 +332,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
github.com/fatih/set v0.2.1 h1:nn2CaJyknWE/6txyUDGwysr3G5QC6xWB/PtVjPBbeaA=
github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0CI=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
Expand Down Expand Up @@ -713,6 +715,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig=
github.com/scylladb/go-set v1.0.2 h1:SkvlMCKhP0wyyct6j+0IHJkBkSZL+TDzZ4E7f7BCcRE=
github.com/scylladb/go-set v1.0.2/go.mod h1:DkpGd78rljTxKAnTDPFqXSGxvETQnJyuSOQwsHycqfs=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
Expand Down
245 changes: 152 additions & 93 deletions internal/pipe/sbom/sbom.go
View file
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/goreleaser/goreleaser/internal/tmpl"
"github.com/goreleaser/goreleaser/pkg/config"
"github.com/goreleaser/goreleaser/pkg/context"
"github.com/scylladb/go-set/strset"
)

// Environment variables to pass through to exec
Expand All @@ -36,43 +37,49 @@ func (Pipe) Default(ctx *context.Context) error {
ids := ids.New("sboms")
for i := range ctx.Config.SBOMs {
cfg := &ctx.Config.SBOMs[i]
if cfg.Cmd == "" {
cfg.Cmd = "syft"
if err := setConfigDefaults(cfg); err != nil {
return err
}
if cfg.Artifacts == "" {
cfg.Artifacts = "archive"
ids.Inc(cfg.ID)
}
return ids.Validate()
}

func setConfigDefaults(cfg *config.SBOM) error {
if cfg.Cmd == "" {
cfg.Cmd = "syft"
}
if cfg.Artifacts == "" {
cfg.Artifacts = "archive"
}
if len(cfg.Documents) == 0 {
switch cfg.Artifacts {
case "binary":
cfg.Documents = []string{"{{ .Binary }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}.sbom"}
case "any":
cfg.Documents = []string{}
default:
cfg.Documents = []string{"{{ .ArtifactName }}.sbom"}
}
if len(cfg.Documents) == 0 {
switch cfg.Artifacts {
case "binary":
cfg.Documents = []string{"{{ .Binary }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}.sbom"}
case "any":
cfg.Documents = []string{}
default:
cfg.Documents = []string{"{{ .ArtifactName }}.sbom"}
}
}
if cfg.Cmd == "syft" {
if len(cfg.Args) == 0 {
cfg.Args = []string{"$artifact", "--file", "$document", "--output", "spdx-json"}
}
if cfg.Cmd == "syft" {
if len(cfg.Args) == 0 {
cfg.Args = []string{"$artifact", "--file", "$document", "--output", "spdx-json"}
}
if len(cfg.Env) == 0 && cfg.Artifacts == "source" || cfg.Artifacts == "archive" {
cfg.Env = []string{
"SYFT_FILE_METADATA_CATALOGER_ENABLED=true",
}
if len(cfg.Env) == 0 && (cfg.Artifacts == "source" || cfg.Artifacts == "archive") {
cfg.Env = []string{
"SYFT_FILE_METADATA_CATALOGER_ENABLED=true",
}
}
if cfg.ID == "" {
cfg.ID = "default"
}

if cfg.Artifacts != "any" && len(cfg.Documents) > 1 {
return fmt.Errorf("multiple SBOM outputs when artifacts=%q is unsupported", cfg.Artifacts)
}
}
if cfg.ID == "" {
cfg.ID = "default"
}

ids.Inc(cfg.ID)
if cfg.Artifacts != "any" && len(cfg.Documents) > 1 {
return fmt.Errorf("multiple SBOM outputs when artifacts=%q is unsupported", cfg.Artifacts)
}
return ids.Validate()
return nil
}

// Run executes the Pipe.
Expand All @@ -96,7 +103,7 @@ func catalogTask(ctx *context.Context, cfg config.SBOM) func() error {
case "archive":
filters = append(filters, artifact.ByType(artifact.UploadableArchive))
case "binary":
filters = append(filters, artifact.ByType(artifact.UploadableBinary))
filters = append(filters, filterBinaryLikeArtifacts(ctx.Artifacts))
case "package":
filters = append(filters, artifact.ByType(artifact.LinuxPackage))
case "any":
Expand All @@ -120,6 +127,35 @@ func catalogTask(ctx *context.Context, cfg config.SBOM) func() error {
}
}

func filterBinaryLikeArtifacts(arts artifact.Artifacts) artifact.Filter {
// find all of the paths for any uploadable binary artifacts
uploadableBins := arts.Filter(artifact.ByType(artifact.UploadableBinary)).List()
uploadableBinPaths := strset.New()
for _, a := range uploadableBins {
uploadableBinPaths.Add(a.Path)
}

// we want to keep any matching artifact that is not a binary that already has a path accounted for
// by another uploadable binary. We always prefer uploadable binary artifacts over binary artifacts.
deduplicateByPath := func(a *artifact.Artifact) bool {
if a.Type == artifact.UploadableBinary {
return true
}
return !uploadableBinPaths.Has(a.Path)
}

return artifact.And(
// allow all of the binary-like artifacts as possible...
artifact.Or(
artifact.ByType(artifact.Binary),
artifact.ByType(artifact.UploadableBinary),
artifact.ByType(artifact.UniversalBinary),
),
// ... but remove any duplicates found
deduplicateByPath,
)
}

func catalog(ctx *context.Context, cfg config.SBOM, artifacts []*artifact.Artifact) error {
for _, a := range artifacts {
newArtifacts, err := catalogArtifact(ctx, cfg, a)
Expand Down Expand Up @@ -152,66 +188,23 @@ func subprocessDistPath(distDir string, pathRelativeToCwd string) (string, error
}

func catalogArtifact(ctx *context.Context, cfg config.SBOM, a *artifact.Artifact) ([]*artifact.Artifact, error) {
env := ctx.Env.Copy()
artifactDisplayName := "(any)"
templater := tmpl.New(ctx).WithEnv(env)
args, envs, paths, err := applyTemplate(ctx, cfg, a)
if err != nil {
return nil, fmt.Errorf("cataloging artifacts failed: %w", err)
}

if a != nil {
procPath, err := subprocessDistPath(ctx.Config.Dist, a.Path)
if err != nil {
return nil, fmt.Errorf("cataloging artifacts failed: cannot determine artifact path for %q: %w", a.Path, err)
}
env["artifact"] = procPath
env["artifactID"] = a.ID()

templater = templater.WithArtifact(a, nil)
artifactDisplayName = a.Path
}

var paths []string
for idx, sbom := range cfg.Documents {
input := filepath.Join(ctx.Config.Dist, expand(sbom, env))

path, err := templater.Apply(input)
if err != nil {
return nil, fmt.Errorf("cataloging artifacts failed: %s: invalid template: %w", input, err)
}

path, err = filepath.Abs(path)
if err != nil {
return nil, fmt.Errorf("cataloging artifacts failed: unable to create artifact path %q: %w", sbom, err)
}

procPath, err := subprocessDistPath(ctx.Config.Dist, path)
if err != nil {
return nil, fmt.Errorf("cataloging artifacts failed: cannot determine document path for %q: %w", path, err)
}

env[fmt.Sprintf("document%d", idx)] = procPath
if idx == 0 {
env["document"] = procPath
}

paths = append(paths, procPath)
}

var names []string
for _, p := range paths {
names = append(names, filepath.Base(p))
}

fields := log.Fields{"cmd": cfg.Cmd, "artifact": artifactDisplayName, "sboms": strings.Join(names, ", ")}

// nolint:prealloc
var args []string
for _, arg := range cfg.Args {
renderedArg, err := templater.Apply(expand(arg, env))
if err != nil {
return nil, fmt.Errorf("cataloging artifacts failed: %s: invalid template: %w", arg, err)
}
args = append(args, renderedArg)
}

// The GoASTScanner flags this as a security risk.
// However, this works as intended. The nosec annotation
// tells the scanner to ignore this.
Expand All @@ -223,7 +216,7 @@ func catalogArtifact(ctx *context.Context, cfg config.SBOM, a *artifact.Artifact
cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", key, value))
}
}
cmd.Env = append(cmd.Env, cfg.Env...)
cmd.Env = append(cmd.Env, envs...)
cmd.Dir = ctx.Config.Dist

var b bytes.Buffer
Expand All @@ -238,27 +231,19 @@ func catalogArtifact(ctx *context.Context, cfg config.SBOM, a *artifact.Artifact

var artifacts []*artifact.Artifact

for _, sbom := range cfg.Documents {
templater = tmpl.New(ctx).WithEnv(env)
if a != nil {
env["artifact"] = a.Name
templater = templater.WithArtifact(a, nil)
for _, path := range paths {
if !filepath.IsAbs(path) {
path = filepath.Join(ctx.Config.Dist, path)
}

name, err := templater.Apply(expand(sbom, env))
matches, err := filepath.Glob(path)
if err != nil {
return nil, fmt.Errorf("cataloging artifacts failed: %s: invalid template: %w", a, err)
}

search := filepath.Join(ctx.Config.Dist, name)
matches, err := filepath.Glob(search)
if err != nil {
return nil, fmt.Errorf("cataloging artifacts: failed to find SBOM artifact %q: %w", search, err)
return nil, fmt.Errorf("cataloging artifacts: failed to find SBOM artifact %q: %w", path, err)
}
for _, match := range matches {
artifacts = append(artifacts, &artifact.Artifact{
Type: artifact.SBOM,
Name: name,
Name: filepath.Base(path),
Path: match,
Extra: map[string]interface{}{
artifact.ExtraID: cfg.ID,
Expand All @@ -271,6 +256,80 @@ func catalogArtifact(ctx *context.Context, cfg config.SBOM, a *artifact.Artifact
return artifacts, nil
}

func applyTemplate(ctx *context.Context, cfg config.SBOM, a *artifact.Artifact) ([]string, []string, []string, error) {
env := ctx.Env.Copy()
templater := tmpl.New(ctx).WithEnv(env)

if a != nil {
procPath, err := subprocessDistPath(ctx.Config.Dist, a.Path)
if err != nil {
return nil, nil, nil, fmt.Errorf("cataloging artifacts failed: cannot determine artifact path for %q: %w", a.Path, err)
}
env["artifact"] = procPath
env["artifactID"] = a.ID()

templater = templater.WithArtifact(a, nil)
}

var envs []string
for _, keyValue := range cfg.Env {
renderedKeyValue, err := templater.Apply(expand(keyValue, env))
if err != nil {
return nil, nil, nil, fmt.Errorf("env %q: invalid template: %w", keyValue, err)
}
envs = append(envs, renderedKeyValue)

fields := strings.Split(renderedKeyValue, "=")
key := fields[0]
renderedValue := strings.Join(fields[1:], "=")
env[key] = renderedValue
}

var paths []string
for idx, sbom := range cfg.Documents {
input := expand(sbom, env)
if !filepath.IsAbs(input) {
// assume any absolute path is handled correctly and assume that any relative path is not already
// adjusted to reference the dist path
input = filepath.Join(ctx.Config.Dist, input)
}

path, err := templater.Apply(input)
if err != nil {
return nil, nil, nil, fmt.Errorf("input %q: invalid template: %w", input, err)
}

path, err = filepath.Abs(path)
if err != nil {
return nil, nil, nil, fmt.Errorf("unable to create artifact path %q: %w", sbom, err)
}

procPath, err := subprocessDistPath(ctx.Config.Dist, path)
if err != nil {
return nil, nil, nil, fmt.Errorf("cannot determine document path for %q: %w", path, err)
}

env[fmt.Sprintf("document%d", idx)] = procPath
if idx == 0 {
env["document"] = procPath
}

paths = append(paths, procPath)
}

// nolint:prealloc
var args []string
for _, arg := range cfg.Args {
renderedArg, err := templater.Apply(expand(arg, env))
if err != nil {
return nil, nil, nil, fmt.Errorf("arg %q: invalid template: %w", arg, err)
}
args = append(args, renderedArg)
}

return args, envs, paths, nil
}

func expand(s string, env map[string]string) string {
return os.Expand(s, func(key string) string {
return env[key]
Expand Down

0 comments on commit 1ff7067

Please sign in to comment.