From a159c70f95b4b700428ff0eb9f06789d528123d4 Mon Sep 17 00:00:00 2001 From: Miccah Castorina Date: Thu, 8 Dec 2022 10:52:32 -0600 Subject: [PATCH 1/2] Add configuration parsing for custom detectors --- main.go | 9 +++++++++ pkg/config/config.go | 48 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 pkg/config/config.go diff --git a/main.go b/main.go index 3835ca5f00e2..6e78c6c9215c 100644 --- a/main.go +++ b/main.go @@ -19,6 +19,7 @@ import ( "gopkg.in/alecthomas/kingpin.v2" "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/config" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/decoders" "github.com/trufflesecurity/trufflehog/v3/pkg/engine" @@ -41,6 +42,7 @@ var ( noVerification = cli.Flag("no-verification", "Don't verify the results.").Bool() onlyVerified = cli.Flag("only-verified", "Only output verified results.").Bool() filterUnverified = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool() + configFilename = cli.Flag("config", "Path to configuration file.").String() // rules = cli.Flag("rules", "Path to file with custom rules.").String() printAvgDetectorTime = cli.Flag("print-avg-detector-time", "Print the average time spent on each detector.").Bool() noUpdate = cli.Flag("no-update", "Don't check for updates.").Bool() @@ -181,10 +183,17 @@ func run(state overseer.State) { defer func() { _ = sync() }() + conf, err := config.Read(*configFilename) + if err != nil { + logger.Error(err, "error parsing the provided configuration file") + os.Exit(1) + } + e := engine.Start(ctx, engine.WithConcurrency(*concurrency), engine.WithDecoders(decoders.DefaultDecoders()...), engine.WithDetectors(!*noVerification, engine.DefaultDetectors()...), + engine.WithDetectors(!*noVerification, conf.Detectors...), engine.WithFilterUnverified(*filterUnverified), ) diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 000000000000..cf5c11c6a581 --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,48 @@ +package config + +import ( + "os" + + "github.com/trufflesecurity/trufflehog/v3/pkg/custom_detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/custom_detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/protoyaml" +) + +// Config holds user supplied configuration. +type Config struct { + Detectors []detectors.Detector +} + +// Read parses a given filename into a Config. +func Read(filename string) (*Config, error) { + if filename == "" { + return &Config{}, nil + } + input, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + return NewYAML(input) +} + +// NewYAML parses the given YAML data into a Config. +func NewYAML(input []byte) (*Config, error) { + // Parse the raw YAML into a structure. + var messages custom_detectorspb.CustomDetectors + if err := protoyaml.UnmarshalStrict(input, &messages); err != nil { + return nil, err + } + // Convert the structured YAML into detectors. + var detectors []detectors.Detector + for _, detectorConfig := range messages.Detectors { + detector, err := custom_detectors.NewWebhookCustomRegex(detectorConfig) + if err != nil { + return nil, err + } + detectors = append(detectors, detector) + } + return &Config{ + Detectors: detectors, + }, nil +} From 4416efb5f88d2d04eb384c20ca6f1d244822a0e2 Mon Sep 17 00:00:00 2001 From: Miccah Castorina Date: Tue, 20 Dec 2022 09:52:56 -0600 Subject: [PATCH 2/2] Error on empty filename --- main.go | 14 +++++++++----- pkg/config/config.go | 3 --- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/main.go b/main.go index 6e78c6c9215c..7c59c35e6da2 100644 --- a/main.go +++ b/main.go @@ -42,7 +42,7 @@ var ( noVerification = cli.Flag("no-verification", "Don't verify the results.").Bool() onlyVerified = cli.Flag("only-verified", "Only output verified results.").Bool() filterUnverified = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool() - configFilename = cli.Flag("config", "Path to configuration file.").String() + configFilename = cli.Flag("config", "Path to configuration file.").ExistingFile() // rules = cli.Flag("rules", "Path to file with custom rules.").String() printAvgDetectorTime = cli.Flag("print-avg-detector-time", "Print the average time spent on each detector.").Bool() noUpdate = cli.Flag("no-update", "Don't check for updates.").Bool() @@ -183,10 +183,14 @@ func run(state overseer.State) { defer func() { _ = sync() }() - conf, err := config.Read(*configFilename) - if err != nil { - logger.Error(err, "error parsing the provided configuration file") - os.Exit(1) + conf := &config.Config{} + if *configFilename != "" { + var err error + conf, err = config.Read(*configFilename) + if err != nil { + logger.Error(err, "error parsing the provided configuration file") + os.Exit(1) + } } e := engine.Start(ctx, diff --git a/pkg/config/config.go b/pkg/config/config.go index cf5c11c6a581..9d5efb886099 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -16,9 +16,6 @@ type Config struct { // Read parses a given filename into a Config. func Read(filename string) (*Config, error) { - if filename == "" { - return &Config{}, nil - } input, err := os.ReadFile(filename) if err != nil { return nil, err