Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Enable skipping of particular key IDs #930

Merged
merged 2 commits into from Nov 18, 2022
Merged

Enable skipping of particular key IDs #930

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
069dec2
Enable skipping of particular key IDs
dustin-decker Nov 18, 2022
4d304e6
update test
dustin-decker Nov 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
41 changes: 36 additions & 5 deletions pkg/detectors/aws/aws.go
View file
Open in desktop
Expand Up @@ -17,10 +17,35 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)

type Scanner struct{}
type scanner struct {
skipIDs map[string]struct{}
}

func New(opts ...func(*scanner)) *scanner {
scanner := &scanner{
skipIDs: map[string]struct{}{},
}
for _, opt := range opts {

opt(scanner)
}

return scanner
}

func WithSkipIDs(skipIDs []string) func(*scanner) {
return func(s *scanner) {
ids := map[string]struct{}{}
for _, id := range skipIDs {
ids[id] = struct{}{}
}

s.skipIDs = ids
}
}

// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
// Ensure the scanner satisfies the interface at compile time.
var _ detectors.Detector = (*scanner)(nil)

var (
client = common.SaneHttpClient()
Expand All @@ -37,7 +62,7 @@ var (

// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
func (s scanner) Keywords() []string {
return []string{
"AKIA",
"ABIA",
Expand All @@ -60,7 +85,7 @@ func GetHMAC(key []byte, data []byte) []byte {
}

// FromData will find and optionally verify AWS secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)

idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
Expand All @@ -72,6 +97,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
resIDMatch := strings.TrimSpace(idMatch[1])

if s.skipIDs != nil {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit (optional): I think s.skipIDs is always non-nil since it's initialized in New()

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, it's probably not needed now since I made the struct not exported.

if _, ok := s.skipIDs[resIDMatch]; ok {
continue
}
}

for _, secretMatch := range secretMatches {
if len(secretMatch) != 2 {
continue
Expand Down
34 changes: 24 additions & 10 deletions pkg/detectors/aws/aws_test.go
View file
Open in desktop
Expand Up @@ -39,14 +39,14 @@ func TestAWS_FromChunk(t *testing.T) {
}
tests := []struct {
name string
s Scanner
s scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
s: scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
Expand All @@ -68,7 +68,7 @@ func TestAWS_FromChunk(t *testing.T) {
},
{
name: "found, unverified",
s: Scanner{},
s: scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
Expand All @@ -86,7 +86,7 @@ func TestAWS_FromChunk(t *testing.T) {
},
{
name: "not found",
s: Scanner{},
s: scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
Expand All @@ -97,7 +97,7 @@ func TestAWS_FromChunk(t *testing.T) {
},
{
name: "found two, one included for every ID found",
s: Scanner{},
s: scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("The verified ID is %s with a secret of %s, but the unverified ID is %s and this is the secret %s", id, secret, inactiveID, inactiveSecret)),
Expand All @@ -124,7 +124,7 @@ func TestAWS_FromChunk(t *testing.T) {
},
{
name: "not found, because unverified secret was a hash",
s: Scanner{},
s: scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", hash, id)), // The secret would satisfy the regex but be filtered out after not passing validation.
Expand All @@ -135,7 +135,7 @@ func TestAWS_FromChunk(t *testing.T) {
},
{
name: "found two, returned both because the active secret for one paired with the inactive ID, despite the hash",
s: Scanner{},
s: scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("The verified ID is %s with a secret of %s, but the unverified ID is %s and the secret is this hash %s", id, secret, inactiveID, hash)),
Expand All @@ -162,7 +162,7 @@ func TestAWS_FromChunk(t *testing.T) {
},
{
name: "found, unverified, with leading +",
s: Scanner{},
s: scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation
Expand All @@ -177,10 +177,24 @@ func TestAWS_FromChunk(t *testing.T) {
},
wantErr: false,
},
{
name: "skipped",
s: scanner{
skipIDs: map[string]struct{}{
"AKIAWARWQKZNHMZBLY4I": {},
},
},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation
verify: true,
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
s := tt.s
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("AWS.FromData() error = %v, wantErr %v", err, tt.wantErr)
Expand All @@ -201,7 +215,7 @@ func TestAWS_FromChunk(t *testing.T) {

func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
s := scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
for n := 0; n < b.N; n++ {
Expand Down
2 changes: 1 addition & 1 deletion pkg/engine/defaults.go
View file
Open in desktop
Expand Up @@ -737,7 +737,7 @@ func DefaultDetectors() []detectors.Detector {
&heroku.Scanner{},
&linearapi.Scanner{},
&alibaba.Scanner{},
&aws.Scanner{},
aws.New(),
&azure.Scanner{},
&slack.Scanner{}, // has 4 secret types
&gitlabv2.Scanner{},
Expand Down