[THOG-793] - Return all unverified results

README.md

@@ -180,6 +180,7 @@ Flags:
       --concurrency=1            Number of concurrent workers.
       --no-verification          Don't verify the results.
       --only-verified            Only output verified results.
+      --filter-unverified        Only output first unverified result per chunk per detector if there are more than one results.
       --print-avg-detector-time  Print the average time spent on each detector.
       --no-update                Don't check for updates.
   -i, --include-paths=INCLUDE-PATHS

main.go

@@ -30,15 +30,16 @@ import (
 )
 
 var (
-	cli            = kingpin.New("TruffleHog", "TruffleHog is a tool for finding credentials.")
-	cmd            string
-	debug          = cli.Flag("debug", "Run in debug mode.").Bool()
-	trace          = cli.Flag("trace", "Run in trace mode.").Bool()
-	jsonOut        = cli.Flag("json", "Output in JSON format.").Short('j').Bool()
-	jsonLegacy     = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()
-	concurrency    = cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int()
-	noVerification = cli.Flag("no-verification", "Don't verify the results.").Bool()
-	onlyVerified   = cli.Flag("only-verified", "Only output verified results.").Bool()
+	cli              = kingpin.New("TruffleHog", "TruffleHog is a tool for finding credentials.")
+	cmd              string
+	debug            = cli.Flag("debug", "Run in debug mode.").Bool()
+	trace            = cli.Flag("trace", "Run in trace mode.").Bool()
+	jsonOut          = cli.Flag("json", "Output in JSON format.").Short('j').Bool()
+	jsonLegacy       = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()
+	concurrency      = cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int()
+	noVerification   = cli.Flag("no-verification", "Don't verify the results.").Bool()
+	onlyVerified     = cli.Flag("only-verified", "Only output verified results.").Bool()
+	filterUnverified = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool()
 	// rules = cli.Flag("rules", "Path to file with custom rules.").String()
 	printAvgDetectorTime = cli.Flag("print-avg-detector-time", "Print the average time spent on each detector.").Bool()
 	noUpdate             = cli.Flag("no-update", "Don't check for updates.").Bool()
@@ -172,6 +173,7 @@ func run(state overseer.State) {
 		engine.WithConcurrency(*concurrency),
 		engine.WithDecoders(decoders.DefaultDecoders()...),
 		engine.WithDetectors(!*noVerification, engine.DefaultDetectors()...),
+		engine.WithFilterUnverified(*filterUnverified),
 	)
 
 	filter, err := common.FilterFromFiles(*gitScanIncludePaths, *gitScanExcludePaths)

pkg/detectors/abbysale/abbysale.go

@@ -70,5 +70,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/abstract/abstract.go

@@ -70,5 +70,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/abuseipdb/abuseipdb.go

@@ -81,5 +81,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/accuweather/accuweather.go

@@ -68,5 +68,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/adafruitio/adafruitio.go

@@ -68,5 +68,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/adobeio/adobeio.go

@@ -80,5 +80,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/adzuna/adzuna.go

@@ -78,5 +78,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/aeroworkflow/aeroworkflow.go

@@ -81,5 +81,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/agora/agora.go

@@ -81,5 +81,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/aha/aha.go

@@ -71,5 +71,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/airbrakeprojectkey/airbrakeprojectkey.go

@@ -85,5 +85,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/airbrakeuserkey/airbrakeuserkey.go

@@ -68,5 +68,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/airship/airship.go

@@ -71,5 +71,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/airtableapikey/airtableapikey.go

@@ -79,5 +79,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/airvisual/airvisual.go

@@ -71,5 +71,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/alconost/alconost.go

@@ -74,5 +74,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/alegra/alegra.go

@@ -80,5 +80,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/aletheiaapi/aletheiaapi.go

@@ -72,5 +72,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/algoliaadminkey/algoliaadminkey.go

@@ -78,5 +78,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			results = append(results, s1)
 		}
 	}
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/alibaba/alibaba.go

@@ -128,5 +128,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/alienvault/alienvault.go

@@ -69,5 +69,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/allsports/allsports.go

@@ -75,5 +75,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/amadeus/amadeus.go

@@ -86,5 +86,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/ambee/ambee.go

@@ -70,5 +70,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/amplitudeapikey/amplitudeapikey.go

@@ -79,5 +79,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/anypoint/anypoint.go

@@ -80,5 +80,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/apacta/apacta.go

@@ -69,5 +69,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/api2cart/api2cart.go

@@ -80,7 +80,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }
 
 type Response struct {

pkg/detectors/apideck/apideck.go

@@ -82,5 +82,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/apiflash/apiflash.go

@@ -78,5 +78,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/apifonica/apifonica.go

@@ -81,5 +81,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/apify/apify.go

@@ -68,5 +68,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/apilayer/apilayer.go

@@ -2,12 +2,12 @@ package apilayer
 
 import (
 	"context"
-	"net/http"
-	"regexp"
-		"strings"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"net/http"
+	"regexp"
+	"strings"
 )
 
 type Scanner struct{}
@@ -18,7 +18,7 @@ var _ detectors.Detector = (*Scanner)(nil)
 var (
 	client = common.SaneHttpClient()
 
-	//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
+	// Make sure that your group is surrounded in boundry characters such as below to reduce false positives
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apilayer"}) + `\b([a-zA-Z0-9]{32})\b`)
 )
 
@@ -57,7 +57,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}
@@ -68,5 +68,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/apimatic/apimatic.go

@@ -82,5 +82,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			results = append(results, s1)
 		}
 	}
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/apiscience/apiscience.go

@@ -70,5 +70,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/apitemplate/apitemplate.go

@@ -70,5 +70,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/apollo/apollo.go

@@ -69,5 +69,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/appcues/appcues.go

@@ -90,5 +90,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/appfollow/appfollow.go

@@ -73,5 +73,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/appointedd/appointedd.go

@@ -74,5 +74,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/appsynergy/appsynergy.go

@@ -70,5 +70,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/apptivo/apptivo.go

@@ -89,5 +89,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/artifactory/artifactory.go

@@ -81,5 +81,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/artsy/artsy.go

@@ -80,5 +80,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/asanaoauth/asanaoauth.go

@@ -70,5 +70,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }

pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken.go

@@ -69,5 +69,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		results = append(results, s1)
 	}
 
-	return detectors.CleanResults(results), nil
+	return results, nil
 }