Add filter as scan option to gitlab module's git scan (#919)

trufflesecurity · Nov 15, 2022 · 3d50197 · 3d50197
1 parent 64cfe4d
commit 3d50197
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 4 deletions.
diff --git a/main.go b/main.go
@@ -66,9 +66,11 @@ var (
 
 	gitlabScan = cli.Command("gitlab", "Find credentials in GitLab repositories.")
 	// TODO: Add more GitLab options
-	gitlabScanEndpoint = gitlabScan.Flag("endpoint", "GitLab endpoint.").Default("https://gitlab.com").String()
-	gitlabScanRepos    = gitlabScan.Flag("repo", "GitLab repo url. You can repeat this flag. Leave empty to scan all repos accessible with provided credential. Example: https://gitlab.com/org/repo.git").Strings()
-	gitlabScanToken    = gitlabScan.Flag("token", "GitLab token. Can be provided with environment variable GITLAB_TOKEN.").Envar("GITLAB_TOKEN").Required().String()
+	gitlabScanEndpoint     = gitlabScan.Flag("endpoint", "GitLab endpoint.").Default("https://gitlab.com").String()
+	gitlabScanRepos        = gitlabScan.Flag("repo", "GitLab repo url. You can repeat this flag. Leave empty to scan all repos accessible with provided credential. Example: https://gitlab.com/org/repo.git").Strings()
+	gitlabScanToken        = gitlabScan.Flag("token", "GitLab token. Can be provided with environment variable GITLAB_TOKEN.").Envar("GITLAB_TOKEN").Required().String()
+	gitlabScanIncludePaths = gitlabScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
+	gitlabScanExcludePaths = gitlabScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
 
 	filesystemScan        = cli.Command("filesystem", "Find credentials in a filesystem.")
 	filesystemDirectories = filesystemScan.Flag("directory", "Path to directory to scan. You can repeat this flag.").Required().Strings()
@@ -223,10 +225,16 @@ func run(state overseer.State) {
 			logrus.WithError(err).Fatal("Failed to scan Github.")
 		}
 	case gitlabScan.FullCommand():
+		filter, err := common.FilterFromFiles(*gitlabScanIncludePaths, *gitlabScanExcludePaths)
+		if err != nil {
+			logrus.WithError(err).Fatal("could not create filter")
+		}
+
 		gitlab := func(c *sources.Config) {
 			c.Endpoint = *gitlabScanEndpoint
 			c.Token = *gitlabScanToken
 			c.Repos = *gitlabScanRepos
+			c.Filter = filter
 		}
 
 		if err = e.ScanGitLab(ctx, sources.NewConfig(gitlab)); err != nil {

diff --git a/pkg/engine/gitlab.go b/pkg/engine/gitlab.go
@@ -5,6 +5,7 @@ import (
 	"runtime"
 
 	"github.com/go-errors/errors"
+	gogit "github.com/go-git/go-git/v5"
 	"github.com/sirupsen/logrus"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
@@ -13,11 +14,19 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/gitlab"
 )
 
 // ScanGitLab scans GitLab with the provided configuration.
 func (e *Engine) ScanGitLab(ctx context.Context, c sources.Config) error {
+	logOptions := &gogit.LogOptions{}
+	opts := []git.ScanOption{
+		git.ScanOptionFilter(c.Filter),
+		git.ScanOptionLogOptions(logOptions),
+	}
+	scanOptions := git.NewScanOptions(opts...)
+
 	connection := &sourcespb.GitLab{}
 
 	switch {
@@ -49,6 +58,7 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.Config) error {
 	if err != nil {
 		return errors.WrapPrefix(err, "could not init GitLab source", 0)
 	}
+	gitlabSource.WithScanOptions(scanOptions)
 
 	e.sourcesWg.Add(1)
 	go func() {

diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.go
@@ -41,6 +41,7 @@ type Source struct {
 	repos           []string
 	ignoreRepos     []string
 	git             *git.Git
+	scanOptions     *git.ScanOptions
 	aCtx            context.Context
 	resumeInfoSlice []string
 	resumeInfoMutex sync.Mutex
@@ -325,7 +326,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
 				return
 			}
 			log.Debugf("Starting to scan repo %d/%d: %s", i+1, len(s.repos), repoURL)
-			err = s.git.ScanRepo(ctx, repo, path, git.NewScanOptions(), chunksChan)
+			err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan)
 			if err != nil {
 				errsMut.Lock()
 				errs = append(errs, err)
@@ -432,3 +433,7 @@ func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL stri
 	// Add the offset to both the index and the repos to give the proper place and proper repo count.
 	s.SetProgressComplete(index+offset, len(s.repos)+offset, fmt.Sprintf("Repo: %s", repoURL), encodedResumeInfo)
 }
+
+func (s *Source) WithScanOptions(scanOptions *git.ScanOptions) {
+	s.scanOptions = scanOptions
+}