From e7a2bf12dbb592ce7602453ddeb580d4345adac0 Mon Sep 17 00:00:00 2001 From: Ahrav Dutta Date: Thu, 3 Nov 2022 13:48:51 -0700 Subject: [PATCH 1/6] Add context to all git methods. --- main.go | 4 +- pkg/engine/git_test.go | 6 ++- pkg/output/legacy_json.go | 6 ++- pkg/sources/git/git.go | 46 +++++++++---------- pkg/sources/git/git_test.go | 10 ++-- pkg/sources/github/github.go | 6 +-- pkg/sources/github/github_integration_test.go | 4 +- pkg/sources/gitlab/gitlab.go | 4 +- 8 files changed, 46 insertions(+), 40 deletions(-) diff --git a/main.go b/main.go index 2278084e8c8b..c80c6bfdd20a 100644 --- a/main.go +++ b/main.go @@ -185,7 +185,7 @@ func run(state overseer.State) { var remote bool switch cmd { case gitScan.FullCommand(): - repoPath, remote, err = git.PrepareRepoSinceCommit(*gitScanURI, *gitScanSinceCommit) + repoPath, remote, err = git.PrepareRepoSinceCommit(ctx, *gitScanURI, *gitScanSinceCommit) if err != nil || repoPath == "" { logrus.WithError(err).Fatal("error preparing git repo for scanning") } @@ -282,7 +282,7 @@ func run(state overseer.State) { switch { case *jsonLegacy: - output.PrintLegacyJSON(&r) + output.PrintLegacyJSON(ctx, &r) case *jsonOut: output.PrintJSON(&r) default: diff --git a/pkg/engine/git_test.go b/pkg/engine/git_test.go index 9803066c01f0..3acabf4ca337 100644 --- a/pkg/engine/git_test.go +++ b/pkg/engine/git_test.go @@ -18,8 +18,9 @@ type expResult struct { } func TestGitEngine(t *testing.T) { + ctx := context.Background() repoUrl := "https://github.com/dustin-decker/secretsandstuff.git" - path, _, err := git.PrepareRepo(repoUrl) + path, _, err := git.PrepareRepo(ctx, repoUrl) if err != nil { t.Error(err) } @@ -89,8 +90,9 @@ func TestGitEngine(t *testing.T) { } func BenchmarkGitEngine(b *testing.B) { + ctx := context.Background() repoUrl := "https://github.com/dustin-decker/secretsandstuff.git" - path, _, err := git.PrepareRepo(repoUrl) + path, _, err := git.PrepareRepo(ctx, repoUrl) if err != nil { b.Error(err) } diff --git a/pkg/output/legacy_json.go b/pkg/output/legacy_json.go index c280d7569c8b..6445a930c99d 100644 --- a/pkg/output/legacy_json.go +++ b/pkg/output/legacy_json.go @@ -13,12 +13,14 @@ import ( "github.com/go-git/go-git/v5/plumbing/object" "github.com/sergi/go-diff/diffmatchpatch" "github.com/sirupsen/logrus" + + "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git" ) -func PrintLegacyJSON(r *detectors.ResultWithMetadata) { +func PrintLegacyJSON(ctx context.Context, r *detectors.ResultWithMetadata) { var repo string switch r.SourceType { case sourcespb.SourceType_SOURCE_TYPE_GIT: @@ -32,7 +34,7 @@ func PrintLegacyJSON(r *detectors.ResultWithMetadata) { } // cloning the repo again here is not great and only works with unauthed repos - repoPath, remote, err := git.PrepareRepo(repo) + repoPath, remote, err := git.PrepareRepo(ctx, repo) if err != nil || repoPath == "" { logrus.WithError(err).Fatal("error preparing git repo for scanning") } diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.go index 725257c1895f..4c202a6887f8 100644 --- a/pkg/sources/git/git.go +++ b/pkg/sources/git/git.go @@ -140,7 +140,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err continue } err := func(repoURI string) error { - path, repo, err := CloneRepoUsingToken(token, repoURI, user) + path, repo, err := CloneRepoUsingToken(ctx, token, repoURI, user) defer os.RemoveAll(path) if err != nil { return err @@ -158,7 +158,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err continue } err := func(repoURI string) error { - path, repo, err := CloneRepoUsingUnauthenticated(repoURI) + path, repo, err := CloneRepoUsingUnauthenticated(ctx, repoURI) defer os.RemoveAll(path) if err != nil { return err @@ -176,7 +176,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err continue } err := func(repoURI string) error { - path, repo, err := CloneRepoUsingSSH(repoURI) + path, repo, err := CloneRepoUsingSSH(ctx, repoURI) defer os.RemoveAll(path) if err != nil { return err @@ -246,7 +246,7 @@ func gitURLParse(gitURL string) (*url.URL, error) { return parsedURL, nil } -func CloneRepo(userInfo *url.Userinfo, gitUrl string, args ...string) (clonePath string, repo *git.Repository, err error) { +func CloneRepo(ctx context.Context, userInfo *url.Userinfo, gitUrl string, args ...string) (clonePath string, repo *git.Repository, err error) { if err = GitCmdCheck(); err != nil { return } @@ -281,7 +281,7 @@ func CloneRepo(userInfo *url.Userinfo, gitUrl string, args ...string) (clonePath if err != nil { log.WithError(err).Errorf("failed to strip credentials from git url") } - log.WithField("exit_code", cloneCmd.ProcessState.ExitCode()).WithField("repo", safeUrl).WithField("output", string(output)).Errorf("failed to clone repo") + ctx.Logger().V(1).Info("git clone failed", "repo", safeUrl, "output", string(output)) return "", nil, fmt.Errorf("could not clone repo: %s", safeUrl) } repo, err = git.PlainOpen(clonePath) @@ -294,20 +294,20 @@ func CloneRepo(userInfo *url.Userinfo, gitUrl string, args ...string) (clonePath } // CloneRepoUsingToken clones a repo using a provided token. -func CloneRepoUsingToken(token, gitUrl, user string, args ...string) (string, *git.Repository, error) { +func CloneRepoUsingToken(ctx context.Context, token, gitUrl, user string, args ...string) (string, *git.Repository, error) { userInfo := url.UserPassword(user, token) - return CloneRepo(userInfo, gitUrl, args...) + return CloneRepo(ctx, userInfo, gitUrl, args...) } // CloneRepoUsingUnauthenticated clones a repo with no authentication required. -func CloneRepoUsingUnauthenticated(url string, args ...string) (string, *git.Repository, error) { - return CloneRepo(nil, url, args...) +func CloneRepoUsingUnauthenticated(ctx context.Context, url string, args ...string) (string, *git.Repository, error) { + return CloneRepo(ctx, nil, url, args...) } // CloneRepoUsingUnauthenticated clones a repo with no authentication required. -func CloneRepoUsingSSH(gitUrl string, args ...string) (string, *git.Repository, error) { +func CloneRepoUsingSSH(ctx context.Context, gitUrl string, args ...string) (string, *git.Repository, error) { userInfo := url.User("git") - return CloneRepo(userInfo, gitUrl, args...) + return CloneRepo(ctx, userInfo, gitUrl, args...) } func GitCmdCheck() error { @@ -656,9 +656,9 @@ func TryAdditionalBaseRefs(repo *git.Repository, base string) (*plumbing.Hash, e } // PrepareRepoSinceCommit clones a repo starting at the given commitHash and returns the cloned repo path. -func PrepareRepoSinceCommit(uriString, commitHash string) (string, bool, error) { +func PrepareRepoSinceCommit(ctx context.Context, uriString, commitHash string) (string, bool, error) { if commitHash == "" { - return PrepareRepo(uriString) + return PrepareRepo(ctx, uriString) } // TODO: refactor with PrepareRepo to remove duplicated logic @@ -673,13 +673,13 @@ func PrepareRepoSinceCommit(uriString, commitHash string) (string, bool, error) } if uri.Scheme == "file" || uri.Host != "github.com" { - return PrepareRepo(uriString) + return PrepareRepo(ctx, uriString) } uriPath := strings.TrimPrefix(uri.Path, "/") owner, repoName, found := strings.Cut(uriPath, "/") if !found { - return PrepareRepo(uriString) + return PrepareRepo(ctx, uriString) } client := github.NewClient(nil) @@ -693,13 +693,13 @@ func PrepareRepoSinceCommit(uriString, commitHash string) (string, bool, error) commit, _, err := client.Git.GetCommit(context.Background(), owner, repoName, commitHash) if err != nil { - return PrepareRepo(uriString) + return PrepareRepo(ctx, uriString) } var timestamp string { author := commit.GetAuthor() if author == nil { - return PrepareRepo(uriString) + return PrepareRepo(ctx, uriString) } timestamp = author.GetDate().Format(time.RFC3339) } @@ -713,13 +713,13 @@ func PrepareRepoSinceCommit(uriString, commitHash string) (string, bool, error) if !ok { return "", true, fmt.Errorf("password must be included in Git repo URL when username is provided") } - path, _, err = CloneRepoUsingToken(password, remotePath, uri.User.Username(), "--shallow-since", timestamp) + path, _, err = CloneRepoUsingToken(ctx, password, remotePath, uri.User.Username(), "--shallow-since", timestamp) if err != nil { return path, true, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", remotePath, err) } default: log.Debugf("Cloning remote Git repo without authentication") - path, _, err = CloneRepoUsingUnauthenticated(remotePath, "--shallow-since", timestamp) + path, _, err = CloneRepoUsingUnauthenticated(ctx, remotePath, "--shallow-since", timestamp) if err != nil { return path, true, fmt.Errorf("failed to clone unauthenticated Git repo (%s): %s", remotePath, err) } @@ -729,7 +729,7 @@ func PrepareRepoSinceCommit(uriString, commitHash string) (string, bool, error) } // PrepareRepo clones a repo if possible and returns the cloned repo path. -func PrepareRepo(uriString string) (string, bool, error) { +func PrepareRepo(ctx context.Context, uriString string) (string, bool, error) { var path string uri, err := gitURLParse(uriString) if err != nil { @@ -750,13 +750,13 @@ func PrepareRepo(uriString string) (string, bool, error) { if !ok { return "", remote, fmt.Errorf("password must be included in Git repo URL when username is provided") } - path, _, err = CloneRepoUsingToken(password, remotePath, uri.User.Username()) + path, _, err = CloneRepoUsingToken(ctx, password, remotePath, uri.User.Username()) if err != nil { return path, remote, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", remotePath, err) } default: log.Debugf("Cloning remote Git repo without authentication") - path, _, err = CloneRepoUsingUnauthenticated(remotePath) + path, _, err = CloneRepoUsingUnauthenticated(ctx, remotePath) if err != nil { return path, remote, fmt.Errorf("failed to clone unauthenticated Git repo (%s): %s", remotePath, err) } @@ -764,7 +764,7 @@ func PrepareRepo(uriString string) (string, bool, error) { case "ssh": remotePath := uri.String() remote = true - path, _, err = CloneRepoUsingSSH(remotePath) + path, _, err = CloneRepoUsingSSH(ctx, remotePath) if err != nil { return path, remote, fmt.Errorf("failed to clone unauthenticated Git repo (%s): %s", remotePath, err) } diff --git a/pkg/sources/git/git_test.go b/pkg/sources/git/git_test.go index 51172fe0fea1..e7cd00eac048 100644 --- a/pkg/sources/git/git_test.go +++ b/pkg/sources/git/git_test.go @@ -210,7 +210,7 @@ func TestSource_Chunks_Integration(t *testing.T) { tests := []struct { name string init init - //verified + // verified repoURL string expectedChunkData map[string]*byteCompare scanOptions ScanOptions @@ -285,7 +285,7 @@ func TestSource_Chunks_Integration(t *testing.T) { chunksCh := make(chan *sources.Chunk, 1) go func() { defer close(chunksCh) - repoPath, repo, err := CloneRepoUsingUnauthenticated(tt.repoURL) + repoPath, repo, err := CloneRepoUsingUnauthenticated(ctx, tt.repoURL) if err != nil { panic(err) } @@ -476,7 +476,8 @@ func TestPrepareRepo(t *testing.T) { } for _, tt := range tests { - repo, b, err := PrepareRepo(tt.uri) + ctx := context.Background() + repo, b, err := PrepareRepo(ctx, tt.uri) var repoLen bool if len(repo) > 0 { repoLen = true @@ -491,8 +492,9 @@ func TestPrepareRepo(t *testing.T) { func BenchmarkPrepareRepo(b *testing.B) { uri := "https://github.com/dustin-decker/secretsandstuff.git" + ctx := context.Background() for i := 0; i < b.N; i++ { - _, _, _ = PrepareRepo(uri) + _, _, _ = PrepareRepo(ctx, uri) } } diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 2980af1590a6..a954bcc2fa76 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -570,7 +570,7 @@ func (s *Source) cloneRepo(ctx context.Context, repoURL string, installationClie switch s.conn.GetCredential().(type) { case *sourcespb.GitHub_Unauthenticated: - path, repo, err = git.CloneRepoUsingUnauthenticated(repoURL) + path, repo, err = git.CloneRepoUsingUnauthenticated(ctx, repoURL) if err != nil { return "", nil, fmt.Errorf("error cloning repo %s: %w", repoURL, err) } @@ -581,7 +581,7 @@ func (s *Source) cloneRepo(ctx context.Context, repoURL string, installationClie return "", nil, fmt.Errorf("error getting token for repo %s: %w", repoURL, err) } - path, repo, err = git.CloneRepoUsingToken(s.githubToken, repoURL, s.githubUser) + path, repo, err = git.CloneRepoUsingToken(ctx, s.githubToken, repoURL, s.githubUser) if err != nil { return "", nil, fmt.Errorf("error cloning repo %s: %w", repoURL, err) } @@ -594,7 +594,7 @@ func (s *Source) cloneRepo(ctx context.Context, repoURL string, installationClie return "", nil, fmt.Errorf("error getting token for repo %s: %w", repoURL, err) } } - path, repo, err = git.CloneRepoUsingToken(s.githubToken, repoURL, s.githubUser) + path, repo, err = git.CloneRepoUsingToken(ctx, s.githubToken, repoURL, s.githubUser) if err != nil { return "", nil, fmt.Errorf("error cloning repo %s: %w", repoURL, err) } diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.go index cb4422e7a85c..70782c81affc 100644 --- a/pkg/sources/github/github_integration_test.go +++ b/pkg/sources/github/github_integration_test.go @@ -68,11 +68,11 @@ func TestSource_Token(t *testing.T) { assert.NoError(t, err) // user provided - _, _, err = git.CloneRepoUsingToken(token, "https://github.com/trufflesecurity/trufflehog-updater.git", user) + _, _, err = git.CloneRepoUsingToken(ctx, token, "https://github.com/trufflesecurity/trufflehog-updater.git", user) assert.NoError(t, err) // no user provided - _, _, err = git.CloneRepoUsingToken(token, "https://github.com/trufflesecurity/trufflehog-updater.git", "") + _, _, err = git.CloneRepoUsingToken(ctx, token, "https://github.com/trufflesecurity/trufflehog-updater.git", "") assert.Error(t, err) _, _, err = s.cloneRepo(ctx, "https://github.com/trufflesecurity/trufflehog-updater.git", installationClient) diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.go index 334a82ab5319..1004d9ba7d8f 100644 --- a/pkg/sources/gitlab/gitlab.go +++ b/pkg/sources/gitlab/gitlab.go @@ -307,7 +307,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) var repo *gogit.Repository var err error if s.authMethod == "UNAUTHENTICATED" { - path, repo, err = git.CloneRepoUsingUnauthenticated(repoURL) + path, repo, err = git.CloneRepoUsingUnauthenticated(ctx, repoURL) } else { // If a username is not provided we need to use a default one in order to clone a private repo. // Not setting "placeholder" as s.user on purpose in case any downstream services rely on a "" value for s.user. @@ -315,7 +315,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) if user == "" { user = "placeholder" } - path, repo, err = git.CloneRepoUsingToken(s.token, repoURL, user) + path, repo, err = git.CloneRepoUsingToken(ctx, s.token, repoURL, user) } defer os.RemoveAll(path) if err != nil { From 6bcd21425f6c51cb62a144d46064e4e4ecfa64b4 Mon Sep 17 00:00:00 2001 From: Ahrav Dutta Date: Thu, 3 Nov 2022 14:14:03 -0700 Subject: [PATCH 2/6] remove logrus. --- pkg/sources/git/git.go | 96 +++++++++++++++++++++--------------------- 1 file changed, 49 insertions(+), 47 deletions(-) diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.go index 4c202a6887f8..02775b3c3a7d 100644 --- a/pkg/sources/git/git.go +++ b/pkg/sources/git/git.go @@ -20,8 +20,6 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/object" "github.com/google/go-github/v42/github" - "github.com/rs/zerolog" - log "github.com/sirupsen/logrus" "golang.org/x/oauth2" "golang.org/x/sync/semaphore" "google.golang.org/protobuf/proto" @@ -217,6 +215,8 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err } } + + ctx.Logger().V(1).Info("Git source finished scanning", "repo-count", len(s.conn.Repositories)) s.SetProgressComplete(len(s.conn.Repositories), len(s.conn.Repositories), fmt.Sprintf("Completed scanning source %s", s.name), "") return nil } @@ -246,14 +246,13 @@ func gitURLParse(gitURL string) (*url.URL, error) { return parsedURL, nil } -func CloneRepo(ctx context.Context, userInfo *url.Userinfo, gitUrl string, args ...string) (clonePath string, repo *git.Repository, err error) { - if err = GitCmdCheck(); err != nil { - return +func CloneRepo(ctx context.Context, userInfo *url.Userinfo, gitUrl string, args ...string) (string, *git.Repository, error) { + if err := gitCmdCheck(); err != nil { + return "", nil, err } - clonePath, err = ioutil.TempDir(os.TempDir(), "trufflehog") + clonePath, err := ioutil.TempDir(os.TempDir(), "trufflehog") if err != nil { - err = errors.New(err) - return + return "", nil, err } defer CleanOnError(&err, clonePath) cloneURL, err := gitURLParse(gitUrl) @@ -279,18 +278,19 @@ func CloneRepo(ctx context.Context, userInfo *url.Userinfo, gitUrl string, args if cloneCmd.ProcessState != nil && cloneCmd.ProcessState.ExitCode() != 0 { safeUrl, err := stripPassword(gitUrl) if err != nil { - log.WithError(err).Errorf("failed to strip credentials from git url") + ctx.Logger().V(1).Info("error stripping password from git url", "error", err) } ctx.Logger().V(1).Info("git clone failed", "repo", safeUrl, "output", string(output)) return "", nil, fmt.Errorf("could not clone repo: %s", safeUrl) } - repo, err = git.PlainOpen(clonePath) + + repo, err := git.PlainOpen(clonePath) if err != nil { - err = errors.WrapPrefix(err, "could not open cloned repo", 0) - return + return "", nil, fmt.Errorf("could not open cloned repo: %w", err) } - log.WithField("clone_path", clonePath).WithField("repo", gitUrl).Debug("cloned repo") - return + + ctx.Logger().V(1).Info("cloned repo", "repo", gitUrl, "clone-path", clonePath) + return clonePath, repo, nil } // CloneRepoUsingToken clones a repo using a provided token. @@ -304,13 +304,14 @@ func CloneRepoUsingUnauthenticated(ctx context.Context, url string, args ...stri return CloneRepo(ctx, nil, url, args...) } -// CloneRepoUsingUnauthenticated clones a repo with no authentication required. +// CloneRepoUsingSSH clones a repo using SSH. func CloneRepoUsingSSH(ctx context.Context, gitUrl string, args ...string) (string, *git.Repository, error) { userInfo := url.User("git") return CloneRepo(ctx, userInfo, gitUrl, args...) } -func GitCmdCheck() error { +// gitCmdCheck checks if git is installed. +func gitCmdCheck() error { if errors.Is(exec.Command("git").Run(), exec.ErrNotFound) { return fmt.Errorf("'git' command not found in $PATH. Make sure git is installed and included in $PATH") } @@ -318,12 +319,9 @@ func GitCmdCheck() error { } func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error { - if err := GitCmdCheck(); err != nil { + if err := gitCmdCheck(); err != nil { return err } - if log.GetLevel() < log.DebugLevel { - zerolog.SetGlobalLevel(zerolog.Disabled) - } commitChan, err := gitparse.RepoPath(ctx, path, scanOptions.HeadHash) if err != nil { @@ -338,11 +336,12 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string var depth int64 var reachedBase = false - log.WithField("repo", urlMetadata).Debugf("Scanning repo") + + ctx.Logger().V(1).Info("scanning repo", "repo", urlMetadata, "base", scanOptions.BaseHash, "head", scanOptions.HeadHash) for commit := range commitChan { - log.Tracef("Scanning commit %s", commit.Hash) + ctx.Logger().V(2).Info("scanning commit", "commit", commit.Hash, "message", commit.Message) if scanOptions.MaxDepth > 0 && depth >= scanOptions.MaxDepth { - log.Debugf("reached max depth") + ctx.Logger().V(1).Info("reached max depth", "depth", depth) break } depth++ @@ -351,13 +350,11 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string } if len(scanOptions.BaseHash) > 0 { if commit.Hash == scanOptions.BaseHash { - log.Debugf("Reached base commit. Finishing scanning files.") + ctx.Logger().V(1).Info("reached base commit", "commit", commit.Hash) reachedBase = true } } for _, diff := range commit.Diffs { - log.WithField("commit", commit.Hash).WithField("file", diff.PathB).Trace("Scanning file from git") - if !scanOptions.Filter.Pass(diff.PathB) { continue } @@ -383,13 +380,13 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string Verify: s.verify, } if err := handleBinary(ctx, repo, chunksChan, chunkSkel, commitHash, fileName); err != nil { - log.WithError(err).WithField("file", fileName).Debug("Error handling binary file") + ctx.Logger().V(1).Info("error handling binary file", "error", err, "filename", fileName, "commit", commitHash, "file", diff.PathB) } continue } if diff.Content.Len() > sources.ChunkSize+sources.PeekSize { - s.gitChunk(diff, fileName, email, hash, when, urlMetadata, chunksChan) + s.gitChunk(ctx, diff, fileName, email, hash, when, urlMetadata, chunksChan) continue } metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, int64(diff.LineStart)) @@ -406,7 +403,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string return nil } -func (s *Git) gitChunk(diff gitparse.Diff, fileName, email, hash, when, urlMetadata string, chunksChan chan *sources.Chunk) { +func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email, hash, when, urlMetadata string, chunksChan chan *sources.Chunk) { originalChunk := bufio.NewScanner(&diff.Content) newChunkBuffer := bytes.Buffer{} lastOffset := 0 @@ -443,9 +440,8 @@ func (s *Git) gitChunk(diff gitparse.Diff, fileName, email, hash, when, urlMetad } } - _, err := newChunkBuffer.Write(line) - if err != nil { - log.WithError(err).Error("Could not write line to git diff buffer.") + if _, err := newChunkBuffer.Write(line); err != nil { + ctx.Logger().V(1).Info("error writing to chunk buffer", "error", err, "filename", fileName, "commit", hash, "file", diff.PathB) } } // Send anything still in the new chunk buffer @@ -477,12 +473,15 @@ func (s *Git) ScanUnstaged(ctx context.Context, repo *git.Repository, path strin var depth int64 var reachedBase = false - log.Debugf("Scanning repo") + + ctx.Logger().V(1).Info("scanning unstaged changes", "path", path) for commit := range commitChan { for _, diff := range commit.Diffs { - log.WithField("commit", commit.Hash).WithField("file", diff.PathB).Trace("Scanning file from git") + logger := ctx.Logger().WithValues("filename", diff.PathB, "commit", commit.Hash, "file", diff.PathB) + logger.V(2).Info("scanning unstaged changes from git") + if scanOptions.MaxDepth > 0 && depth >= scanOptions.MaxDepth { - log.Debugf("reached max depth") + logger.V(1).Info("reached max depth") break } depth++ @@ -491,7 +490,7 @@ func (s *Git) ScanUnstaged(ctx context.Context, repo *git.Repository, path strin } if len(scanOptions.BaseHash) > 0 { if commit.Hash == scanOptions.BaseHash { - log.Debugf("Reached base commit. Finishing scanning files.") + logger.V(1).Info("reached base hash, finishing scanning files") reachedBase = true } } @@ -521,7 +520,7 @@ func (s *Git) ScanUnstaged(ctx context.Context, repo *git.Repository, path strin Verify: s.verify, } if err := handleBinary(ctx, repo, chunksChan, chunkSkel, commitHash, fileName); err != nil { - log.WithError(err).WithField("file", fileName).Debug("Error handling binary file") + logger.V(1).Info("error handling binary file", "error", err, "filename", fileName) } continue } @@ -549,10 +548,11 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath strin return err } if err := s.ScanUnstaged(ctx, repo, repoPath, scanOptions, chunksChan); err != nil { - log.WithError(err).Error("Error scanning unstaged changes") + ctx.Logger().V(1).Info("error scanning unstaged changes", "error", err) } + scanTime := time.Now().UnixNano() - start - log.Debugf("Scanning complete. Scan time: %f", time.Duration(scanTime).Seconds()) + ctx.Logger().V(1).Info("scanning git repo complete", "path", repoPath, "time", scanTime) return nil } @@ -708,7 +708,7 @@ func PrepareRepoSinceCommit(ctx context.Context, uriString, commitHash string) ( var path string switch { case uri.User != nil: - log.Debugf("Cloning remote Git repo with authentication") + ctx.Logger().V(1).Info("cloning repo with authentication", "uri", uri) password, ok := uri.User.Password() if !ok { return "", true, fmt.Errorf("password must be included in Git repo URL when username is provided") @@ -718,13 +718,14 @@ func PrepareRepoSinceCommit(ctx context.Context, uriString, commitHash string) ( return path, true, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", remotePath, err) } default: - log.Debugf("Cloning remote Git repo without authentication") + ctx.Logger().V(1).Info("cloning repo without authentication", "uri", uri) path, _, err = CloneRepoUsingUnauthenticated(ctx, remotePath, "--shallow-since", timestamp) if err != nil { return path, true, fmt.Errorf("failed to clone unauthenticated Git repo (%s): %s", remotePath, err) } } - log.Debugf("Git repo local path: %s", path) + + ctx.Logger().V(1).Info("cloned repo", "path", path) return path, true, nil } @@ -745,7 +746,7 @@ func PrepareRepo(ctx context.Context, uriString string) (string, bool, error) { remote = true switch { case uri.User != nil: - log.Debugf("Cloning remote Git repo with authentication") + ctx.Logger().V(1).Info("cloning repo with authentication", "uri", uri) password, ok := uri.User.Password() if !ok { return "", remote, fmt.Errorf("password must be included in Git repo URL when username is provided") @@ -755,7 +756,7 @@ func PrepareRepo(ctx context.Context, uriString string) (string, bool, error) { return path, remote, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", remotePath, err) } default: - log.Debugf("Cloning remote Git repo without authentication") + ctx.Logger().V(1).Info("cloning repo without authentication", "uri", uri) path, _, err = CloneRepoUsingUnauthenticated(ctx, remotePath) if err != nil { return path, remote, fmt.Errorf("failed to clone unauthenticated Git repo (%s): %s", remotePath, err) @@ -771,7 +772,8 @@ func PrepareRepo(ctx context.Context, uriString string) (string, bool, error) { default: return "", remote, fmt.Errorf("unsupported Git URI: %s", uriString) } - log.Debugf("Git repo local path: %s", path) + + ctx.Logger().V(1).Info("cloned repo", "path", path) return path, remote, nil } @@ -799,7 +801,7 @@ func getSafeRemoteURL(repo *git.Repository, preferred string) string { } func handleBinary(ctx context.Context, repo *git.Repository, chunksChan chan *sources.Chunk, chunkSkel *sources.Chunk, commitHash plumbing.Hash, path string) error { - log.WithField("path", path).Trace("Binary file found in repository.") + ctx.Logger().V(1).Info("handling binary file", "path", path) commit, err := repo.CommitObject(commitHash) if err != nil { return err @@ -825,7 +827,7 @@ func handleBinary(ctx context.Context, repo *git.Repository, chunksChan chan *so return nil } - log.WithField("path", path).Trace("Binary file is not recognized by file handlers. Chunking raw.") + ctx.Logger().V(1).Info("binary file not handled, chunking raw", "path", path) if err := reader.Reset(); err != nil { return err } From 40a5e779893fd236a27fad9f552f6fe5a881aa54 Mon Sep 17 00:00:00 2001 From: Ahrav Dutta Date: Thu, 3 Nov 2022 14:21:20 -0700 Subject: [PATCH 3/6] Add ctx. --- hack/snifftest/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/snifftest/main.go b/hack/snifftest/main.go index 5f4082157b86..1222b29fbd5c 100644 --- a/hack/snifftest/main.go +++ b/hack/snifftest/main.go @@ -162,7 +162,7 @@ func main() { defer sem.Release(1) defer wgChunkers.Done() log.Infof("cloning %s", r) - path, repo, err := git.CloneRepoUsingUnauthenticated(r) + path, repo, err := git.CloneRepoUsingUnauthenticated(ctx, r) if err != nil { log.Fatal(err) } From 79c0f01ac01cb9db62ea937ad5587eea3e8d03d5 Mon Sep 17 00:00:00 2001 From: Ahrav Dutta Date: Thu, 3 Nov 2022 14:38:35 -0700 Subject: [PATCH 4/6] Address comments. --- pkg/sources/git/git.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.go index 02775b3c3a7d..a1c37f1b018e 100644 --- a/pkg/sources/git/git.go +++ b/pkg/sources/git/git.go @@ -339,7 +339,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string ctx.Logger().V(1).Info("scanning repo", "repo", urlMetadata, "base", scanOptions.BaseHash, "head", scanOptions.HeadHash) for commit := range commitChan { - ctx.Logger().V(2).Info("scanning commit", "commit", commit.Hash, "message", commit.Message) + ctx.Logger().V(5).Info("scanning commit", "commit", commit.Hash, "message", commit.Message) if scanOptions.MaxDepth > 0 && depth >= scanOptions.MaxDepth { ctx.Logger().V(1).Info("reached max depth", "depth", depth) break @@ -441,7 +441,7 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email, } if _, err := newChunkBuffer.Write(line); err != nil { - ctx.Logger().V(1).Info("error writing to chunk buffer", "error", err, "filename", fileName, "commit", hash, "file", diff.PathB) + ctx.Logger().Error(err, "error writing to chunk buffer", "filename", fileName, "commit", hash, "file", diff.PathB) } } // Send anything still in the new chunk buffer @@ -801,7 +801,7 @@ func getSafeRemoteURL(repo *git.Repository, preferred string) string { } func handleBinary(ctx context.Context, repo *git.Repository, chunksChan chan *sources.Chunk, chunkSkel *sources.Chunk, commitHash plumbing.Hash, path string) error { - ctx.Logger().V(1).Info("handling binary file", "path", path) + ctx.Logger().V(5).Info("handling binary file", "path", path) commit, err := repo.CommitObject(commitHash) if err != nil { return err From 7827c845800964fa01f72520957b2d44c302ddf3 Mon Sep 17 00:00:00 2001 From: Ahrav Dutta Date: Thu, 3 Nov 2022 15:51:40 -0700 Subject: [PATCH 5/6] Add error to clone failing. --- pkg/sources/git/git.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.go index a1c37f1b018e..5389b5423a32 100644 --- a/pkg/sources/git/git.go +++ b/pkg/sources/git/git.go @@ -280,7 +280,7 @@ func CloneRepo(ctx context.Context, userInfo *url.Userinfo, gitUrl string, args if err != nil { ctx.Logger().V(1).Info("error stripping password from git url", "error", err) } - ctx.Logger().V(1).Info("git clone failed", "repo", safeUrl, "output", string(output)) + ctx.Logger().V(1).Info("git clone failed", "error", err, "repo", safeUrl, "output", string(output)) return "", nil, fmt.Errorf("could not clone repo: %s", safeUrl) } From a8ce81f5890ae3424c8535a4679d65226ce8951a Mon Sep 17 00:00:00 2001 From: Ahrav Dutta Date: Thu, 3 Nov 2022 16:18:13 -0700 Subject: [PATCH 6/6] Return error. --- pkg/sources/git/git.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.go index 5389b5423a32..4646a04724b2 100644 --- a/pkg/sources/git/git.go +++ b/pkg/sources/git/git.go @@ -281,7 +281,7 @@ func CloneRepo(ctx context.Context, userInfo *url.Userinfo, gitUrl string, args ctx.Logger().V(1).Info("error stripping password from git url", "error", err) } ctx.Logger().V(1).Info("git clone failed", "error", err, "repo", safeUrl, "output", string(output)) - return "", nil, fmt.Errorf("could not clone repo: %s", safeUrl) + return "", nil, fmt.Errorf("could not clone repo: %s, %w", safeUrl, err) } repo, err := git.PlainOpen(clonePath)