split up detectors

pkg/detectors/ftp/ftp.go

@@ -0,0 +1,103 @@
+package uri
+
+import (
+	"context"
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/jlaffaye/ftp"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct {
+	allowKnownTestSites bool
+}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	keyPat = regexp.MustCompile(`\bftp:\/\/[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`)
+
+	client = common.SaneHttpClient()
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"ftp://"}
+}
+
+// FromData will find and optionally verify URI secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+	for _, match := range matches {
+		urlMatch := match[0]
+		password := match[1]
+
+		// Skip findings where the password only has "*" characters, this is a redacted password
+		if strings.Trim(password, "*") == "" {
+			continue
+		}
+
+		parsedURL, err := url.Parse(urlMatch)
+		if err != nil {
+			continue
+		}
+		if _, ok := parsedURL.User.Password(); !ok {
+			continue
+		}
+
+		redact := strings.TrimSpace(strings.Replace(urlMatch, password, strings.Repeat("*", len(password)), -1))
+
+		s := detectors.Result{
+			DetectorType: detectorspb.DetectorType_FTP,
+			Raw:          []byte(urlMatch),
+			Redacted:     redact,
+		}
+
+		if verify {
+			s.Verified = verifyFTP(ctx, parsedURL)
+		}
+
+		if !s.Verified {
+			// Skip unverified findings where the password starts with a `$` - it's almost certainly a variable.
+			if strings.HasPrefix(password, "$") {
+				continue
+			}
+		}
+
+		if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) {
+			continue
+		}
+
+		results = append(results, s)
+	}
+
+	return detectors.CleanResults(results), nil
+}
+
+func verifyFTP(ctx context.Context, u *url.URL) bool {
+	host := u.Host
+	if !strings.Contains(host, ":") {
+		host = host + ":21"
+	}
+
+	c, err := ftp.Dial(host, ftp.DialWithTimeout(5*time.Second))
+	if err != nil {
+		return false
+	}
+
+	password, _ := u.User.Password()
+	err = c.Login(u.User.Username(), password)
+
+	return err == nil
+}

pkg/detectors/ftp/ftp_test.go

@@ -0,0 +1,109 @@
+//go:build detectors
+// +build detectors
+
+package uri
+
+import (
+	"context"
+	"testing"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestURI_FromChunk(t *testing.T) {
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "bad scheme",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("file://user:pass@foo.com:123/wh/at/ever"),
+				verify: true,
+			},
+			wantErr: false,
+		},
+		{
+			name: "verified FTP",
+			s:    Scanner{},
+			args: args{
+				ctx: context.Background(),
+				// https://dlptest.com/ftp-test/
+				data:   []byte("ftp://dlpuser:rNrKYTX9g7z3RgJRmxWuGHbeu@ftp.dlptest.com"),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_FTP,
+					Verified:     true,
+					Redacted:     "ftp://dlpuser:*************************@ftp.dlptest.com",
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "unverified FTP",
+			s:    Scanner{},
+			args: args{
+				ctx: context.Background(),
+				// https://dlptest.com/ftp-test/
+				data:   []byte("ftp://dlpuser:invalid@ftp.dlptest.com"),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_FTP,
+					Verified:     false,
+					Redacted:     "ftp://dlpuser:*******@ftp.dlptest.com",
+				},
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{allowKnownTestSites: true}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("URI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			// if os.Getenv("FORCE_PASS_DIFF") == "true" {
+			// 	return
+			// }
+			for i := range got {
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/mongodb/mongodb.go

@@ -21,7 +21,7 @@ var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://[-.%\w{}]{1,50}:([-.%\S]{3,50})@[-.%\w\/:]+)\b`)
+	keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+)\b`)
 	// TODO: Add support for sharded cluster, replica set and Atlas Deployment.
 )
 

pkg/detectors/npmtokenv2/npmtokenv2_test.go

@@ -1,7 +1,7 @@
 //go:build detectors
 // +build detectors
 
-package npmtoken_new
+package npmtokenv2
 
 import (
 	"context"

pkg/detectors/redis/redis.go

@@ -0,0 +1,101 @@
+package uri
+
+import (
+	"context"
+	"net/url"
+	"regexp"
+	"strings"
+
+	"github.com/go-redis/redis"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct {
+	allowKnownTestSites bool
+}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	keyPat = regexp.MustCompile(`\bredis:\/\/[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`)
+
+	client = common.SaneHttpClient()
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"redis"}
+}
+
+// FromData will find and optionally verify URI secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+	for _, match := range matches {
+		urlMatch := match[0]
+		password := match[1]
+
+		// Skip findings where the password only has "*" characters, this is a redacted password
+		if strings.Trim(password, "*") == "" {
+			continue
+		}
+
+		parsedURL, err := url.Parse(urlMatch)
+		if err != nil {
+			continue
+		}
+		if _, ok := parsedURL.User.Password(); !ok {
+			continue
+		}
+
+		redact := strings.TrimSpace(strings.Replace(urlMatch, password, strings.Repeat("*", len(password)), -1))
+
+		s := detectors.Result{
+			DetectorType: detectorspb.DetectorType_Redis,
+			Raw:          []byte(urlMatch),
+			Redacted:     redact,
+		}
+
+		if verify {
+			s.Verified = verifyRedis(ctx, parsedURL)
+		}
+
+		if !s.Verified {
+			// Skip unverified findings where the password starts with a `$` - it's almost certainly a variable.
+			if strings.HasPrefix(password, "$") {
+				continue
+			}
+		}
+
+		if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) {
+			continue
+		}
+
+		results = append(results, s)
+	}
+
+	return detectors.CleanResults(results), nil
+}
+
+func verifyRedis(ctx context.Context, u *url.URL) bool {
+	opt, err := redis.ParseURL(u.String())
+	if err != nil {
+		return false
+	}
+
+	client := redis.NewClient(opt)
+
+	status, err := client.Ping().Result()
+	if err == nil && status == "PONG" {
+		return true
+	}
+
+	return false
+}