Do local URI verification, while attempting to defuse SSRF

go.mod

@@ -27,13 +27,15 @@ require (
 	github.com/go-git/go-git/v5 v5.4.2
 	github.com/go-logr/logr v1.2.3
 	github.com/go-logr/zapr v1.2.3
+	github.com/go-redis/redis v6.15.9+incompatible
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/google/go-cmp v0.5.9
 	github.com/google/go-github/v42 v42.0.0
 	github.com/gorilla/mux v1.8.0
 	github.com/h2non/filetype v1.1.3
 	github.com/hashicorp/go-retryablehttp v0.7.1
+	github.com/jlaffaye/ftp v0.1.0
 	github.com/joho/godotenv v1.4.0
 	github.com/jpillora/overseer v1.1.6
 	github.com/kylelemons/godebug v1.1.0
@@ -97,7 +99,9 @@ require (
 	github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
 	github.com/googleapis/gax-go/v2 v2.6.0 // indirect
 	github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
+	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
@@ -109,6 +113,8 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
 	github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
+	github.com/onsi/ginkgo v1.16.5 // indirect
+	github.com/onsi/gomega v1.23.0 // indirect
 	github.com/pierrec/lz4/v4 v4.1.14 // indirect
 	github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -124,9 +130,9 @@ require (
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
-	golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458 // indirect
-	golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec // indirect
-	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/net v0.1.0 // indirect
+	golang.org/x/sys v0.1.0 // indirect
+	golang.org/x/text v0.4.0 // indirect
 	golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect
 	golang.org/x/tools v0.1.12 // indirect
 	google.golang.org/api v0.99.0 // indirect

pkg/detectors/ftp/ftp.go

@@ -0,0 +1,103 @@
+package uri
+
+import (
+	"context"
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/jlaffaye/ftp"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct {
+	allowKnownTestSites bool
+}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	keyPat = regexp.MustCompile(`\bftp:\/\/[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`)
+
+	client = common.SaneHttpClient()
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"ftp://"}
+}
+
+// FromData will find and optionally verify URI secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+	for _, match := range matches {
+		urlMatch := match[0]
+		password := match[1]
+
+		// Skip findings where the password only has "*" characters, this is a redacted password
+		if strings.Trim(password, "*") == "" {
+			continue
+		}
+
+		parsedURL, err := url.Parse(urlMatch)
+		if err != nil {
+			continue
+		}
+		if _, ok := parsedURL.User.Password(); !ok {
+			continue
+		}
+
+		redact := strings.TrimSpace(strings.Replace(urlMatch, password, strings.Repeat("*", len(password)), -1))
+
+		s := detectors.Result{
+			DetectorType: detectorspb.DetectorType_FTP,
+			Raw:          []byte(urlMatch),
+			Redacted:     redact,
+		}
+
+		if verify {
+			s.Verified = verifyFTP(ctx, parsedURL)
+		}
+
+		if !s.Verified {
+			// Skip unverified findings where the password starts with a `$` - it's almost certainly a variable.
+			if strings.HasPrefix(password, "$") {
+				continue
+			}
+		}
+
+		if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) {
+			continue
+		}
+
+		results = append(results, s)
+	}
+
+	return results, nil
+}
+
+func verifyFTP(ctx context.Context, u *url.URL) bool {
+	host := u.Host
+	if !strings.Contains(host, ":") {
+		host = host + ":21"
+	}
+
+	c, err := ftp.Dial(host, ftp.DialWithTimeout(5*time.Second))
+	if err != nil {
+		return false
+	}
+
+	password, _ := u.User.Password()
+	err = c.Login(u.User.Username(), password)
+
+	return err == nil
+}

pkg/detectors/ftp/ftp_test.go

@@ -0,0 +1,109 @@
+//go:build detectors
+// +build detectors
+
+package uri
+
+import (
+	"context"
+	"testing"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestURI_FromChunk(t *testing.T) {
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "bad scheme",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("file://user:pass@foo.com:123/wh/at/ever"),
+				verify: true,
+			},
+			wantErr: false,
+		},
+		{
+			name: "verified FTP",
+			s:    Scanner{},
+			args: args{
+				ctx: context.Background(),
+				// https://dlptest.com/ftp-test/
+				data:   []byte("ftp://dlpuser:rNrKYTX9g7z3RgJRmxWuGHbeu@ftp.dlptest.com"),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_FTP,
+					Verified:     true,
+					Redacted:     "ftp://dlpuser:*************************@ftp.dlptest.com",
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "unverified FTP",
+			s:    Scanner{},
+			args: args{
+				ctx: context.Background(),
+				// https://dlptest.com/ftp-test/
+				data:   []byte("ftp://dlpuser:invalid@ftp.dlptest.com"),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_FTP,
+					Verified:     false,
+					Redacted:     "ftp://dlpuser:*******@ftp.dlptest.com",
+				},
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{allowKnownTestSites: true}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("URI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			// if os.Getenv("FORCE_PASS_DIFF") == "true" {
+			// 	return
+			// }
+			for i := range got {
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/mongodb/mongodb.go

@@ -21,7 +21,7 @@ var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://(?:[^:]+:(?:[^@]+)?@)?(?:[^/]+|/.+.sock?,?)+(?:/([^/."*<>:|?]*))?\?(?:(.+=?=\S+)&?)+)\b`)
+	keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+)\b`)
 	// TODO: Add support for sharded cluster, replica set and Atlas Deployment.
 )
 

pkg/detectors/npmtokenv2/npmtokenv2_test.go

@@ -1,7 +1,7 @@
 //go:build detectors
 // +build detectors
 
-package npmtoken_new
+package npmtokenv2
 
 import (
 	"context"

pkg/detectors/redis/redis.go

@@ -0,0 +1,101 @@
+package uri
+
+import (
+	"context"
+	"net/url"
+	"regexp"
+	"strings"
+
+	"github.com/go-redis/redis"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct {
+	allowKnownTestSites bool
+}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	keyPat = regexp.MustCompile(`\bredis:\/\/[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`)
+
+	client = common.SaneHttpClient()
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"redis"}
+}
+
+// FromData will find and optionally verify URI secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+	for _, match := range matches {
+		urlMatch := match[0]
+		password := match[1]
+
+		// Skip findings where the password only has "*" characters, this is a redacted password
+		if strings.Trim(password, "*") == "" {
+			continue
+		}
+
+		parsedURL, err := url.Parse(urlMatch)
+		if err != nil {
+			continue
+		}
+		if _, ok := parsedURL.User.Password(); !ok {
+			continue
+		}
+
+		redact := strings.TrimSpace(strings.Replace(urlMatch, password, strings.Repeat("*", len(password)), -1))
+
+		s := detectors.Result{
+			DetectorType: detectorspb.DetectorType_Redis,
+			Raw:          []byte(urlMatch),
+			Redacted:     redact,
+		}
+
+		if verify {
+			s.Verified = verifyRedis(ctx, parsedURL)
+		}
+
+		if !s.Verified {
+			// Skip unverified findings where the password starts with a `$` - it's almost certainly a variable.
+			if strings.HasPrefix(password, "$") {
+				continue
+			}
+		}
+
+		if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) {
+			continue
+		}
+
+		results = append(results, s)
+	}
+
+	return results, nil
+}
+
+func verifyRedis(ctx context.Context, u *url.URL) bool {
+	opt, err := redis.ParseURL(u.String())
+	if err != nil {
+		return false
+	}
+
+	client := redis.NewClient(opt)
+
+	status, err := client.Ping().Result()
+	if err == nil && status == "PONG" {
+		return true
+	}
+
+	return false
+}