From fc404e46e4e40c95d08780a28be7382342ede646 Mon Sep 17 00:00:00 2001 From: Kevin Stilwell Date: Thu, 27 Oct 2022 11:52:49 -0400 Subject: [PATCH 1/5] Fixes/work based on testing --- pkg/detectors/shopify/shopify.go | 84 ++++++++++++++++++ pkg/detectors/shopify/shopify_test.go | 119 ++++++++++++++++++++++++++ pkg/pb/detectorspb/detectors.pb.go | 16 ++-- proto/detectors.proto | 1 + 4 files changed, 214 insertions(+), 6 deletions(-) create mode 100644 pkg/detectors/shopify/shopify.go create mode 100644 pkg/detectors/shopify/shopify_test.go diff --git a/pkg/detectors/shopify/shopify.go b/pkg/detectors/shopify/shopify.go new file mode 100644 index 000000000000..bf2bcc7059de --- /dev/null +++ b/pkg/detectors/shopify/shopify.go @@ -0,0 +1,84 @@ +package shopify + +import ( + "context" + "fmt" + "net/http" + "regexp" + "strings" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +type Scanner struct{} + +// Ensure the Scanner satisfies the interface at compile time +var _ detectors.Detector = (*Scanner)(nil) + +var ( + client = common.SaneHttpClient() + + //Make sure that your group is surrounded in boundry characters such as below to reduce false positives + keyPat = regexp.MustCompile(`\b(shppa_|shpat_)([0-9A-Fa-f]{32})\b`) + domainPat = regexp.MustCompile((`[a-zA-Z0-9-]+\.myshopify\.com`)) +) + +// Keywords are used for efficiently pre-filtering chunks. +// Use identifiers in the secret preferably, or the provider name. +func (s Scanner) Keywords() []string { + return []string{"shppa_", "shpat_", "X-Shopify-Access-Token"} +} + +// FromData will find and optionally verify Shopify secrets in a given set of bytes. +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1) + domainMatches := domainPat.FindAllStringSubmatch(dataStr, -1) + + for _, match := range keyMatches { + // if len(match) != 3 { + // continue + // } + resMatch := strings.TrimSpace(match[0]) + + for _, domainMatch := range domainMatches { + // if len(domainMatch) != 2 { + // continue + // } + domainRes := strings.TrimSpace(domainMatch[0]) + + s1 := detectors.Result{ + DetectorType: detectorspb.DetectorType_Shopify, + Redacted: domainRes, + Raw: []byte(resMatch), + } + + if verify { + req, err := http.NewRequestWithContext(ctx, "GET", "https://"+domainRes+"/admin/api/2022-10/shop.json", nil) + if err != nil { + continue + } + req.Header.Add("X-Shopify-Access-Token", fmt.Sprintf("%s", resMatch)) + res, err := client.Do(req) + if err == nil { + defer res.Body.Close() + if res.StatusCode >= 200 && res.StatusCode < 300 { + s1.Verified = true + } else { + //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key + if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { + continue + } + } + } + } + + results = append(results, s1) + } + } + + return detectors.CleanResults(results), nil +} diff --git a/pkg/detectors/shopify/shopify_test.go b/pkg/detectors/shopify/shopify_test.go new file mode 100644 index 000000000000..0f82e5327186 --- /dev/null +++ b/pkg/detectors/shopify/shopify_test.go @@ -0,0 +1,119 @@ +package shopify + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestShopify_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("SHOPIFY") + inactiveSecret := testSecrets.MustGetField("SHOPIFY_INACTIVE") + domain := testSecrets.MustGetField("SHOPIFY_DOMAIN") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a shopify secret %s domain https://%s", secret, domain)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Shopify, + Redacted: domain, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a shopify secret %s within (domain https://%s) but not valid", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Shopify, + Redacted: domain, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Shopify.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Shopify.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/pb/detectorspb/detectors.pb.go b/pkg/pb/detectorspb/detectors.pb.go index 8776b96c3df1..a26a22e944f7 100644 --- a/pkg/pb/detectorspb/detectors.pb.go +++ b/pkg/pb/detectorspb/detectors.pb.go @@ -966,6 +966,7 @@ const ( DetectorType_MongoDB DetectorType = 895 DetectorType_NGC DetectorType = 896 DetectorType_DigitalOceanV2 DetectorType = 897 + DetectorType_Shopify DetectorType = 898 ) // Enum value maps for DetectorType. @@ -1865,6 +1866,7 @@ var ( 895: "MongoDB", 896: "NGC", 897: "DigitalOceanV2", + 898: "Shopify", } DetectorType_value = map[string]int32{ "Alibaba": 0, @@ -2761,6 +2763,7 @@ var ( "MongoDB": 895, "NGC": 896, "DigitalOceanV2": 897, + "Shopify": 898, } ) @@ -3125,7 +3128,7 @@ var file_detectors_proto_rawDesc = []byte{ 0x0a, 0x0b, 0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49, 0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, - 0x02, 0x2a, 0xb1, 0x70, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, + 0x02, 0x2a, 0xbf, 0x70, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x10, 0x03, 0x12, 0x0a, 0x0a, @@ -4024,11 +4027,12 @@ var file_detectors_proto_rawDesc = []byte{ 0x72, 0x6b, 0x64, 0x61, 0x79, 0x10, 0xfe, 0x06, 0x12, 0x0c, 0x0a, 0x07, 0x4d, 0x6f, 0x6e, 0x67, 0x6f, 0x44, 0x42, 0x10, 0xff, 0x06, 0x12, 0x08, 0x0a, 0x03, 0x4e, 0x47, 0x43, 0x10, 0x80, 0x07, 0x12, 0x13, 0x0a, 0x0e, 0x44, 0x69, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x4f, 0x63, 0x65, 0x61, 0x6e, - 0x56, 0x32, 0x10, 0x81, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, - 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, - 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, - 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, - 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x56, 0x32, 0x10, 0x81, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x53, 0x68, 0x6f, 0x70, 0x69, 0x66, 0x79, + 0x10, 0x82, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, + 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, + 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, + 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/proto/detectors.proto b/proto/detectors.proto index 759cf0c0097d..e9d4d4056185 100644 --- a/proto/detectors.proto +++ b/proto/detectors.proto @@ -905,6 +905,7 @@ enum DetectorType { MongoDB = 895; NGC = 896; DigitalOceanV2 = 897; + Shopify = 898; } message Result { From 11be3dc8ae3e571d159ae3615a5cb56f8a47237f Mon Sep 17 00:00:00 2001 From: Kevin Stilwell Date: Thu, 27 Oct 2022 12:11:15 -0400 Subject: [PATCH 2/5] Remove some commented code --- pkg/detectors/shopify/shopify.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/detectors/shopify/shopify.go b/pkg/detectors/shopify/shopify.go index bf2bcc7059de..f1fedcf98094 100644 --- a/pkg/detectors/shopify/shopify.go +++ b/pkg/detectors/shopify/shopify.go @@ -39,15 +39,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result domainMatches := domainPat.FindAllStringSubmatch(dataStr, -1) for _, match := range keyMatches { - // if len(match) != 3 { - // continue - // } resMatch := strings.TrimSpace(match[0]) for _, domainMatch := range domainMatches { - // if len(domainMatch) != 2 { - // continue - // } domainRes := strings.TrimSpace(domainMatch[0]) s1 := detectors.Result{ From 28c43d70c8aefc06b8f431b3b90c334848959cec Mon Sep 17 00:00:00 2001 From: Kevin Stilwell Date: Wed, 2 Nov 2022 10:56:52 -0400 Subject: [PATCH 3/5] Change how verification happens and grab additional information --- pkg/detectors/shopify/shopify.go | 36 +++++++++++++++++++++------ pkg/detectors/shopify/shopify_test.go | 12 ++++++--- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/pkg/detectors/shopify/shopify.go b/pkg/detectors/shopify/shopify.go index f1fedcf98094..40fa9f678dbf 100644 --- a/pkg/detectors/shopify/shopify.go +++ b/pkg/detectors/shopify/shopify.go @@ -2,6 +2,7 @@ package shopify import ( "context" + "encoding/json" "fmt" "net/http" "regexp" @@ -51,28 +52,49 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://"+domainRes+"/admin/api/2022-10/shop.json", nil) + req, err := http.NewRequestWithContext(ctx, "GET", "https://"+domainRes+"/admin/oauth/access_scopes.json", nil) if err != nil { continue } req.Header.Add("X-Shopify-Access-Token", fmt.Sprintf("%s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } else { - //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key - if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { - continue + shopifyTokenAccessScopes := shopifyTokenAccessScopes{} + err := json.NewDecoder(res.Body).Decode(&shopifyTokenAccessScopes) + if err == nil { + handle_array := []string{} + for _, handle := range shopifyTokenAccessScopes.AccessScopes { + handle_array = append(handle_array, handle.Handle) + + } + s1.Verified = true + s1.ExtraData = map[string]string{ + "handles": strings.Join(handle_array, ","), + } } + res.Body.Close() + } + } else { + //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key + if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { + continue } } } results = append(results, s1) + } + } return detectors.CleanResults(results), nil + +} + +type shopifyTokenAccessScopes struct { + AccessScopes []struct { + Handle string `json:"handle"` + } `json:"access_scopes"` } diff --git a/pkg/detectors/shopify/shopify_test.go b/pkg/detectors/shopify/shopify_test.go index 0f82e5327186..a4f0d7439fd9 100644 --- a/pkg/detectors/shopify/shopify_test.go +++ b/pkg/detectors/shopify/shopify_test.go @@ -41,14 +41,17 @@ func TestShopify_FromChunk(t *testing.T) { s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a shopify secret %s domain https://%s", secret, domain)), + data: []byte(fmt.Sprintf("You can find a shopify secret %s domain https://%s", secret, domain)), verify: true, }, want: []detectors.Result{ { DetectorType: detectorspb.DetectorType_Shopify, - Redacted: domain, + Redacted: domain, Verified: true, + ExtraData: map[string]string{ + "handles": "read_analytics,write_assigned_fulfillment_orders,read_assigned_fulfillment_orders", + }, }, }, wantErr: false, @@ -58,14 +61,15 @@ func TestShopify_FromChunk(t *testing.T) { s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a shopify secret %s within (domain https://%s) but not valid", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation + data: []byte(fmt.Sprintf("You can find a shopify secret %s within (domain https://%s) but not valid", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation verify: true, }, want: []detectors.Result{ { DetectorType: detectorspb.DetectorType_Shopify, - Redacted: domain, + Redacted: domain, Verified: false, + ExtraData: nil, }, }, wantErr: false, From 8d2c2fcbf6dc120fcc1a83f90dffede7052ce0cf Mon Sep 17 00:00:00 2001 From: Ahrav Dutta Date: Tue, 8 Nov 2022 16:18:29 -0800 Subject: [PATCH 4/5] Address linter warnings. --- pkg/detectors/shopify/shopify.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/pkg/detectors/shopify/shopify.go b/pkg/detectors/shopify/shopify.go index c6687a5b47f2..0f4ce05e13ea 100644 --- a/pkg/detectors/shopify/shopify.go +++ b/pkg/detectors/shopify/shopify.go @@ -3,7 +3,6 @@ package shopify import ( "context" "encoding/json" - "fmt" "net/http" "regexp" "strings" @@ -21,9 +20,9 @@ var _ detectors.Detector = (*Scanner)(nil) var ( client = common.SaneHttpClient() - //Make sure that your group is surrounded in boundry characters such as below to reduce false positives + // Make sure that your group is surrounded in boundry characters such as below to reduce false positives. keyPat = regexp.MustCompile(`\b(shppa_|shpat_)([0-9A-Fa-f]{32})\b`) - domainPat = regexp.MustCompile((`[a-zA-Z0-9-]+\.myshopify\.com`)) + domainPat = regexp.MustCompile(`[a-zA-Z0-9-]+\.myshopify\.com`) ) // Keywords are used for efficiently pre-filtering chunks. @@ -56,27 +55,27 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err != nil { continue } - req.Header.Add("X-Shopify-Access-Token", fmt.Sprintf("%s", key)) + req.Header.Add("X-Shopify-Access-Token", key) res, err := client.Do(req) if err == nil { if res.StatusCode >= 200 && res.StatusCode < 300 { shopifyTokenAccessScopes := shopifyTokenAccessScopes{} err := json.NewDecoder(res.Body).Decode(&shopifyTokenAccessScopes) if err == nil { - handle_array := []string{} + var handleArray []string for _, handle := range shopifyTokenAccessScopes.AccessScopes { - handle_array = append(handle_array, handle.Handle) + handleArray = append(handleArray, handle.Handle) } s1.Verified = true s1.ExtraData = map[string]string{ - "access_scopes": strings.Join(handle_array, ","), + "access_scopes": strings.Join(handleArray, ","), } } res.Body.Close() } } else { - //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key + // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. if detectors.IsKnownFalsePositive(key, detectors.DefaultFalsePositives, true) { continue } From 8fcfeb4e7ce3b09319e55404b06a71bb4a1a7dc1 Mon Sep 17 00:00:00 2001 From: Ahrav Dutta Date: Tue, 8 Nov 2022 16:20:51 -0800 Subject: [PATCH 5/5] add shopify detector to default detectors. --- pkg/engine/defaults.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/engine/defaults.go b/pkg/engine/defaults.go index 1ea37ba4c88f..a6b252ef3126 100644 --- a/pkg/engine/defaults.go +++ b/pkg/engine/defaults.go @@ -564,6 +564,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sherpadesk" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/shipday" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/shodankey" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/shopify" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/shortcut" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/shotstack" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/shutterstock" @@ -1488,5 +1489,6 @@ func DefaultDetectors() []detectors.Detector { sqlserver.Scanner{}, redis.Scanner{}, ftp.Scanner{}, + shopify.Scanner{}, } }