From 09ccbe626094e6ab641ac204219994925b4ea7e2 Mon Sep 17 00:00:00 2001 From: Dustin Decker Date: Fri, 28 Oct 2022 10:23:53 -0700 Subject: [PATCH 1/7] simplify monogo pattern --- pkg/detectors/mongodb/mongodb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/detectors/mongodb/mongodb.go b/pkg/detectors/mongodb/mongodb.go index 3d65a4806fb4..2841f54c38b1 100644 --- a/pkg/detectors/mongodb/mongodb.go +++ b/pkg/detectors/mongodb/mongodb.go @@ -21,7 +21,7 @@ var _ detectors.Detector = (*Scanner)(nil) var ( // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://(?:[^:]+:(?:[^@]+)?@)?(?:[^/]+|/.+.sock?,?)+(?:/([^/."*<>:|?]*))?\?(?:(.+=?=\S+)&?)+)\b`) + keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://[-.%\w{}]{1,50}:([-.%\S]{3,50})@[-.%\w\/:]+)\b`) // TODO: Add support for sharded cluster, replica set and Atlas Deployment. ) From 75ab1f16af7091f4affdc9f5ae94738cba74123e Mon Sep 17 00:00:00 2001 From: Dustin Decker Date: Fri, 28 Oct 2022 15:56:44 -0700 Subject: [PATCH 2/7] do URI verification locally, while attempting to defuse SSRF --- go.mod | 12 +++- go.sum | 48 +++++++++++-- pkg/detectors/uri/uri.go | 131 ++++++++++++++++++++++++++-------- pkg/detectors/uri/uri_test.go | 57 ++++++++++----- 4 files changed, 193 insertions(+), 55 deletions(-) diff --git a/go.mod b/go.mod index cf05134192a7..93211f26c89e 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/go-git/go-git/v5 v5.4.2 github.com/go-logr/logr v1.2.3 github.com/go-logr/zapr v1.2.3 + github.com/go-redis/redis v6.15.9+incompatible github.com/go-sql-driver/mysql v1.6.0 github.com/golang-jwt/jwt v3.2.2+incompatible github.com/google/go-cmp v0.5.9 @@ -34,6 +35,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/h2non/filetype v1.1.3 github.com/hashicorp/go-retryablehttp v0.7.1 + github.com/jlaffaye/ftp v0.1.0 github.com/joho/godotenv v1.4.0 github.com/jpillora/overseer v1.1.6 github.com/kylelemons/godebug v1.1.0 @@ -98,7 +100,9 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect github.com/googleapis/gax-go/v2 v2.5.1 // indirect github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -110,6 +114,8 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect + github.com/onsi/ginkgo v1.16.5 // indirect + github.com/onsi/gomega v1.23.0 // indirect github.com/pierrec/lz4/v4 v4.1.14 // indirect github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -125,9 +131,9 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/net v0.0.0-20221002022538-bcab6841153b // indirect - golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec // indirect - golang.org/x/text v0.3.7 // indirect + golang.org/x/net v0.1.0 // indirect + golang.org/x/sys v0.1.0 // indirect + golang.org/x/text v0.4.0 // indirect golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect golang.org/x/tools v0.1.12 // indirect google.golang.org/api v0.96.0 // indirect diff --git a/go.sum b/go.sum index b821e539fddf..6c1fd5bb1b37 100644 --- a/go.sum +++ b/go.sum @@ -169,6 +169,9 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/getsentry/sentry-go v0.14.0 h1:rlOBkuFZRKKdUnKO+0U3JclRDQKlRu5vVQtkWSQvC70= github.com/getsentry/sentry-go v0.14.0/go.mod h1:RZPJKSw+adu8PBNygiri/A98FqVr2HtRckJk9XVxJ9I= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -195,8 +198,11 @@ github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= +github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= @@ -317,15 +323,20 @@ github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.7.1 h1:sUiuQAnLlbvmExtFQs72iFW/HXeUn8Z1aJLQ4LJJbTQ= github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -335,6 +346,8 @@ github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/jlaffaye/ftp v0.1.0 h1:DLGExl5nBoSFoNshAUHwXAezXwXBvFdx7/qwhucWNSE= +github.com/jlaffaye/ftp v0.1.0/go.mod h1:hhq4G4crv+nW2qXtNYcuzLeOudG92Ps37HEKeg2e3lE= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -393,6 +406,17 @@ github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uY github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nwaples/rardecode/v2 v2.0.0-beta.2 h1:e3mzJFJs4k83GXBEiTaQ5HgSc/kOK8q0rDaRO0MPaOk= github.com/nwaples/rardecode/v2 v2.0.0-beta.2/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= +github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= @@ -552,6 +576,7 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVD golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -572,6 +597,7 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -599,8 +625,8 @@ golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220907135653-1e95f45603a7/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.0.0-20221002022538-bcab6841153b h1:6e93nYa3hNqAvLr0pD4PN1fFS+gKzp2zAXqrnTCstqU= -golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -640,6 +666,7 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+v golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -648,8 +675,11 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -671,6 +701,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -710,11 +741,11 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908150016-7ac13a9a928d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI= -golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -723,8 +754,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -776,6 +808,7 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= @@ -985,8 +1018,11 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY= gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/detectors/uri/uri.go b/pkg/detectors/uri/uri.go index b68e32f2fb8e..2da80eeb6d23 100644 --- a/pkg/detectors/uri/uri.go +++ b/pkg/detectors/uri/uri.go @@ -1,14 +1,15 @@ package uri import ( - "bytes" "context" - "encoding/json" - "io" "net/http" "net/url" "regexp" "strings" + "time" + + "github.com/go-redis/redis" + "github.com/jlaffaye/ftp" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" @@ -24,6 +25,8 @@ var _ detectors.Detector = (*Scanner)(nil) var ( keyPat = regexp.MustCompile(`\b[a-zA-Z]{1,10}:?\/\/[-.%\w{}]{1,50}:([-.%\S]{3,50})@[-.%\w\/:]+\b`) + + client = common.SaneHttpClient() ) type proxyRes struct { @@ -37,7 +40,7 @@ func (s Scanner) Keywords() []string { } func allowlistedProtos(scheme string) bool { - allowlisted := []string{"http", "https", "mongodb", "redis", "ftp"} + allowlisted := []string{"http", "https", "redis", "ftp"} for _, s := range allowlisted { if s == scheme { return true @@ -48,11 +51,6 @@ func allowlistedProtos(scheme string) bool { // FromData will find and optionally verify URI secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { - //Prevent SSRF "https://us-central1-ssrfproxy.cloudfunctions.net/ssrfproxy-6d96c399-74bb-4299-b49d-c1c870277eb9" - //TODO add as config option - //TODO extend to other http outbound calls in other areas of code - ssrfProtectorURL := "https://us-central1-ssrfproxy.cloudfunctions.net/ssrfproxy-6d96c399-74bb-4299-b49d-c1c870277eb9" - dataStr := string(data) matches := keyPat.FindAllStringSubmatch(dataStr, -1) @@ -101,29 +99,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - client := common.SaneHttpClient() - // whitelist protocols - - // Assume a 200 response is a valid credential - postValues := map[string]string{"protocol": parsedURL.Scheme, "credentialed_uri": urlMatch} - jsonValue, _ := json.Marshal(postValues) - req, err := http.NewRequestWithContext(ctx, "POST", ssrfProtectorURL, bytes.NewBuffer(jsonValue)) - if err != nil { + switch parsedURL.Scheme { + case "http", "https": + s.Verified = verifyURL(ctx, parsedURL) + case "redis": + s.Verified = verifyRedis(ctx, parsedURL) + case "ftp": + s.Verified = verifyFTP(ctx, parsedURL) + default: continue } - req.Header.Add("Content-Type", "application/json") - res, err := client.Do(req) - if err == nil { - result := proxyRes{} - body, err := io.ReadAll(res.Body) - res.Body.Close() - if len(body) != 0 && err == nil { - err = json.Unmarshal(body, &result) - if err == nil && result.Verified { - s.Verified = true - } - } - } } if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) { @@ -135,3 +120,89 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return detectors.CleanResults(results), nil } + +func verifyFTP(ctx context.Context, u *url.URL) bool { + host := u.Host + if !strings.Contains(host, ":") { + host = host + ":21" + } + + c, err := ftp.Dial(host, ftp.DialWithTimeout(5*time.Second)) + if err != nil { + return false + } + + password, _ := u.User.Password() + err = c.Login(u.User.Username(), password) + if err != nil { + return false + } + + return true +} + +func verifyRedis(ctx context.Context, u *url.URL) bool { + opt, err := redis.ParseURL(u.String()) + if err != nil { + return false + } + + client := redis.NewClient(opt) + + status, err := client.Ping().Result() + if err == nil && status == "PONG" { + return true + } + + return false +} + +func verifyURL(ctx context.Context, u *url.URL) bool { + // defuse most SSRF payloads + u.Path = strings.TrimSuffix(u.Path, "/") + u.RawQuery = "" + u.Fragment = "" + + credentialedURL := u.String() + + u.User = nil + nonCredentialedURL := u.String() + + req, err := http.NewRequest("GET", credentialedURL, nil) + if err != nil { + return false + } + req = req.WithContext(ctx) + credentialedRes, err := client.Do(req) + if err != nil { + return false + } + credentialedRes.Body.Close() + + // If the credentialed URL returns a non 2XX code, we can assume it's a false positive. + if credentialedRes.StatusCode < 200 || credentialedRes.StatusCode > 299 { + return false + } + + time.Sleep(time.Millisecond * 10) + + req, err = http.NewRequest("GET", nonCredentialedURL, nil) + if err != nil { + return false + } + req = req.WithContext(ctx) + nonCredentialedRes, err := client.Do(req) + if err != nil { + return false + } + nonCredentialedRes.Body.Close() + + // If the non-credentialed URL returns a non 400-428 code and basic auth header, we can assume it's verified now. + if nonCredentialedRes.StatusCode >= 400 && nonCredentialedRes.StatusCode < 429 { + if nonCredentialedRes.Header.Get("WWW-Authenticate") != "" { + return true + } + } + + return false +} diff --git a/pkg/detectors/uri/uri_test.go b/pkg/detectors/uri/uri_test.go index c924a60d5cca..626d649f4ee9 100644 --- a/pkg/detectors/uri/uri_test.go +++ b/pkg/detectors/uri/uri_test.go @@ -7,23 +7,13 @@ import ( "context" "fmt" "testing" - "time" "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) func TestURI_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("URI_SECRET") - secretInactive := testSecrets.MustGetField("URI_INACTIVE") type args struct { ctx context.Context data []byte @@ -36,20 +26,19 @@ func TestURI_FromChunk(t *testing.T) { want []detectors.Result wantErr bool }{ - { name: "found, unverified, wrong username", s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a uri secret %s within", secretInactive)), + data: []byte(fmt.Sprintf("You can find a uri secret %s within", "https://user:pass@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx")), verify: true, }, want: []detectors.Result{ { DetectorType: detectorspb.DetectorType_URI, Verified: false, - Redacted: "https://user:**********@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx", + Redacted: "https://user:****@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx", }, }, wantErr: false, @@ -59,20 +48,20 @@ func TestURI_FromChunk(t *testing.T) { s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a uri secret %s within", secret)), + data: []byte(fmt.Sprintf("You can find a uri secret %s within", "https://httpwatch:pass@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx")), verify: true, }, want: []detectors.Result{ { DetectorType: detectorspb.DetectorType_URI, Verified: true, - Redacted: "https://httpwatch:**********@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx", + Redacted: "https://httpwatch:****@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx", }, }, wantErr: false, }, { - name: "bad proto", + name: "bad scheme", s: Scanner{}, args: args{ ctx: context.Background(), @@ -81,6 +70,42 @@ func TestURI_FromChunk(t *testing.T) { }, wantErr: false, }, + { + name: "verified FTP", + s: Scanner{}, + args: args{ + ctx: context.Background(), + // https://dlptest.com/ftp-test/ + data: []byte("ftp://dlpuser:rNrKYTX9g7z3RgJRmxWuGHbeu@ftp.dlptest.com"), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_URI, + Verified: true, + Redacted: "ftp://dlpuser:*************************@ftp.dlptest.com", + }, + }, + wantErr: false, + }, + { + name: "unverified FTP", + s: Scanner{}, + args: args{ + ctx: context.Background(), + // https://dlptest.com/ftp-test/ + data: []byte("ftp://dlpuser:invalid@ftp.dlptest.com"), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_URI, + Verified: false, + Redacted: "ftp://dlpuser:*******@ftp.dlptest.com", + }, + }, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From d27c2e68d9947cc8b24d9a405a6ba90f3a0051d0 Mon Sep 17 00:00:00 2001 From: Dustin Decker Date: Fri, 28 Oct 2022 15:57:56 -0700 Subject: [PATCH 3/7] test SSRF defuse --- pkg/detectors/uri/uri_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pkg/detectors/uri/uri_test.go b/pkg/detectors/uri/uri_test.go index 626d649f4ee9..ddd5cc02abc3 100644 --- a/pkg/detectors/uri/uri_test.go +++ b/pkg/detectors/uri/uri_test.go @@ -60,6 +60,23 @@ func TestURI_FromChunk(t *testing.T) { }, wantErr: false, }, + { + name: "found, verified, defused", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a uri secret %s within", "https://httpwatch:pass@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx?foo=bar")), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_URI, + Verified: true, + Redacted: "https://httpwatch:****@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx", + }, + }, + wantErr: false, + }, { name: "bad scheme", s: Scanner{}, From 0c3810b270806c9e301b8bad9a2f1f34657ce7f6 Mon Sep 17 00:00:00 2001 From: Dustin Decker Date: Fri, 28 Oct 2022 17:05:00 -0700 Subject: [PATCH 4/7] simplify err check logic per linter recommendation --- pkg/detectors/uri/uri.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/detectors/uri/uri.go b/pkg/detectors/uri/uri.go index 2da80eeb6d23..3ebae84aa16b 100644 --- a/pkg/detectors/uri/uri.go +++ b/pkg/detectors/uri/uri.go @@ -134,11 +134,8 @@ func verifyFTP(ctx context.Context, u *url.URL) bool { password, _ := u.User.Password() err = c.Login(u.User.Username(), password) - if err != nil { - return false - } - return true + return err == nil } func verifyRedis(ctx context.Context, u *url.URL) bool { From e6de528e40d2c2ef03268a812d1391e331e12e42 Mon Sep 17 00:00:00 2001 From: Dustin Decker Date: Tue, 1 Nov 2022 15:40:09 -0700 Subject: [PATCH 5/7] split up detectors --- pkg/detectors/ftp/ftp.go | 103 ++++++++++++++++++ pkg/detectors/ftp/ftp_test.go | 109 ++++++++++++++++++++ pkg/detectors/mongodb/mongodb.go | 2 +- pkg/detectors/npmtokenv2/npmtokenv2_test.go | 2 +- pkg/detectors/redis/redis.go | 101 ++++++++++++++++++ pkg/detectors/redis/redis_test.go | 90 ++++++++++++++++ pkg/detectors/uri/uri.go | 70 ++----------- pkg/detectors/uri/uri_test.go | 36 ------- pkg/pb/detectorspb/detectors.pb.go | 20 ++-- proto/detectors.proto | 2 + 10 files changed, 428 insertions(+), 107 deletions(-) create mode 100644 pkg/detectors/ftp/ftp.go create mode 100644 pkg/detectors/ftp/ftp_test.go create mode 100644 pkg/detectors/redis/redis.go create mode 100644 pkg/detectors/redis/redis_test.go diff --git a/pkg/detectors/ftp/ftp.go b/pkg/detectors/ftp/ftp.go new file mode 100644 index 000000000000..b75cb6d9037c --- /dev/null +++ b/pkg/detectors/ftp/ftp.go @@ -0,0 +1,103 @@ +package uri + +import ( + "context" + "net/url" + "regexp" + "strings" + "time" + + "github.com/jlaffaye/ftp" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +type Scanner struct { + allowKnownTestSites bool +} + +// Ensure the Scanner satisfies the interface at compile time. +var _ detectors.Detector = (*Scanner)(nil) + +var ( + keyPat = regexp.MustCompile(`\bftp:\/\/[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`) + + client = common.SaneHttpClient() +) + +// Keywords are used for efficiently pre-filtering chunks. +// Use identifiers in the secret preferably, or the provider name. +func (s Scanner) Keywords() []string { + return []string{"ftp://"} +} + +// FromData will find and optionally verify URI secrets in a given set of bytes. +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + matches := keyPat.FindAllStringSubmatch(dataStr, -1) + + for _, match := range matches { + urlMatch := match[0] + password := match[1] + + // Skip findings where the password only has "*" characters, this is a redacted password + if strings.Trim(password, "*") == "" { + continue + } + + parsedURL, err := url.Parse(urlMatch) + if err != nil { + continue + } + if _, ok := parsedURL.User.Password(); !ok { + continue + } + + redact := strings.TrimSpace(strings.Replace(urlMatch, password, strings.Repeat("*", len(password)), -1)) + + s := detectors.Result{ + DetectorType: detectorspb.DetectorType_FTP, + Raw: []byte(urlMatch), + Redacted: redact, + } + + if verify { + s.Verified = verifyFTP(ctx, parsedURL) + } + + if !s.Verified { + // Skip unverified findings where the password starts with a `$` - it's almost certainly a variable. + if strings.HasPrefix(password, "$") { + continue + } + } + + if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) { + continue + } + + results = append(results, s) + } + + return detectors.CleanResults(results), nil +} + +func verifyFTP(ctx context.Context, u *url.URL) bool { + host := u.Host + if !strings.Contains(host, ":") { + host = host + ":21" + } + + c, err := ftp.Dial(host, ftp.DialWithTimeout(5*time.Second)) + if err != nil { + return false + } + + password, _ := u.User.Password() + err = c.Login(u.User.Username(), password) + + return err == nil +} diff --git a/pkg/detectors/ftp/ftp_test.go b/pkg/detectors/ftp/ftp_test.go new file mode 100644 index 000000000000..72821eb2dea5 --- /dev/null +++ b/pkg/detectors/ftp/ftp_test.go @@ -0,0 +1,109 @@ +//go:build detectors +// +build detectors + +package uri + +import ( + "context" + "testing" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestURI_FromChunk(t *testing.T) { + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "bad scheme", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("file://user:pass@foo.com:123/wh/at/ever"), + verify: true, + }, + wantErr: false, + }, + { + name: "verified FTP", + s: Scanner{}, + args: args{ + ctx: context.Background(), + // https://dlptest.com/ftp-test/ + data: []byte("ftp://dlpuser:rNrKYTX9g7z3RgJRmxWuGHbeu@ftp.dlptest.com"), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_FTP, + Verified: true, + Redacted: "ftp://dlpuser:*************************@ftp.dlptest.com", + }, + }, + wantErr: false, + }, + { + name: "unverified FTP", + s: Scanner{}, + args: args{ + ctx: context.Background(), + // https://dlptest.com/ftp-test/ + data: []byte("ftp://dlpuser:invalid@ftp.dlptest.com"), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_FTP, + Verified: false, + Redacted: "ftp://dlpuser:*******@ftp.dlptest.com", + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{allowKnownTestSites: true} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("URI.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + // if os.Getenv("FORCE_PASS_DIFF") == "true" { + // return + // } + for i := range got { + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/mongodb/mongodb.go b/pkg/detectors/mongodb/mongodb.go index 2841f54c38b1..3c1d4a0677fe 100644 --- a/pkg/detectors/mongodb/mongodb.go +++ b/pkg/detectors/mongodb/mongodb.go @@ -21,7 +21,7 @@ var _ detectors.Detector = (*Scanner)(nil) var ( // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://[-.%\w{}]{1,50}:([-.%\S]{3,50})@[-.%\w\/:]+)\b`) + keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+)\b`) // TODO: Add support for sharded cluster, replica set and Atlas Deployment. ) diff --git a/pkg/detectors/npmtokenv2/npmtokenv2_test.go b/pkg/detectors/npmtokenv2/npmtokenv2_test.go index 4714e13b795a..4d8e35af23b0 100644 --- a/pkg/detectors/npmtokenv2/npmtokenv2_test.go +++ b/pkg/detectors/npmtokenv2/npmtokenv2_test.go @@ -1,7 +1,7 @@ //go:build detectors // +build detectors -package npmtoken_new +package npmtokenv2 import ( "context" diff --git a/pkg/detectors/redis/redis.go b/pkg/detectors/redis/redis.go new file mode 100644 index 000000000000..0e833bc78b19 --- /dev/null +++ b/pkg/detectors/redis/redis.go @@ -0,0 +1,101 @@ +package uri + +import ( + "context" + "net/url" + "regexp" + "strings" + + "github.com/go-redis/redis" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +type Scanner struct { + allowKnownTestSites bool +} + +// Ensure the Scanner satisfies the interface at compile time. +var _ detectors.Detector = (*Scanner)(nil) + +var ( + keyPat = regexp.MustCompile(`\bredis:\/\/[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`) + + client = common.SaneHttpClient() +) + +// Keywords are used for efficiently pre-filtering chunks. +// Use identifiers in the secret preferably, or the provider name. +func (s Scanner) Keywords() []string { + return []string{"redis"} +} + +// FromData will find and optionally verify URI secrets in a given set of bytes. +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + matches := keyPat.FindAllStringSubmatch(dataStr, -1) + + for _, match := range matches { + urlMatch := match[0] + password := match[1] + + // Skip findings where the password only has "*" characters, this is a redacted password + if strings.Trim(password, "*") == "" { + continue + } + + parsedURL, err := url.Parse(urlMatch) + if err != nil { + continue + } + if _, ok := parsedURL.User.Password(); !ok { + continue + } + + redact := strings.TrimSpace(strings.Replace(urlMatch, password, strings.Repeat("*", len(password)), -1)) + + s := detectors.Result{ + DetectorType: detectorspb.DetectorType_Redis, + Raw: []byte(urlMatch), + Redacted: redact, + } + + if verify { + s.Verified = verifyRedis(ctx, parsedURL) + } + + if !s.Verified { + // Skip unverified findings where the password starts with a `$` - it's almost certainly a variable. + if strings.HasPrefix(password, "$") { + continue + } + } + + if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) { + continue + } + + results = append(results, s) + } + + return detectors.CleanResults(results), nil +} + +func verifyRedis(ctx context.Context, u *url.URL) bool { + opt, err := redis.ParseURL(u.String()) + if err != nil { + return false + } + + client := redis.NewClient(opt) + + status, err := client.Ping().Result() + if err == nil && status == "PONG" { + return true + } + + return false +} diff --git a/pkg/detectors/redis/redis_test.go b/pkg/detectors/redis/redis_test.go new file mode 100644 index 000000000000..3b7224b16d33 --- /dev/null +++ b/pkg/detectors/redis/redis_test.go @@ -0,0 +1,90 @@ +//go:build detectors +// +build detectors + +package uri + +import ( + "context" + "testing" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestURI_FromChunk(t *testing.T) { + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "bad scheme", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("file://user:pass@foo.com:123/wh/at/ever"), + verify: true, + }, + wantErr: false, + }, + { + name: "unverified Redis", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("redis://user:invalid@redis.com"), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Redis, + Verified: false, + Redacted: "redis://user:*******@redis.com", + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{allowKnownTestSites: true} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("URI.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + // if os.Getenv("FORCE_PASS_DIFF") == "true" { + // return + // } + for i := range got { + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/uri/uri.go b/pkg/detectors/uri/uri.go index 3ebae84aa16b..492c8e1abfe8 100644 --- a/pkg/detectors/uri/uri.go +++ b/pkg/detectors/uri/uri.go @@ -8,9 +8,6 @@ import ( "strings" "time" - "github.com/go-redis/redis" - "github.com/jlaffaye/ftp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -24,7 +21,7 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - keyPat = regexp.MustCompile(`\b[a-zA-Z]{1,10}:?\/\/[-.%\w{}]{1,50}:([-.%\S]{3,50})@[-.%\w\/:]+\b`) + keyPat = regexp.MustCompile(`\b(?:http(?:s)?:)?\/\/[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`) client = common.SaneHttpClient() ) @@ -39,16 +36,6 @@ func (s Scanner) Keywords() []string { return []string{"http"} } -func allowlistedProtos(scheme string) bool { - allowlisted := []string{"http", "https", "redis", "ftp"} - for _, s := range allowlisted { - if s == scheme { - return true - } - } - return false -} - // FromData will find and optionally verify URI secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) @@ -69,11 +56,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result urlMatch := match[0] password := match[1] - // Skip findings where the password starts with a `$` - it's almost certainly a variable. - if strings.HasPrefix(password, "$") { - continue - } - // Skip findings where the password only has "*" characters, this is a redacted password if strings.Trim(password, "*") == "" { continue @@ -86,9 +68,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if _, ok := parsedURL.User.Password(); !ok { continue } - if !allowlistedProtos(parsedURL.Scheme) { - continue - } redact := strings.TrimSpace(strings.Replace(urlMatch, password, strings.Repeat("*", len(password)), -1)) @@ -99,14 +78,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - switch parsedURL.Scheme { - case "http", "https": - s.Verified = verifyURL(ctx, parsedURL) - case "redis": - s.Verified = verifyRedis(ctx, parsedURL) - case "ftp": - s.Verified = verifyFTP(ctx, parsedURL) - default: + s.Verified = verifyURL(ctx, parsedURL) + } + + if !s.Verified { + // Skip unverified findings where the password starts with a `$` - it's almost certainly a variable. + if strings.HasPrefix(password, "$") { continue } } @@ -121,39 +98,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return detectors.CleanResults(results), nil } -func verifyFTP(ctx context.Context, u *url.URL) bool { - host := u.Host - if !strings.Contains(host, ":") { - host = host + ":21" - } - - c, err := ftp.Dial(host, ftp.DialWithTimeout(5*time.Second)) - if err != nil { - return false - } - - password, _ := u.User.Password() - err = c.Login(u.User.Username(), password) - - return err == nil -} - -func verifyRedis(ctx context.Context, u *url.URL) bool { - opt, err := redis.ParseURL(u.String()) - if err != nil { - return false - } - - client := redis.NewClient(opt) - - status, err := client.Ping().Result() - if err == nil && status == "PONG" { - return true - } - - return false -} - func verifyURL(ctx context.Context, u *url.URL) bool { // defuse most SSRF payloads u.Path = strings.TrimSuffix(u.Path, "/") diff --git a/pkg/detectors/uri/uri_test.go b/pkg/detectors/uri/uri_test.go index ddd5cc02abc3..012b5d84621e 100644 --- a/pkg/detectors/uri/uri_test.go +++ b/pkg/detectors/uri/uri_test.go @@ -87,42 +87,6 @@ func TestURI_FromChunk(t *testing.T) { }, wantErr: false, }, - { - name: "verified FTP", - s: Scanner{}, - args: args{ - ctx: context.Background(), - // https://dlptest.com/ftp-test/ - data: []byte("ftp://dlpuser:rNrKYTX9g7z3RgJRmxWuGHbeu@ftp.dlptest.com"), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_URI, - Verified: true, - Redacted: "ftp://dlpuser:*************************@ftp.dlptest.com", - }, - }, - wantErr: false, - }, - { - name: "unverified FTP", - s: Scanner{}, - args: args{ - ctx: context.Background(), - // https://dlptest.com/ftp-test/ - data: []byte("ftp://dlpuser:invalid@ftp.dlptest.com"), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_URI, - Verified: false, - Redacted: "ftp://dlpuser:*******@ftp.dlptest.com", - }, - }, - wantErr: false, - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/pb/detectorspb/detectors.pb.go b/pkg/pb/detectorspb/detectors.pb.go index 108890eb9139..724bb8dd76a5 100644 --- a/pkg/pb/detectorspb/detectors.pb.go +++ b/pkg/pb/detectorspb/detectors.pb.go @@ -967,6 +967,8 @@ const ( DetectorType_NGC DetectorType = 896 DetectorType_DigitalOceanV2 DetectorType = 897 DetectorType_SQLServer DetectorType = 898 + DetectorType_FTP DetectorType = 899 + DetectorType_Redis DetectorType = 900 ) // Enum value maps for DetectorType. @@ -1867,6 +1869,8 @@ var ( 896: "NGC", 897: "DigitalOceanV2", 898: "SQLServer", + 899: "FTP", + 900: "Redis", } DetectorType_value = map[string]int32{ "Alibaba": 0, @@ -2764,6 +2768,8 @@ var ( "NGC": 896, "DigitalOceanV2": 897, "SQLServer": 898, + "FTP": 899, + "Redis": 900, } ) @@ -3128,7 +3134,7 @@ var file_detectors_proto_rawDesc = []byte{ 0x0a, 0x0b, 0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49, 0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, - 0x02, 0x2a, 0xc1, 0x70, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, + 0x02, 0x2a, 0xd7, 0x70, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x10, 0x03, 0x12, 0x0a, 0x0a, @@ -4028,11 +4034,13 @@ var file_detectors_proto_rawDesc = []byte{ 0x6f, 0x44, 0x42, 0x10, 0xff, 0x06, 0x12, 0x08, 0x0a, 0x03, 0x4e, 0x47, 0x43, 0x10, 0x80, 0x07, 0x12, 0x13, 0x0a, 0x0e, 0x44, 0x69, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x4f, 0x63, 0x65, 0x61, 0x6e, 0x56, 0x32, 0x10, 0x81, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x53, 0x51, 0x4c, 0x53, 0x65, 0x72, 0x76, - 0x65, 0x72, 0x10, 0x82, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, - 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, - 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, - 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, - 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x72, 0x10, 0x82, 0x07, 0x12, 0x08, 0x0a, 0x03, 0x46, 0x54, 0x50, 0x10, 0x83, 0x07, 0x12, + 0x0a, 0x0a, 0x05, 0x52, 0x65, 0x64, 0x69, 0x73, 0x10, 0x84, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, + 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, + 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, + 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( diff --git a/proto/detectors.proto b/proto/detectors.proto index e7e809165f62..289ea950a58c 100644 --- a/proto/detectors.proto +++ b/proto/detectors.proto @@ -906,6 +906,8 @@ enum DetectorType { NGC = 896; DigitalOceanV2 = 897; SQLServer = 898; + FTP = 899; + Redis = 900; } message Result { From ef18b1721873018d6424f47f28a7622d876953c2 Mon Sep 17 00:00:00 2001 From: Dustin Decker Date: Tue, 1 Nov 2022 17:15:46 -0700 Subject: [PATCH 6/7] address comments --- pkg/detectors/ftp/ftp.go | 11 +++-------- pkg/detectors/ftp/ftp_test.go | 4 ++-- pkg/detectors/npmtokenv2/npmtokenv2_test.go | 4 ++-- pkg/detectors/redis/redis.go | 8 +++----- pkg/detectors/redis/redis_test.go | 4 ++-- pkg/detectors/uri/uri.go | 2 +- pkg/engine/defaults.go | 4 ++++ 7 files changed, 17 insertions(+), 20 deletions(-) diff --git a/pkg/detectors/ftp/ftp.go b/pkg/detectors/ftp/ftp.go index 0d9af7ba88ae..a7c34840bb86 100644 --- a/pkg/detectors/ftp/ftp.go +++ b/pkg/detectors/ftp/ftp.go @@ -1,4 +1,4 @@ -package uri +package ftp import ( "context" @@ -9,22 +9,17 @@ import ( "github.com/jlaffaye/ftp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct { - allowKnownTestSites bool -} +type Scanner struct{} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) var ( - keyPat = regexp.MustCompile(`\bftp:\/\/[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`) - - client = common.SaneHttpClient() + keyPat = regexp.MustCompile(`\bftp://[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`) ) // Keywords are used for efficiently pre-filtering chunks. diff --git a/pkg/detectors/ftp/ftp_test.go b/pkg/detectors/ftp/ftp_test.go index 72821eb2dea5..16a6d61bf496 100644 --- a/pkg/detectors/ftp/ftp_test.go +++ b/pkg/detectors/ftp/ftp_test.go @@ -1,7 +1,7 @@ //go:build detectors // +build detectors -package uri +package ftp import ( "context" @@ -74,7 +74,7 @@ func TestURI_FromChunk(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := Scanner{allowKnownTestSites: true} + s := Scanner{} got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) if (err != nil) != tt.wantErr { t.Errorf("URI.FromData() error = %v, wantErr %v", err, tt.wantErr) diff --git a/pkg/detectors/npmtokenv2/npmtokenv2_test.go b/pkg/detectors/npmtokenv2/npmtokenv2_test.go index 4d8e35af23b0..2fd623b81565 100644 --- a/pkg/detectors/npmtokenv2/npmtokenv2_test.go +++ b/pkg/detectors/npmtokenv2/npmtokenv2_test.go @@ -48,7 +48,7 @@ func TestNpmToken_New_FromChunk(t *testing.T) { }, want: []detectors.Result{ { - DetectorType: detectorspb.DetectorType_NpmToken_New, + DetectorType: detectorspb.DetectorType_NpmToken, Verified: true, }, }, @@ -64,7 +64,7 @@ func TestNpmToken_New_FromChunk(t *testing.T) { }, want: []detectors.Result{ { - DetectorType: detectorspb.DetectorType_NpmToken_New, + DetectorType: detectorspb.DetectorType_NpmToken, Verified: false, }, }, diff --git a/pkg/detectors/redis/redis.go b/pkg/detectors/redis/redis.go index 15d70b093026..044d63422749 100644 --- a/pkg/detectors/redis/redis.go +++ b/pkg/detectors/redis/redis.go @@ -1,4 +1,4 @@ -package uri +package redis import ( "context" @@ -13,15 +13,13 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct { - allowKnownTestSites bool -} +type Scanner struct{} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) var ( - keyPat = regexp.MustCompile(`\bredis:\/\/[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`) + keyPat = regexp.MustCompile(`\bredis://[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`) client = common.SaneHttpClient() ) diff --git a/pkg/detectors/redis/redis_test.go b/pkg/detectors/redis/redis_test.go index 3b7224b16d33..6ad2084fbbdf 100644 --- a/pkg/detectors/redis/redis_test.go +++ b/pkg/detectors/redis/redis_test.go @@ -1,7 +1,7 @@ //go:build detectors // +build detectors -package uri +package redis import ( "context" @@ -55,7 +55,7 @@ func TestURI_FromChunk(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := Scanner{allowKnownTestSites: true} + s := Scanner{} got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) if (err != nil) != tt.wantErr { t.Errorf("URI.FromData() error = %v, wantErr %v", err, tt.wantErr) diff --git a/pkg/detectors/uri/uri.go b/pkg/detectors/uri/uri.go index 231fadc2bf2b..9112eaed7ee9 100644 --- a/pkg/detectors/uri/uri.go +++ b/pkg/detectors/uri/uri.go @@ -21,7 +21,7 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - keyPat = regexp.MustCompile(`\b(?:http(?:s)?:)?\/\/[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`) + keyPat = regexp.MustCompile(`\b(?:https?:)?\/\/[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`) client = common.SaneHttpClient() ) diff --git a/pkg/engine/defaults.go b/pkg/engine/defaults.go index b5b8d10c022a..1ea37ba4c88f 100644 --- a/pkg/engine/defaults.go +++ b/pkg/engine/defaults.go @@ -255,6 +255,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/freshbooks" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/freshdesk" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/front" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ftp" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/fulcrum" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/fullstory" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/fusebill" @@ -510,6 +511,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/reallysimplesystems" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/rebrandly" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/rechargepayments" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/redis" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/refiner" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/rentman" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/repairshopr" @@ -1484,5 +1486,7 @@ func DefaultDetectors() []detectors.Detector { npmtoken.Scanner{}, npmtokenv2.Scanner{}, sqlserver.Scanner{}, + redis.Scanner{}, + ftp.Scanner{}, } } From e57a010432e700182ef90e2625ef67deb8fb090f Mon Sep 17 00:00:00 2001 From: Dustin Decker Date: Tue, 1 Nov 2022 17:23:33 -0700 Subject: [PATCH 7/7] remove unused var --- pkg/detectors/redis/redis.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/detectors/redis/redis.go b/pkg/detectors/redis/redis.go index 044d63422749..f68ab3b0f442 100644 --- a/pkg/detectors/redis/redis.go +++ b/pkg/detectors/redis/redis.go @@ -8,7 +8,6 @@ import ( "github.com/go-redis/redis" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -20,8 +19,6 @@ var _ detectors.Detector = (*Scanner)(nil) var ( keyPat = regexp.MustCompile(`\bredis://[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`) - - client = common.SaneHttpClient() ) // Keywords are used for efficiently pre-filtering chunks.