diff --git a/go.mod b/go.mod index 36067b2b78f0..75d44b1267ab 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/go-git/go-git/v5 v5.4.2 github.com/go-logr/logr v1.2.3 github.com/go-logr/zapr v1.2.3 + github.com/go-redis/redis v6.15.9+incompatible github.com/go-sql-driver/mysql v1.6.0 github.com/golang-jwt/jwt v3.2.2+incompatible github.com/google/go-cmp v0.5.9 @@ -34,6 +35,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/h2non/filetype v1.1.3 github.com/hashicorp/go-retryablehttp v0.7.1 + github.com/jlaffaye/ftp v0.1.0 github.com/joho/godotenv v1.4.0 github.com/jpillora/overseer v1.1.6 github.com/kylelemons/godebug v1.1.0 @@ -97,7 +99,9 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect github.com/googleapis/gax-go/v2 v2.6.0 // indirect github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -109,6 +113,8 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect + github.com/onsi/ginkgo v1.16.5 // indirect + github.com/onsi/gomega v1.23.0 // indirect github.com/pierrec/lz4/v4 v4.1.14 // indirect github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -124,9 +130,9 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458 // indirect - golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec // indirect - golang.org/x/text v0.3.7 // indirect + golang.org/x/net v0.1.0 // indirect + golang.org/x/sys v0.1.0 // indirect + golang.org/x/text v0.4.0 // indirect golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect golang.org/x/tools v0.1.12 // indirect google.golang.org/api v0.99.0 // indirect diff --git a/go.sum b/go.sum index 21087e34ac6c..57a5ee0695aa 100644 --- a/go.sum +++ b/go.sum @@ -135,6 +135,9 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/getsentry/sentry-go v0.14.0 h1:rlOBkuFZRKKdUnKO+0U3JclRDQKlRu5vVQtkWSQvC70= github.com/getsentry/sentry-go v0.14.0/go.mod h1:RZPJKSw+adu8PBNygiri/A98FqVr2HtRckJk9XVxJ9I= github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= @@ -160,8 +163,11 @@ github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= +github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= @@ -261,15 +267,20 @@ github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.7.1 h1:sUiuQAnLlbvmExtFQs72iFW/HXeUn8Z1aJLQ4LJJbTQ= github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -279,6 +290,8 @@ github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/jlaffaye/ftp v0.1.0 h1:DLGExl5nBoSFoNshAUHwXAezXwXBvFdx7/qwhucWNSE= +github.com/jlaffaye/ftp v0.1.0/go.mod h1:hhq4G4crv+nW2qXtNYcuzLeOudG92Ps37HEKeg2e3lE= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -337,6 +350,17 @@ github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uY github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nwaples/rardecode/v2 v2.0.0-beta.2 h1:e3mzJFJs4k83GXBEiTaQ5HgSc/kOK8q0rDaRO0MPaOk= github.com/nwaples/rardecode/v2 v2.0.0-beta.2/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= +github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= @@ -491,6 +515,7 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVD golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -511,6 +536,7 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -527,8 +553,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220907135653-1e95f45603a7/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458 h1:MgJ6t2zo8v0tbmLCueaCbF1RM+TtB0rs3Lv8DGtOIpY= -golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -556,6 +582,7 @@ golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0 h1:cu5kTvlzcw1Q5S9f5ip1/cpi golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -564,8 +591,11 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -587,6 +617,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -605,11 +636,11 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908150016-7ac13a9a928d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI= -golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -617,8 +648,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -670,6 +702,7 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= @@ -788,8 +821,11 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY= gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/detectors/ftp/ftp.go b/pkg/detectors/ftp/ftp.go new file mode 100644 index 000000000000..a7c34840bb86 --- /dev/null +++ b/pkg/detectors/ftp/ftp.go @@ -0,0 +1,98 @@ +package ftp + +import ( + "context" + "net/url" + "regexp" + "strings" + "time" + + "github.com/jlaffaye/ftp" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +type Scanner struct{} + +// Ensure the Scanner satisfies the interface at compile time. +var _ detectors.Detector = (*Scanner)(nil) + +var ( + keyPat = regexp.MustCompile(`\bftp://[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`) +) + +// Keywords are used for efficiently pre-filtering chunks. +// Use identifiers in the secret preferably, or the provider name. +func (s Scanner) Keywords() []string { + return []string{"ftp://"} +} + +// FromData will find and optionally verify URI secrets in a given set of bytes. +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + matches := keyPat.FindAllStringSubmatch(dataStr, -1) + + for _, match := range matches { + urlMatch := match[0] + password := match[1] + + // Skip findings where the password only has "*" characters, this is a redacted password + if strings.Trim(password, "*") == "" { + continue + } + + parsedURL, err := url.Parse(urlMatch) + if err != nil { + continue + } + if _, ok := parsedURL.User.Password(); !ok { + continue + } + + redact := strings.TrimSpace(strings.Replace(urlMatch, password, strings.Repeat("*", len(password)), -1)) + + s := detectors.Result{ + DetectorType: detectorspb.DetectorType_FTP, + Raw: []byte(urlMatch), + Redacted: redact, + } + + if verify { + s.Verified = verifyFTP(ctx, parsedURL) + } + + if !s.Verified { + // Skip unverified findings where the password starts with a `$` - it's almost certainly a variable. + if strings.HasPrefix(password, "$") { + continue + } + } + + if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) { + continue + } + + results = append(results, s) + } + + return results, nil +} + +func verifyFTP(ctx context.Context, u *url.URL) bool { + host := u.Host + if !strings.Contains(host, ":") { + host = host + ":21" + } + + c, err := ftp.Dial(host, ftp.DialWithTimeout(5*time.Second)) + if err != nil { + return false + } + + password, _ := u.User.Password() + err = c.Login(u.User.Username(), password) + + return err == nil +} diff --git a/pkg/detectors/ftp/ftp_test.go b/pkg/detectors/ftp/ftp_test.go new file mode 100644 index 000000000000..16a6d61bf496 --- /dev/null +++ b/pkg/detectors/ftp/ftp_test.go @@ -0,0 +1,109 @@ +//go:build detectors +// +build detectors + +package ftp + +import ( + "context" + "testing" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestURI_FromChunk(t *testing.T) { + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "bad scheme", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("file://user:pass@foo.com:123/wh/at/ever"), + verify: true, + }, + wantErr: false, + }, + { + name: "verified FTP", + s: Scanner{}, + args: args{ + ctx: context.Background(), + // https://dlptest.com/ftp-test/ + data: []byte("ftp://dlpuser:rNrKYTX9g7z3RgJRmxWuGHbeu@ftp.dlptest.com"), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_FTP, + Verified: true, + Redacted: "ftp://dlpuser:*************************@ftp.dlptest.com", + }, + }, + wantErr: false, + }, + { + name: "unverified FTP", + s: Scanner{}, + args: args{ + ctx: context.Background(), + // https://dlptest.com/ftp-test/ + data: []byte("ftp://dlpuser:invalid@ftp.dlptest.com"), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_FTP, + Verified: false, + Redacted: "ftp://dlpuser:*******@ftp.dlptest.com", + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("URI.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + // if os.Getenv("FORCE_PASS_DIFF") == "true" { + // return + // } + for i := range got { + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/mongodb/mongodb.go b/pkg/detectors/mongodb/mongodb.go index 357f8130c18a..2bd64e009461 100644 --- a/pkg/detectors/mongodb/mongodb.go +++ b/pkg/detectors/mongodb/mongodb.go @@ -21,7 +21,7 @@ var _ detectors.Detector = (*Scanner)(nil) var ( // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://(?:[^:]+:(?:[^@]+)?@)?(?:[^/]+|/.+.sock?,?)+(?:/([^/."*<>:|?]*))?\?(?:(.+=?=\S+)&?)+)\b`) + keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+)\b`) // TODO: Add support for sharded cluster, replica set and Atlas Deployment. ) diff --git a/pkg/detectors/npmtokenv2/npmtokenv2_test.go b/pkg/detectors/npmtokenv2/npmtokenv2_test.go index 4714e13b795a..2fd623b81565 100644 --- a/pkg/detectors/npmtokenv2/npmtokenv2_test.go +++ b/pkg/detectors/npmtokenv2/npmtokenv2_test.go @@ -1,7 +1,7 @@ //go:build detectors // +build detectors -package npmtoken_new +package npmtokenv2 import ( "context" @@ -48,7 +48,7 @@ func TestNpmToken_New_FromChunk(t *testing.T) { }, want: []detectors.Result{ { - DetectorType: detectorspb.DetectorType_NpmToken_New, + DetectorType: detectorspb.DetectorType_NpmToken, Verified: true, }, }, @@ -64,7 +64,7 @@ func TestNpmToken_New_FromChunk(t *testing.T) { }, want: []detectors.Result{ { - DetectorType: detectorspb.DetectorType_NpmToken_New, + DetectorType: detectorspb.DetectorType_NpmToken, Verified: false, }, }, diff --git a/pkg/detectors/redis/redis.go b/pkg/detectors/redis/redis.go new file mode 100644 index 000000000000..f68ab3b0f442 --- /dev/null +++ b/pkg/detectors/redis/redis.go @@ -0,0 +1,96 @@ +package redis + +import ( + "context" + "net/url" + "regexp" + "strings" + + "github.com/go-redis/redis" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +type Scanner struct{} + +// Ensure the Scanner satisfies the interface at compile time. +var _ detectors.Detector = (*Scanner)(nil) + +var ( + keyPat = regexp.MustCompile(`\bredis://[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`) +) + +// Keywords are used for efficiently pre-filtering chunks. +// Use identifiers in the secret preferably, or the provider name. +func (s Scanner) Keywords() []string { + return []string{"redis"} +} + +// FromData will find and optionally verify URI secrets in a given set of bytes. +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + matches := keyPat.FindAllStringSubmatch(dataStr, -1) + + for _, match := range matches { + urlMatch := match[0] + password := match[1] + + // Skip findings where the password only has "*" characters, this is a redacted password + if strings.Trim(password, "*") == "" { + continue + } + + parsedURL, err := url.Parse(urlMatch) + if err != nil { + continue + } + if _, ok := parsedURL.User.Password(); !ok { + continue + } + + redact := strings.TrimSpace(strings.Replace(urlMatch, password, strings.Repeat("*", len(password)), -1)) + + s := detectors.Result{ + DetectorType: detectorspb.DetectorType_Redis, + Raw: []byte(urlMatch), + Redacted: redact, + } + + if verify { + s.Verified = verifyRedis(ctx, parsedURL) + } + + if !s.Verified { + // Skip unverified findings where the password starts with a `$` - it's almost certainly a variable. + if strings.HasPrefix(password, "$") { + continue + } + } + + if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) { + continue + } + + results = append(results, s) + } + + return results, nil +} + +func verifyRedis(ctx context.Context, u *url.URL) bool { + opt, err := redis.ParseURL(u.String()) + if err != nil { + return false + } + + client := redis.NewClient(opt) + + status, err := client.Ping().Result() + if err == nil && status == "PONG" { + return true + } + + return false +} diff --git a/pkg/detectors/redis/redis_test.go b/pkg/detectors/redis/redis_test.go new file mode 100644 index 000000000000..6ad2084fbbdf --- /dev/null +++ b/pkg/detectors/redis/redis_test.go @@ -0,0 +1,90 @@ +//go:build detectors +// +build detectors + +package redis + +import ( + "context" + "testing" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestURI_FromChunk(t *testing.T) { + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "bad scheme", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("file://user:pass@foo.com:123/wh/at/ever"), + verify: true, + }, + wantErr: false, + }, + { + name: "unverified Redis", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("redis://user:invalid@redis.com"), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Redis, + Verified: false, + Redacted: "redis://user:*******@redis.com", + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("URI.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + // if os.Getenv("FORCE_PASS_DIFF") == "true" { + // return + // } + for i := range got { + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/uri/uri.go b/pkg/detectors/uri/uri.go index 6a4278ef028f..9112eaed7ee9 100644 --- a/pkg/detectors/uri/uri.go +++ b/pkg/detectors/uri/uri.go @@ -1,14 +1,12 @@ package uri import ( - "bytes" "context" - "encoding/json" - "io" "net/http" "net/url" "regexp" "strings" + "time" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" @@ -23,7 +21,9 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - keyPat = regexp.MustCompile(`\b[a-zA-Z]{1,10}:?\/\/[-.%\w{}]{1,50}:([-.%\S]{3,50})@[-.%\w\/:]+\b`) + keyPat = regexp.MustCompile(`\b(?:https?:)?\/\/[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`) + + client = common.SaneHttpClient() ) type proxyRes struct { @@ -36,23 +36,8 @@ func (s Scanner) Keywords() []string { return []string{"http"} } -func allowlistedProtos(scheme string) bool { - allowlisted := []string{"http", "https", "mongodb", "redis", "ftp"} - for _, s := range allowlisted { - if s == scheme { - return true - } - } - return false -} - // FromData will find and optionally verify URI secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { - // Prevent SSRF "https://us-central1-ssrfproxy.cloudfunctions.net/ssrfproxy-6d96c399-74bb-4299-b49d-c1c870277eb9" - // TODO add as config option - // TODO extend to other http outbound calls in other areas of code - ssrfProtectorURL := "https://us-central1-ssrfproxy.cloudfunctions.net/ssrfproxy-6d96c399-74bb-4299-b49d-c1c870277eb9" - dataStr := string(data) matches := keyPat.FindAllStringSubmatch(dataStr, -1) @@ -71,11 +56,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result urlMatch := match[0] password := match[1] - // Skip findings where the password starts with a `$` - it's almost certainly a variable. - if strings.HasPrefix(password, "$") { - continue - } - // Skip findings where the password only has "*" characters, this is a redacted password if strings.Trim(password, "*") == "" { continue @@ -88,9 +68,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if _, ok := parsedURL.User.Password(); !ok { continue } - if !allowlistedProtos(parsedURL.Scheme) { - continue - } redact := strings.TrimSpace(strings.Replace(urlMatch, password, strings.Repeat("*", len(password)), -1)) @@ -101,29 +78,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - client := common.SaneHttpClient() - // whitelist protocols - - // Assume a 200 response is a valid credential - postValues := map[string]string{"protocol": parsedURL.Scheme, "credentialed_uri": urlMatch} - jsonValue, _ := json.Marshal(postValues) - req, err := http.NewRequestWithContext(ctx, "POST", ssrfProtectorURL, bytes.NewBuffer(jsonValue)) - if err != nil { + s.Verified = verifyURL(ctx, parsedURL) + } + + if !s.Verified { + // Skip unverified findings where the password starts with a `$` - it's almost certainly a variable. + if strings.HasPrefix(password, "$") { continue } - req.Header.Add("Content-Type", "application/json") - res, err := client.Do(req) - if err == nil { - result := proxyRes{} - body, err := io.ReadAll(res.Body) - res.Body.Close() - if len(body) != 0 && err == nil { - err = json.Unmarshal(body, &result) - if err == nil && result.Verified { - s.Verified = true - } - } - } } if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) { @@ -135,3 +97,53 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } + +func verifyURL(ctx context.Context, u *url.URL) bool { + // defuse most SSRF payloads + u.Path = strings.TrimSuffix(u.Path, "/") + u.RawQuery = "" + u.Fragment = "" + + credentialedURL := u.String() + + u.User = nil + nonCredentialedURL := u.String() + + req, err := http.NewRequest("GET", credentialedURL, nil) + if err != nil { + return false + } + req = req.WithContext(ctx) + credentialedRes, err := client.Do(req) + if err != nil { + return false + } + credentialedRes.Body.Close() + + // If the credentialed URL returns a non 2XX code, we can assume it's a false positive. + if credentialedRes.StatusCode < 200 || credentialedRes.StatusCode > 299 { + return false + } + + time.Sleep(time.Millisecond * 10) + + req, err = http.NewRequest("GET", nonCredentialedURL, nil) + if err != nil { + return false + } + req = req.WithContext(ctx) + nonCredentialedRes, err := client.Do(req) + if err != nil { + return false + } + nonCredentialedRes.Body.Close() + + // If the non-credentialed URL returns a non 400-428 code and basic auth header, we can assume it's verified now. + if nonCredentialedRes.StatusCode >= 400 && nonCredentialedRes.StatusCode < 429 { + if nonCredentialedRes.Header.Get("WWW-Authenticate") != "" { + return true + } + } + + return false +} diff --git a/pkg/detectors/uri/uri_test.go b/pkg/detectors/uri/uri_test.go index c924a60d5cca..012b5d84621e 100644 --- a/pkg/detectors/uri/uri_test.go +++ b/pkg/detectors/uri/uri_test.go @@ -7,23 +7,13 @@ import ( "context" "fmt" "testing" - "time" "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) func TestURI_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("URI_SECRET") - secretInactive := testSecrets.MustGetField("URI_INACTIVE") type args struct { ctx context.Context data []byte @@ -36,20 +26,19 @@ func TestURI_FromChunk(t *testing.T) { want []detectors.Result wantErr bool }{ - { name: "found, unverified, wrong username", s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a uri secret %s within", secretInactive)), + data: []byte(fmt.Sprintf("You can find a uri secret %s within", "https://user:pass@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx")), verify: true, }, want: []detectors.Result{ { DetectorType: detectorspb.DetectorType_URI, Verified: false, - Redacted: "https://user:**********@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx", + Redacted: "https://user:****@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx", }, }, wantErr: false, @@ -59,20 +48,37 @@ func TestURI_FromChunk(t *testing.T) { s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a uri secret %s within", secret)), + data: []byte(fmt.Sprintf("You can find a uri secret %s within", "https://httpwatch:pass@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx")), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_URI, + Verified: true, + Redacted: "https://httpwatch:****@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx", + }, + }, + wantErr: false, + }, + { + name: "found, verified, defused", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a uri secret %s within", "https://httpwatch:pass@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx?foo=bar")), verify: true, }, want: []detectors.Result{ { DetectorType: detectorspb.DetectorType_URI, Verified: true, - Redacted: "https://httpwatch:**********@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx", + Redacted: "https://httpwatch:****@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx", }, }, wantErr: false, }, { - name: "bad proto", + name: "bad scheme", s: Scanner{}, args: args{ ctx: context.Background(), diff --git a/pkg/engine/defaults.go b/pkg/engine/defaults.go index b5b8d10c022a..1ea37ba4c88f 100644 --- a/pkg/engine/defaults.go +++ b/pkg/engine/defaults.go @@ -255,6 +255,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/freshbooks" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/freshdesk" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/front" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ftp" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/fulcrum" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/fullstory" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/fusebill" @@ -510,6 +511,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/reallysimplesystems" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/rebrandly" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/rechargepayments" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/redis" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/refiner" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/rentman" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/repairshopr" @@ -1484,5 +1486,7 @@ func DefaultDetectors() []detectors.Detector { npmtoken.Scanner{}, npmtokenv2.Scanner{}, sqlserver.Scanner{}, + redis.Scanner{}, + ftp.Scanner{}, } } diff --git a/pkg/pb/detectorspb/detectors.pb.go b/pkg/pb/detectorspb/detectors.pb.go index 108890eb9139..724bb8dd76a5 100644 --- a/pkg/pb/detectorspb/detectors.pb.go +++ b/pkg/pb/detectorspb/detectors.pb.go @@ -967,6 +967,8 @@ const ( DetectorType_NGC DetectorType = 896 DetectorType_DigitalOceanV2 DetectorType = 897 DetectorType_SQLServer DetectorType = 898 + DetectorType_FTP DetectorType = 899 + DetectorType_Redis DetectorType = 900 ) // Enum value maps for DetectorType. @@ -1867,6 +1869,8 @@ var ( 896: "NGC", 897: "DigitalOceanV2", 898: "SQLServer", + 899: "FTP", + 900: "Redis", } DetectorType_value = map[string]int32{ "Alibaba": 0, @@ -2764,6 +2768,8 @@ var ( "NGC": 896, "DigitalOceanV2": 897, "SQLServer": 898, + "FTP": 899, + "Redis": 900, } ) @@ -3128,7 +3134,7 @@ var file_detectors_proto_rawDesc = []byte{ 0x0a, 0x0b, 0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49, 0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, - 0x02, 0x2a, 0xc1, 0x70, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, + 0x02, 0x2a, 0xd7, 0x70, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x10, 0x03, 0x12, 0x0a, 0x0a, @@ -4028,11 +4034,13 @@ var file_detectors_proto_rawDesc = []byte{ 0x6f, 0x44, 0x42, 0x10, 0xff, 0x06, 0x12, 0x08, 0x0a, 0x03, 0x4e, 0x47, 0x43, 0x10, 0x80, 0x07, 0x12, 0x13, 0x0a, 0x0e, 0x44, 0x69, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x4f, 0x63, 0x65, 0x61, 0x6e, 0x56, 0x32, 0x10, 0x81, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x53, 0x51, 0x4c, 0x53, 0x65, 0x72, 0x76, - 0x65, 0x72, 0x10, 0x82, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, - 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, - 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, - 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, - 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x72, 0x10, 0x82, 0x07, 0x12, 0x08, 0x0a, 0x03, 0x46, 0x54, 0x50, 0x10, 0x83, 0x07, 0x12, + 0x0a, 0x0a, 0x05, 0x52, 0x65, 0x64, 0x69, 0x73, 0x10, 0x84, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, + 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, + 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, + 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( diff --git a/proto/detectors.proto b/proto/detectors.proto index e7e809165f62..289ea950a58c 100644 --- a/proto/detectors.proto +++ b/proto/detectors.proto @@ -906,6 +906,8 @@ enum DetectorType { NGC = 896; DigitalOceanV2 = 897; SQLServer = 898; + FTP = 899; + Redis = 900; } message Result {