From 7de9bdd12daaf8057aefdd8cd8609bc0eb381fb9 Mon Sep 17 00:00:00 2001 From: Dustin Decker Date: Fri, 9 Dec 2022 12:10:42 -0800 Subject: [PATCH] Support globbing with ignore repos (#967) --- go.mod | 1 + go.sum | 2 ++ pkg/sources/github/github.go | 15 +++++++++++---- pkg/sources/github/github_test.go | 2 +- pkg/sources/gitlab/gitlab.go | 20 ++++++++++++++++++-- pkg/sources/gitlab/gitlab_test.go | 19 ++++++++++++++++++- 6 files changed, 51 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 849b45e9127a..f5874867a405 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( github.com/go-logr/zapr v1.2.3 github.com/go-redis/redis v6.15.9+incompatible github.com/go-sql-driver/mysql v1.7.0 + github.com/gobwas/glob v0.2.3 github.com/golang-jwt/jwt v3.2.2+incompatible github.com/google/go-cmp v0.5.9 github.com/google/go-github/v42 v42.0.0 diff --git a/go.sum b/go.sum index cac12f281d23..c083bb269829 100644 --- a/go.sum +++ b/go.sum @@ -176,6 +176,8 @@ github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8w github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 18cbea9abef6..b3123996d600 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -17,9 +17,9 @@ import ( "github.com/bradleyfalzon/ghinstallation/v2" "github.com/go-errors/errors" gogit "github.com/go-git/go-git/v5" + "github.com/gobwas/glob" "github.com/google/go-github/v42/github" log "github.com/sirupsen/logrus" - "golang.org/x/exp/slices" "golang.org/x/oauth2" "golang.org/x/sync/errgroup" "google.golang.org/protobuf/proto" @@ -745,9 +745,16 @@ func (s *Source) getReposByUser(ctx context.Context, user string) ([]string, err } func (s *Source) ignoreRepo(r string) bool { - if slices.Contains(s.ignoreRepos, r) { - s.log.Debugf("ignoring repo %s", r) - return true + for _, ignore := range s.ignoreRepos { + g, err := glob.Compile(ignore) + if err != nil { + s.log.WithError(err).Errorf("could not compile ignore repo glob %s", ignore) + continue + } + if g.Match(r) { + s.log.Debugf("ignoring repo %s", r) + return true + } } return false } diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go index a337cb98b4a2..5a9124ad056c 100644 --- a/pkg/sources/github/github_test.go +++ b/pkg/sources/github/github_test.go @@ -71,7 +71,7 @@ func TestAddReposByOrg(t *testing.T) { }) s := initTestSource(nil) - s.ignoreRepos = []string{"secret/super-secret-repo2"} + s.ignoreRepos = []string{"secret/super-*-repo2"} // gock works here because github.NewClient is using the default HTTP Transport err := s.addRepos(context.TODO(), "super-secret-org", s.getReposByOrg) assert.Nil(t, err) diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.go index 33fa410cd439..74a83602ab5a 100644 --- a/pkg/sources/gitlab/gitlab.go +++ b/pkg/sources/gitlab/gitlab.go @@ -20,6 +20,7 @@ import ( "github.com/go-errors/errors" gogit "github.com/go-git/go-git/v5" + "github.com/gobwas/glob" log "github.com/sirupsen/logrus" "github.com/xanzy/go-gitlab" "golang.org/x/exp/slices" @@ -344,6 +345,21 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) return errs } +func (s *Source) ignoreRepo(r string) bool { + for _, ignore := range s.ignoreRepos { + g, err := glob.Compile(ignore) + if err != nil { + log.WithError(err).Errorf("could not compile ignore repo glob %s", ignore) + continue + } + if g.Match(r) { + log.Debugf("Ignoring repo %s", r) + return true + } + } + return false +} + // Chunks emits chunks of bytes over a channel. func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error { // Start client. @@ -370,10 +386,10 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err } // Turn projects into URLs for Git cloner. for _, prj := range projects { - if slices.Contains(s.ignoreRepos, prj.PathWithNamespace) { - log.Debugf("Ignoring repo %s", prj.PathWithNamespace) + if s.ignoreRepo(prj.PathWithNamespace) { continue } + // Ensure the urls are valid before adding them to the repo list. _, err := url.Parse(prj.HTTPURLToRepo) if err != nil { diff --git a/pkg/sources/gitlab/gitlab_test.go b/pkg/sources/gitlab/gitlab_test.go index 265ebbab5e88..500fc4ac5c82 100644 --- a/pkg/sources/gitlab/gitlab_test.go +++ b/pkg/sources/gitlab/gitlab_test.go @@ -46,7 +46,7 @@ func TestSource_Scan(t *testing.T) { wantErr bool }{ { - name: "token auth, enumerate repo", + name: "token auth, enumerate repo, with explicit ignore", init: init{ name: "test source", connection: &sourcespb.GitLab{ @@ -62,6 +62,23 @@ func TestSource_Scan(t *testing.T) { }, wantReposScanned: 2, }, + { + name: "token auth, enumerate repo, with glob ignore", + init: init{ + name: "test source", + connection: &sourcespb.GitLab{ + Credential: &sourcespb.GitLab_Token{ + Token: token, + }, + IgnoreRepos: []string{"tes1188/*-gitlab"}, + }, + }, + wantChunk: &sources.Chunk{ + SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB, + SourceName: "test source", + }, + wantReposScanned: 2, + }, { name: "token auth, scoped repo", init: init{