Support globbing with ignore repos (#967)

go.mod

@@ -30,6 +30,7 @@ require (
 	github.com/go-logr/zapr v1.2.3
 	github.com/go-redis/redis v6.15.9+incompatible
 	github.com/go-sql-driver/mysql v1.7.0
+	github.com/gobwas/glob v0.2.3
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/google/go-cmp v0.5.9
 	github.com/google/go-github/v42 v42.0.0

go.sum

@@ -176,6 +176,8 @@ github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8w
 github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
 github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
 github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
 github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=

pkg/sources/github/github.go

@@ -17,9 +17,9 @@ import (
 	"github.com/bradleyfalzon/ghinstallation/v2"
 	"github.com/go-errors/errors"
 	gogit "github.com/go-git/go-git/v5"
+	"github.com/gobwas/glob"
 	"github.com/google/go-github/v42/github"
 	log "github.com/sirupsen/logrus"
-	"golang.org/x/exp/slices"
 	"golang.org/x/oauth2"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/protobuf/proto"
@@ -745,9 +745,16 @@ func (s *Source) getReposByUser(ctx context.Context, user string) ([]string, err
 }
 
 func (s *Source) ignoreRepo(r string) bool {
-	if slices.Contains(s.ignoreRepos, r) {
-		s.log.Debugf("ignoring repo %s", r)
-		return true
+	for _, ignore := range s.ignoreRepos {
+		g, err := glob.Compile(ignore)
+		if err != nil {
+			s.log.WithError(err).Errorf("could not compile ignore repo glob %s", ignore)
+			continue
+		}
+		if g.Match(r) {
+			s.log.Debugf("ignoring repo %s", r)
+			return true
+		}
 	}
 	return false
 }

pkg/sources/github/github_test.go

@@ -71,7 +71,7 @@ func TestAddReposByOrg(t *testing.T) {
 		})
 
 	s := initTestSource(nil)
-	s.ignoreRepos = []string{"secret/super-secret-repo2"}
+	s.ignoreRepos = []string{"secret/super-*-repo2"}
 	// gock works here because github.NewClient is using the default HTTP Transport
 	err := s.addRepos(context.TODO(), "super-secret-org", s.getReposByOrg)
 	assert.Nil(t, err)

pkg/sources/gitlab/gitlab.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/go-errors/errors"
 	gogit "github.com/go-git/go-git/v5"
+	"github.com/gobwas/glob"
 	log "github.com/sirupsen/logrus"
 	"github.com/xanzy/go-gitlab"
 	"golang.org/x/exp/slices"
@@ -344,6 +345,21 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
 	return errs
 }
 
+func (s *Source) ignoreRepo(r string) bool {
+	for _, ignore := range s.ignoreRepos {
+		g, err := glob.Compile(ignore)
+		if err != nil {
+			log.WithError(err).Errorf("could not compile ignore repo glob %s", ignore)
+			continue
+		}
+		if g.Match(r) {
+			log.Debugf("Ignoring repo %s", r)
+			return true
+		}
+	}
+	return false
+}
+
 // Chunks emits chunks of bytes over a channel.
 func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
 	// Start client.
@@ -370,10 +386,10 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 		}
 		// Turn projects into URLs for Git cloner.
 		for _, prj := range projects {
-			if slices.Contains(s.ignoreRepos, prj.PathWithNamespace) {
-				log.Debugf("Ignoring repo %s", prj.PathWithNamespace)
+			if s.ignoreRepo(prj.PathWithNamespace) {
 				continue
 			}
+
 			// Ensure the urls are valid before adding them to the repo list.
 			_, err := url.Parse(prj.HTTPURLToRepo)
 			if err != nil {

pkg/sources/gitlab/gitlab_test.go

@@ -46,7 +46,7 @@ func TestSource_Scan(t *testing.T) {
 		wantErr          bool
 	}{
 		{
-			name: "token auth, enumerate repo",
+			name: "token auth, enumerate repo, with explicit ignore",
 			init: init{
 				name: "test source",
 				connection: &sourcespb.GitLab{
@@ -62,6 +62,23 @@ func TestSource_Scan(t *testing.T) {
 			},
 			wantReposScanned: 2,
 		},
+		{
+			name: "token auth, enumerate repo, with glob ignore",
+			init: init{
+				name: "test source",
+				connection: &sourcespb.GitLab{
+					Credential: &sourcespb.GitLab_Token{
+						Token: token,
+					},
+					IgnoreRepos: []string{"tes1188/*-gitlab"},
+				},
+			},
+			wantChunk: &sources.Chunk{
+				SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,
+				SourceName: "test source",
+			},
+			wantReposScanned: 2,
+		},
 		{
 			name: "token auth, scoped repo",
 			init: init{