feat: move file patterns to a global level to be able to use it on an…

…y analyzer (#2539)

docs/docs/misconfiguration/options/others.md

@@ -2,21 +2,3 @@
 
 !!! hint
     See also [Others](../../vulnerability/examples/others.md) in Vulnerability section.
-
-## File patterns
-When a directory is given as an input, Trivy will recursively look for and test all files based on file patterns.
-The default file patterns are [here](../custom/index.md).
-
-In addition to the default file patterns, the `--file-patterns` option takes regexp patterns to look for your files.
-For example, it may be useful when your file name of Dockerfile doesn't match the default patterns.
-
-This can be repeated for specifying multiple file patterns.
-Allowed values are here:
-
-- dockerfile
-- yaml
-- json
-- toml
-- hcl
-
-For more details, see [an example](https://github.com/aquasecurity/trivy/tree/{{ git.commit }}/examples/misconf/file-patterns)

docs/docs/references/customization/config-file.md

@@ -82,6 +82,11 @@ Available in client/server mode
 
 ```yaml
 scan:
+  # Same as '--file-patterns'
+  # Default is empty
+  file-patterns:
+    -
+
   # Same as '--skip-dirs'
   # Default is empty
   skip-dirs:
@@ -195,11 +200,6 @@ Available with misconfiguration scanning
 
 ```yaml
 misconfiguration:
-  # Same as '--file-patterns'
-  # Default is empty
-  file-patterns:
-    -
-
   # Same as '--include-non-failures'
   # Default is false
   include-non-failures: false

docs/docs/vulnerability/examples/others.md

@@ -16,6 +16,22 @@ If your image contains lock files which are not maintained by you, you can skip
 $ trivy image --skip-dirs /var/lib/gems/2.5.0/gems/fluent-plugin-detect-exceptions-0.0.13 --skip-dirs "/var/lib/gems/2.5.0/gems/http_parser.rb-0.6.0" quay.io/fluentd_elasticsearch/fluentd:v2.9.0
 ```
 
+## File patterns
+When a directory is given as an input, Trivy will recursively look for and test all files based on file patterns.
+The default file patterns are [here](../custom/index.md).
+
+In addition to the default file patterns, the `--file-patterns` option takes regexp patterns to look for your files.
+For example, it may be useful when your file name of Dockerfile doesn't match the default patterns.
+
+This can be repeated for specifying multiple file patterns.
+
+A file pattern contains the analyzer it is used for, and the pattern itself, joined by a semicolon. For example:
+```
+--file-patterns "dockerfile:.*.docker" --file-patterns "yaml:deployment" --file-patterns "pip:requirements-.*\.txt"
+```
+
+For more details, see [an example](https://github.com/aquasecurity/trivy/tree/{{ git.commit }}/examples/misconf/file-patterns)
+
 ## Exit Code
 By default, `Trivy` exits with code 0 even when vulnerabilities are detected.
 Use the `--exit-code` option if you want to exit with a non-zero exit code.

pkg/commands/artifact/run.go

@@ -448,6 +448,7 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi
 		ScanRemovedPackages: opts.ScanRemovedPkgs, // this is valid only for 'image' subcommand
 		ListAllPackages:     opts.ListAllPkgs,
 		LicenseCategories:   opts.LicenseCategories,
+		FilePatterns:        opts.FilePatterns,
 	}
 
 	if slices.Contains(opts.SecurityChecks, types.SecurityCheckVulnerability) {
@@ -464,7 +465,6 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi
 			Namespaces:       append(opts.PolicyNamespaces, defaultPolicyNamespaces...),
 			PolicyPaths:      opts.PolicyPaths,
 			DataPaths:        opts.DataPaths,
-			FilePatterns:     opts.FilePatterns,
 			HelmValues:       opts.HelmValues,
 			HelmValueFiles:   opts.HelmValueFiles,
 			HelmFileValues:   opts.HelmFileValues,
@@ -504,6 +504,7 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi
 			DisabledAnalyzers: disabledAnalyzers(opts),
 			SkipFiles:         opts.SkipFiles,
 			SkipDirs:          opts.SkipDirs,
+			FilePatterns:      opts.FilePatterns,
 			InsecureSkipTLS:   opts.Insecure,
 			Offline:           opts.OfflineScan,
 			NoProgress:        opts.NoProgress || opts.Quiet,

pkg/fanal/analyzer/all/import.go

@@ -3,6 +3,7 @@ package all
 import (
 	_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/buildinfo"
 	_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/command/apk"
+	_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/config/all"
 	_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language/dotnet/deps"
 	_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language/dotnet/nuget"
 	_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language/golang/binary"

pkg/fanal/analyzer/analyzer.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"io/fs"
 	"os"
+	"regexp"
 	"sort"
 	"strings"
 	"sync"
@@ -91,6 +92,7 @@ type Opener func() (dio.ReadSeekCloserAt, error)
 type AnalyzerGroup struct {
 	analyzers       []analyzer
 	configAnalyzers []configAnalyzer
+	filePatterns    map[Type][]*regexp.Regexp
 }
 
 type AnalysisResult struct {
@@ -266,12 +268,36 @@ func belongToGroup(groupName Group, analyzerType Type, disabledAnalyzers []Type,
 	return true
 }
 
-func NewAnalyzerGroup(groupName Group, disabledAnalyzers []Type) AnalyzerGroup {
+const separator = ":"
+
+func NewAnalyzerGroup(groupName Group, disabledAnalyzers []Type, filePatterns []string) (AnalyzerGroup, error) {
 	if groupName == "" {
 		groupName = GroupBuiltin
 	}
 
-	var group AnalyzerGroup
+	group := AnalyzerGroup{
+		filePatterns: map[Type][]*regexp.Regexp{},
+	}
+	for _, p := range filePatterns {
+		// e.g. "dockerfile:my_dockerfile_*"
+		s := strings.SplitN(p, separator, 2)
+		if len(s) != 2 {
+			return group, xerrors.Errorf("invalid file pattern (%s)", p)
+		}
+
+		fileType, pattern := s[0], s[1]
+		r, err := regexp.Compile(pattern)
+		if err != nil {
+			return group, xerrors.Errorf("invalid file regexp (%s): %w", p, err)
+		}
+
+		if _, ok := group.filePatterns[Type(fileType)]; !ok {
+			group.filePatterns[Type(fileType)] = []*regexp.Regexp{}
+		}
+
+		group.filePatterns[Type(fileType)] = append(group.filePatterns[Type(fileType)], r)
+	}
+
 	for analyzerType, a := range analyzers {
 		if !belongToGroup(groupName, analyzerType, disabledAnalyzers, a) {
 			continue
@@ -286,7 +312,7 @@ func NewAnalyzerGroup(groupName Group, disabledAnalyzers []Type) AnalyzerGroup {
 		group.configAnalyzers = append(group.configAnalyzers, a)
 	}
 
-	return group
+	return group, nil
 }
 
 // AnalyzerVersions returns analyzer version identifier used for cache keys.
@@ -313,14 +339,16 @@ func (ag AnalyzerGroup) AnalyzeFile(ctx context.Context, wg *sync.WaitGroup, lim
 		return nil
 	}
 
+	// filepath extracted from tar file doesn't have the prefix "/"
+	cleanPath := strings.TrimLeft(filePath, "/")
+
 	for _, a := range ag.analyzers {
 		// Skip disabled analyzers
 		if slices.Contains(disabled, a.Type()) {
 			continue
 		}
 
-		// filepath extracted from tar file doesn't have the prefix "/"
-		if !a.Required(strings.TrimLeft(filePath, "/"), info) {
+		if !ag.filePatternMatch(a.Type(), cleanPath) && !a.Required(cleanPath, info) {
 			continue
 		}
 		rc, err := opener()
@@ -375,3 +403,12 @@ func (ag AnalyzerGroup) AnalyzeImageConfig(targetOS types.OS, configBlob []byte)
 	}
 	return nil
 }
+
+func (ag AnalyzerGroup) filePatternMatch(analyzerType Type, filePath string) bool {
+	for _, pattern := range ag.filePatterns[analyzerType] {
+		if pattern.MatchString(filePath) {
+			return true
+		}
+	}
+	return false
+}

pkg/fanal/analyzer/analyzer_test.go

@@ -281,6 +281,7 @@ func TestAnalyzeFile(t *testing.T) {
 		filePath          string
 		testFilePath      string
 		disabledAnalyzers []analyzer.Type
+		filePatterns      []string
 	}
 	tests := []struct {
 		name    string
@@ -377,6 +378,28 @@ func TestAnalyzeFile(t *testing.T) {
 			},
 			want: &analyzer.AnalysisResult{},
 		},
+		{
+			name: "happy path with library analyzer file pattern regex",
+			args: args{
+				filePath:     "/app/Gemfile-dev.lock",
+				testFilePath: "testdata/app/Gemfile.lock",
+				filePatterns: []string{"bundler:Gemfile(-.*)?\\.lock"},
+			},
+			want: &analyzer.AnalysisResult{
+				Applications: []types.Application{
+					{
+						Type:     "bundler",
+						FilePath: "/app/Gemfile-dev.lock",
+						Libraries: []types.Package{
+							{
+								Name:    "actioncable",
+								Version: "5.2.3",
+							},
+						},
+					},
+				},
+			},
+		},
 		{
 			name: "ignore permission error",
 			args: args{
@@ -393,14 +416,38 @@ func TestAnalyzeFile(t *testing.T) {
 			},
 			wantErr: "unable to open /lib/apk/db/installed",
 		},
+		{
+			name: "sad path with broken file pattern regex",
+			args: args{
+				filePath:     "/app/Gemfile-dev.lock",
+				testFilePath: "testdata/app/Gemfile.lock",
+				filePatterns: []string{"bundler:Gemfile(-.*?\\.lock"},
+			},
+			wantErr: "error parsing regexp",
+		},
+		{
+			name: "sad path with broken file pattern",
+			args: args{
+				filePath:     "/app/Gemfile-dev.lock",
+				testFilePath: "testdata/app/Gemfile.lock",
+				filePatterns: []string{"Gemfile(-.*)?\\.lock"},
+			},
+			wantErr: "invalid file pattern",
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			var wg sync.WaitGroup
 			limit := semaphore.NewWeighted(3)
 
 			got := new(analyzer.AnalysisResult)
-			a := analyzer.NewAnalyzerGroup(analyzer.GroupBuiltin, tt.args.disabledAnalyzers)
+			a, err := analyzer.NewAnalyzerGroup(analyzer.GroupBuiltin, tt.args.disabledAnalyzers, tt.args.filePatterns)
+			if err != nil && tt.wantErr != "" {
+				require.NotNil(t, err)
+				assert.Contains(t, err.Error(), tt.wantErr)
+				return
+			}
+			require.NoError(t, err)
 
 			info, err := os.Stat(tt.args.testFilePath)
 			require.NoError(t, err)
@@ -440,6 +487,7 @@ func TestAnalyzeConfig(t *testing.T) {
 		targetOS          types.OS
 		configBlob        []byte
 		disabledAnalyzers []analyzer.Type
+		filePatterns      []string
 	}
 	tests := []struct {
 		name string
@@ -482,7 +530,8 @@ func TestAnalyzeConfig(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			a := analyzer.NewAnalyzerGroup(analyzer.GroupBuiltin, tt.args.disabledAnalyzers)
+			a, err := analyzer.NewAnalyzerGroup(analyzer.GroupBuiltin, tt.args.disabledAnalyzers, tt.args.filePatterns)
+			require.NoError(t, err)
 			got := a.AnalyzeImageConfig(tt.args.targetOS, tt.args.configBlob)
 			assert.Equal(t, tt.want, got)
 		})
@@ -517,7 +566,8 @@ func TestAnalyzer_AnalyzerVersions(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			a := analyzer.NewAnalyzerGroup(analyzer.GroupBuiltin, tt.disabled)
+			a, err := analyzer.NewAnalyzerGroup(analyzer.GroupBuiltin, tt.disabled, nil)
+			require.NoError(t, err)
 			got := a.AnalyzerVersions()
 			fmt.Printf("%v\n", got)
 			assert.Equal(t, tt.want, got)
@@ -549,7 +599,8 @@ func TestAnalyzer_ImageConfigAnalyzerVersions(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			a := analyzer.NewAnalyzerGroup(analyzer.GroupBuiltin, tt.disabled)
+			a, err := analyzer.NewAnalyzerGroup(analyzer.GroupBuiltin, tt.disabled, nil)
+			require.NoError(t, err)
 			got := a.ImageConfigAnalyzerVersions()
 			assert.Equal(t, tt.want, got)
 		})

pkg/fanal/analyzer/config/all/import.go

@@ -0,0 +1,9 @@
+package all
+
+import (
+	_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/config/dockerfile"
+	_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/config/helm"
+	_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/config/json"
+	_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/config/terraform"
+	_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/config/yaml"
+)

pkg/fanal/analyzer/config/config.go

@@ -1,29 +1,13 @@
 package config
 
 import (
-	"regexp"
 	"sort"
-	"strings"
-
-	"golang.org/x/xerrors"
-
-	"github.com/aquasecurity/trivy/pkg/fanal/analyzer/config/helm"
-
-	"github.com/aquasecurity/trivy/pkg/fanal/analyzer"
-	"github.com/aquasecurity/trivy/pkg/fanal/analyzer/config/dockerfile"
-	"github.com/aquasecurity/trivy/pkg/fanal/analyzer/config/json"
-	"github.com/aquasecurity/trivy/pkg/fanal/analyzer/config/terraform"
-	"github.com/aquasecurity/trivy/pkg/fanal/analyzer/config/yaml"
-	"github.com/aquasecurity/trivy/pkg/fanal/types"
 )
 
-const separator = ":"
-
 type ScannerOption struct {
 	Trace                   bool
 	RegoOnly                bool
 	Namespaces              []string
-	FilePatterns            []string
 	PolicyPaths             []string
 	DataPaths               []string
 	DisableEmbeddedPolicies bool
@@ -37,44 +21,6 @@ type ScannerOption struct {
 
 func (o *ScannerOption) Sort() {
 	sort.Strings(o.Namespaces)
-	sort.Strings(o.FilePatterns)
 	sort.Strings(o.PolicyPaths)
 	sort.Strings(o.DataPaths)
 }
-
-func RegisterConfigAnalyzers(filePatterns []string) error {
-	var dockerRegexp, jsonRegexp, yamlRegexp, helmRegexp *regexp.Regexp
-	for _, p := range filePatterns {
-		// e.g. "dockerfile:my_dockerfile_*"
-		s := strings.SplitN(p, separator, 2)
-		if len(s) != 2 {
-			return xerrors.Errorf("invalid file pattern (%s)", p)
-		}
-		fileType, pattern := s[0], s[1]
-		r, err := regexp.Compile(pattern)
-		if err != nil {
-			return xerrors.Errorf("invalid file regexp (%s): %w", p, err)
-		}
-
-		switch fileType {
-		case types.Dockerfile:
-			dockerRegexp = r
-		case types.JSON:
-			jsonRegexp = r
-		case types.YAML:
-			yamlRegexp = r
-		case types.Helm:
-			helmRegexp = r
-		default:
-			return xerrors.Errorf("unknown file type: %s, pattern: %s", fileType, pattern)
-		}
-	}
-
-	analyzer.RegisterAnalyzer(dockerfile.NewConfigAnalyzer(dockerRegexp))
-	analyzer.RegisterAnalyzer(terraform.NewConfigAnalyzer())
-	analyzer.RegisterAnalyzer(json.NewConfigAnalyzer(jsonRegexp))
-	analyzer.RegisterAnalyzer(yaml.NewConfigAnalyzer(yamlRegexp))
-	analyzer.RegisterAnalyzer(helm.NewConfigAnalyzer(helmRegexp))
-
-	return nil
-}