experimental glob support (#9646)

silent/tests/testdata/vu-glob.txt

@@ -0,0 +1,77 @@
+dependabot update -f input.yml --local . --updater-image ghcr.io/dependabot/dependabot-updater-silent
+stdout -count=4 create_pull_request
+pr-created expected-0.json
+pr-created frontend/expected-1.json
+pr-created frontend/expected-2.json
+pr-created backend/expected-3.json
+
+# Testing glob configuration without a group.
+
+-- manifest.json --
+{
+  "dependency-a": { "version": "1.2.3" }
+}
+
+-- expected-0.json --
+{
+  "dependency-a": { "version": "1.2.5" }
+}
+
+-- frontend/manifest.json --
+{
+  "dependency-a": { "version": "1.2.3" },
+  "dependency-b": { "version": "1.2.3" }
+}
+
+-- frontend/expected-1.json --
+{
+  "dependency-a": { "version": "1.2.5" },
+  "dependency-b": { "version": "1.2.3" }
+}
+
+-- frontend/expected-2.json --
+{
+  "dependency-a": { "version": "1.2.3" },
+  "dependency-b": { "version": "1.2.5" }
+}
+
+-- backend/manifest.json --
+{
+  "dependency-a": { "version": "1.2.3" }
+}
+
+-- backend/expected-3.json --
+{
+  "dependency-a": { "version": "1.2.5" }
+}
+
+-- dependency-a --
+{
+  "versions": [
+    "1.2.3",
+    "1.2.4",
+    "1.2.5"
+  ]
+}
+
+-- dependency-b --
+{
+  "versions": [
+    "1.2.3",
+    "1.2.4",
+    "1.2.5"
+  ]
+}
+
+-- input.yml --
+job:
+  package-manager: "silent"
+  source:
+    directories:
+      - "**/*"
+    provider: example
+    hostname: example.com
+    api-endpoint: https://example.com/api/v3
+    repo: dependabot/smoke-tests
+  experiments:
+    globs: true

updater/lib/dependabot/dependency_snapshot.rb

@@ -28,6 +28,11 @@ def self.create_from_job_definition(job:, job_definition:)
         file
       end
 
+      if Dependabot::Experiments.enabled?(:globs) && job.source.directories
+        # The job.source.directory may contain globs, so we use the directories from the fetched files
+        job.source.directories = decoded_dependency_files.flat_map(&:directory).uniq
+      end
+
       new(
         job: job,
         base_commit_sha: job_definition.fetch("base_commit_sha"),

updater/lib/dependabot/file_fetcher_command.rb

@@ -99,8 +99,11 @@ def file_fetcher_for_directory(directory)
       @file_fetchers[directory] ||= create_file_fetcher(directory: directory)
     end
 
-    # Fetch dependency files for multiple directories
     def dependency_files_for_multi_directories
+      if Dependabot::Experiments.enabled?(:globs)
+        return @dependency_files_for_multi_directories ||= dependency_files_for_globs
+      end
+
       @dependency_files_for_multi_directories ||= job.source.directories.flat_map do |dir|
         ff = with_retries { file_fetcher_for_directory(dir) }
         files = ff.files
@@ -109,6 +112,35 @@ def dependency_files_for_multi_directories
       end
     end
 
+    def dependency_files_for_globs
+      has_glob = T.let(false, T::Boolean)
+      directories = Dir.chdir(job.repo_contents_path) do
+        job.source.directories.map do |dir|
+          next dir unless glob?(dir)
+
+          has_glob = true
+          dir = dir.delete_prefix("/")
+          Dir.glob(dir, File::FNM_DOTMATCH).select { |d| File.directory?(d) }
+        end.flatten
+      end
+
+      directories.flat_map do |dir|
+        ff = with_retries { file_fetcher_for_directory(dir) }
+
+        begin
+          files = ff.files
+        rescue Dependabot::DependencyFileNotFound
+          # skip directories that don't contain manifests if globbing is used
+          next if has_glob
+
+          raise
+        end
+
+        post_ecosystem_versions(ff) if should_record_ecosystem_versions?
+        files
+      end.compact
+    end
+
     def dependency_files
       return @dependency_files if defined?(@dependency_files)
 
@@ -251,5 +283,10 @@ def github_connectivity_client(job)
         }
       })
     end
+
+    def glob?(directory)
+      # We could tighten this up, but it's probably close enough.
+      directory.include?("*") || directory.include?("?") || (directory.include?("[") && directory.include?("]"))
+    end
   end
 end