[Fix #8646] Optimize finding target files further (#8815)

After sleeping on it, I realized there was a simpler (less code) and slightly faster version of the previous #8806.

By _recursively_ getting all directories and then at each directory level testing Rubocop Exclude on directories.  Once directories are found at all levels, we search for files.

In addition, I replaced many of the `base_dir` string concatenations with the safer `File#join`.

Benchmarks on a large project removes an additional 500+ milliseconds.

lib/rubocop/target_finder.rb

@@ -82,48 +82,33 @@ def to_inspect?(file, hidden_files, base_dir_config)
     # normal way (dir/**/*).
     def find_files(base_dir, flags)
       # get all wanted directories first to improve speed of finding all files
-      patterns = wanted_dir_patterns(base_dir, flags)
+      exclude_pattern = combined_exclude_glob_patterns(base_dir)
+      dir_flags = flags | File::FNM_PATHNAME | File::FNM_EXTGLOB
+      patterns = wanted_dir_patterns(base_dir, exclude_pattern, dir_flags)
+      patterns.map! { |dir| File.join(dir, '*') }
       # We need this special case to avoid creating the pattern
       # /**/* which searches the entire file system.
-      patterns = ["#{base_dir}/**/*"] if patterns.empty?
+      patterns = [File.join(dir, '**/*')] if patterns.empty?
 
       Dir.glob(patterns, flags).select { |path| FileTest.file?(path) }
     end
 
-    def wanted_dir_patterns(base_dir, flags)
-      exclude_pattern = combined_exclude_glob_patterns(base_dir)
-      flags = flags | File::FNM_PATHNAME | File::FNM_EXTGLOB | File::FNM_DOTMATCH
-      wanted_toplevel_dirs = toplevel_dirs(base_dir, flags) -
-                             excluded_dirs(base_dir)
-      wanted_toplevel_dirs.map! { |dir| dir << '/**/' }
-
-      Dir.glob(wanted_toplevel_dirs, flags)
-         .map { |dir| dir << '*' } # add file glob pattern to end of each dir
-         .reject { |dir| File.fnmatch?(exclude_pattern, dir, flags) }
-         .unshift("#{base_dir}/*")
+    def wanted_dir_patterns(base_dir, exclude_pattern, flags)
+      dirs = Dir.glob(File.join(base_dir, '*/'), flags)
+                .reject do |dir|
+                  dir.end_with?('/./', '/../') || File.fnmatch?(exclude_pattern, dir, flags)
+                end
+      dirs.flat_map { |dir| wanted_dir_patterns(dir, exclude_pattern, flags) }
+          .unshift(base_dir)
     end
 
     def combined_exclude_glob_patterns(base_dir)
-      all_cops_config = @config_store.for(base_dir).for_all_cops
-      patterns = all_cops_config['Exclude'].select { |pattern| pattern.is_a? String }
-                                           .map { |pattern| pattern.sub("#{base_dir}/", '') }
+      exclude = @config_store.for(base_dir).for_all_cops['Exclude']
+      patterns = exclude.select { |pattern| pattern.is_a?(String) && pattern.end_with?('/**/*') }
+                        .map { |pattern| pattern.sub("#{base_dir}/", '') }
       "#{base_dir}/{#{patterns.join(',')}}"
     end
 
-    def toplevel_dirs(base_dir, flags)
-      Dir.glob(File.join(base_dir, '*'), flags).select do |dir|
-        File.directory?(dir) && !dir.end_with?('/.', '/..')
-      end
-    end
-
-    def excluded_dirs(base_dir)
-      all_cops_config = @config_store.for(base_dir).for_all_cops
-      dir_tree_excludes = all_cops_config['Exclude'].select do |pattern|
-        pattern.is_a?(String) && pattern.end_with?('/**/*')
-      end
-      dir_tree_excludes.map { |pattern| pattern.sub(%r{/\*\*/\*$}, '') }
-    end
-
     def ruby_extension?(file)
       ruby_extensions.include?(File.extname(file))
     end