Merge pull request #7161 from dvz/improve-project-files-discovery

Improve project files discovery performance
vimeo · Dec 21, 2021 · cc9111f · cc9111f
2 parents ea22f87 + 4034959
commit cc9111f
Show file tree

Hide file tree

Showing 3 changed files with 53 additions and 30 deletions.
diff --git a/src/Psalm/Internal/Analyzer/ProjectAnalyzer.php b/src/Psalm/Internal/Analyzer/ProjectAnalyzer.php
@@ -309,7 +309,11 @@ public function __construct(
         $file_extensions = $this->config->getFileExtensions();
 
         foreach ($this->config->getProjectDirectories() as $dir_name) {
-            $file_paths = $this->file_provider->getFilesInDir($dir_name, $file_extensions);
+            $file_paths = $this->file_provider->getFilesInDir(
+                $dir_name,
+                $file_extensions,
+                [$this->config, 'isInProjectDirs']
+            );
 
             foreach ($file_paths as $file_path) {
                 if ($this->config->isInProjectDirs($file_path)) {
@@ -319,7 +323,11 @@ public function __construct(
         }
 
         foreach ($this->config->getExtraDirectories() as $dir_name) {
-            $file_paths = $this->file_provider->getFilesInDir($dir_name, $file_extensions);
+            $file_paths = $this->file_provider->getFilesInDir(
+                $dir_name,
+                $file_extensions,
+                [$this->config, 'isInExtraDirs']
+            );
 
             foreach ($file_paths as $file_path) {
                 if ($this->config->isInExtraDirs($file_path)) {
@@ -611,11 +619,7 @@ public function check(string $base_dir, bool $is_diff = false): void
             && $this->project_cache_provider->canDiffFiles()
         ) {
             $deleted_files = $this->file_reference_provider->getDeletedReferencedFiles();
-            $diff_files = $deleted_files;
-
-            foreach ($this->config->getProjectDirectories() as $dir_name) {
-                $diff_files = array_merge($diff_files, $this->getDiffFilesInDir($dir_name, $this->config));
-            }
+            $diff_files = array_merge($deleted_files, $this->getDiffFiles());
         }
 
         $this->progress->write($this->generatePHPVersionMessage());
@@ -1053,8 +1057,13 @@ public function checkDir(string $dir_name): void
     private function checkDirWithConfig(string $dir_name, Config $config, bool $allow_non_project_files = false): void
     {
         $file_extensions = $config->getFileExtensions();
+        $directory_filter = $allow_non_project_files ? null : [$this->config, 'isInProjectDirs'];
 
-        $file_paths = $this->file_provider->getFilesInDir($dir_name, $file_extensions);
+        $file_paths = $this->file_provider->getFilesInDir(
+            $dir_name,
+            $file_extensions,
+            $directory_filter
+        );
 
         $files_to_scan = [];
 
@@ -1080,10 +1089,8 @@ public function addExtraFile(string $file_path): void
     /**
      * @return list<string>
      */
-    protected function getDiffFilesInDir(string $dir_name, Config $config): array
+    protected function getDiffFiles(): array
     {
-        $file_extensions = $config->getFileExtensions();
-
         if (!$this->parser_cache_provider || !$this->project_cache_provider) {
             throw new UnexpectedValueException('Parser cache provider cannot be null here');
         }
@@ -1092,16 +1099,12 @@ protected function getDiffFilesInDir(string $dir_name, Config $config): array
 
         $last_run = $this->project_cache_provider->getLastRun(PSALM_VERSION);
 
-        $file_paths = $this->file_provider->getFilesInDir($dir_name, $file_extensions);
-
-        foreach ($file_paths as $file_path) {
-            if ($config->isInProjectDirs($file_path)) {
-                if ($this->file_provider->getModifiedTime($file_path) >= $last_run
-                    && $this->parser_cache_provider->loadExistingFileContentsFromCache($file_path)
-                        !== $this->file_provider->getContents($file_path)
-                ) {
-                    $diff_files[] = $file_path;
-                }
+        foreach ($this->project_files as $file_path) {
+            if ($this->file_provider->getModifiedTime($file_path) >= $last_run
+                && $this->parser_cache_provider->loadExistingFileContentsFromCache($file_path)
+                    !== $this->file_provider->getContents($file_path)
+            ) {
+                $diff_files[] = $file_path;
             }
         }
 

diff --git a/src/Psalm/Internal/Provider/FakeFileProvider.php b/src/Psalm/Internal/Provider/FakeFileProvider.php
@@ -57,12 +57,13 @@ public function registerFile(string $file_path, string $file_contents): void
 
     /**
      * @param array<string> $file_extensions
+     * @param null|callable(string):bool $directory_filter
      *
      * @return list<string>
      */
-    public function getFilesInDir(string $dir_path, array $file_extensions): array
+    public function getFilesInDir(string $dir_path, array $file_extensions, callable $directory_filter = null): array
     {
-        $file_paths = parent::getFilesInDir($dir_path, $file_extensions);
+        $file_paths = parent::getFilesInDir($dir_path, $file_extensions, $directory_filter);
 
         foreach ($this->fake_files as $file_path => $_) {
             if (strpos(strtolower($file_path), strtolower($dir_path)) === 0) {

diff --git a/src/Psalm/Internal/Provider/FileProvider.php b/src/Psalm/Internal/Provider/FileProvider.php
@@ -2,7 +2,10 @@
 
 namespace Psalm\Internal\Provider;
 
+use FilesystemIterator;
+use RecursiveCallbackFilterIterator;
 use RecursiveDirectoryIterator;
+use RecursiveIterator;
 use RecursiveIteratorIterator;
 use UnexpectedValueException;
 
@@ -14,6 +17,8 @@
 use function is_dir;
 use function strtolower;
 
+use const DIRECTORY_SEPARATOR;
+
 class FileProvider
 {
     /**
@@ -113,23 +118,37 @@ public function fileExists(string $file_path): bool
 
     /**
      * @param array<string> $file_extensions
+     * @param null|callable(string):bool $directory_filter
      *
      * @return list<string>
      */
-    public function getFilesInDir(string $dir_path, array $file_extensions): array
+    public function getFilesInDir(string $dir_path, array $file_extensions, callable $directory_filter = null): array
     {
         $file_paths = [];
 
+        $iterator = new RecursiveDirectoryIterator(
+            $dir_path,
+            FilesystemIterator::CURRENT_AS_PATHNAME | FilesystemIterator::SKIP_DOTS
+        );
+
+        if ($directory_filter !== null) {
+            $iterator = new RecursiveCallbackFilterIterator(
+                $iterator,
+                /** @param mixed $_ */
+                function (string $current, $_, RecursiveIterator $iterator) use ($directory_filter): bool {
+                    return !$iterator->hasChildren() || $directory_filter($current . DIRECTORY_SEPARATOR);
+                }
+            );
+        }
+
         /** @var RecursiveDirectoryIterator */
-        $iterator = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($dir_path));
+        $iterator = new RecursiveIteratorIterator($iterator);
         $iterator->rewind();
 
         while ($iterator->valid()) {
-            if (!$iterator->isDot()) {
-                $extension = $iterator->getExtension();
-                if (in_array($extension, $file_extensions, true)) {
-                    $file_paths[] = (string)$iterator->getRealPath();
-                }
+            $extension = $iterator->getExtension();
+            if (in_array($extension, $file_extensions, true)) {
+                $file_paths[] = (string)$iterator->getRealPath();
             }
 
             $iterator->next();