diff --git a/isort/settings.py b/isort/settings.py
index 0d227a61a..7cec61048 100644
--- a/isort/settings.py
+++ b/isort/settings.py
@@ -548,7 +548,15 @@ def _check_folder_gitignore(self, folder: str) -> Optional[Path]:
 
         git_folder = Path(topfolder_result.rstrip()).resolve()
 
-        files = [str(p) for p in Path(git_folder).rglob("*")]
+        files: List[str] = []
+        # don't check symlinks; either part of the repo and would be checked
+        # twice, or is external to the repo and git won't konw anything about it
+        for root, _dirs, git_files in os.walk(git_folder, followlinks=False):
+            for git_file in git_files:
+                git_path = os.path.join(root, git_file)
+                # followlinks only disables walking into linked dirs
+                if not os.path.islink(git_path):
+                    files.append(git_path)
         git_options = ["-C", str(git_folder), "-c", "core.quotePath="]
         try:
             ignored = subprocess.check_output(  # nosec # skipcq: PYL-W1510