Fix how file handling works so hidden files like .pages can be copied (…

…#394)

examples/ok-mkdocs-docs-include/mkdocs-test.yml

@@ -2,4 +2,4 @@
 site_name: ok-mkdocs-docs-include
 plugins:
   - simple:
-      include_folders: ["./subfolder**"]
+      include_folders: ["subfolder/**"]

mkdocs_simple_plugin/simple.py

@@ -4,7 +4,7 @@
 import shutil
 import stat
 import sys
-import glob
+import pathlib
 
 from mkdocs import utils
 from mkdocs_simple_plugin.semiliterate import Semiliterate
@@ -49,29 +49,49 @@ def __init__(
             ignore_paths: list,
             semiliterate: dict,
             **kwargs):
-        """Initialize module instance with settings."""
+        """Initialize module instance with settings.
+
+        Args:
+            build_docs_dir (str): Output directory for processed files
+            include_folders (list): Glob of folders to search for files
+            include_extensions (list): Glob of filenames to copy directly to
+                output
+            ignore_folders (list): Glob of paths to exclude
+            ignore_hidden (bool): Whether to ignore hidden files for processing
+            ignore_paths (list): Absolute filepaths to exclude
+            semiliterate (dict): Settings for processing file content in
+                Semiliterate
+
+        """
         self.build_dir = build_docs_dir
         self.include_folders = set(include_folders)
-        self.include_extensions = set(include_extensions)
-        self.ignore_folders = set(ignore_folders)
+        self.copy_glob = set(include_extensions)
+        self.ignore_glob = set(ignore_folders)
         self.ignore_hidden = ignore_hidden
         self.hidden_prefix = set([".", "__"])
         self.ignore_paths = set(ignore_paths)
         self.semiliterate = []
         for item in semiliterate:
             self.semiliterate.append(Semiliterate(**item))
 
-    def get_included(self) -> list:
-        """Get a list of folders and files to include."""
-        included = []
-        for pattern in self.include_folders:
-            included.extend(glob.glob(pattern))
-        # Return filtered list of included files
-        return [f for f in included if not self.is_path_ignored(f)]
-
-    def in_extensions(self, name: str) -> bool:
-        """Check if file is in include extensions."""
-        return any(extension in name for extension in self.include_extensions)
+    def get_files(self) -> list:
+        """Get a list of files to process, excluding ignored files."""
+        files = []
+        # Get all of the folders that match the include pattern.
+        entries = []
+        for entry in self.include_folders:
+            entries.extend(pathlib.Path().glob(entry))
+        # Get all of the files in the folder that aren't ignored.
+        for entry in entries:
+            # Add files to list
+            if entry.is_file() and not self.is_path_ignored(entry):
+                files.extend([os.path.normpath(entry)])
+            # Add all files in folders to list
+            for root, _, filenames in os.walk(entry):
+                files.extend([os.path.join(root, f)
+                             for f in filenames if not self.is_ignored(root, f)]
+                             )
+        return files
 
     def is_ignored(self, base_path: str, name: str) -> bool:
         """Check if directory and filename should be ignored."""
@@ -81,12 +101,11 @@ def is_path_ignored(self, path: str = None) -> bool:
         """Check if path should be ignored."""
         path = os.path.normpath(path)
         base_path = os.path.dirname(path)
-        # Check if it's hidden
-        if self.ignore_hidden and self.is_hidden(path):
-            return True
+
         # Check if its an internally required ignore path
-        if os.path.abspath(path) in self.ignore_paths:
-            return True
+        for ignored in self.ignore_paths:
+            if ignored in os.path.abspath(path):
+                return True
 
         # Update ignore patterns from .mkdocsignore file
         mkdocsignore = os.path.join(base_path, ".mkdocsignore")
@@ -98,14 +117,25 @@ def is_path_ignored(self, path: str = None) -> bool:
                 ignore_list = [x for x in ignore_list if not x.startswith('#')]
             if not ignore_list:
                 ignore_list = ["*"]
-            self.ignore_folders.update(
+            self.ignore_glob.update(
                 set(os.path.join(base_path, filter) for filter in ignore_list))
         # Check for ignore paths in patterns
-        if any(fnmatch.fnmatch(os.path.normpath(path), filter)
-                for filter in self.ignore_folders):
+        if any(fnmatch.fnmatch(path, filter)
+                for filter in self.ignore_glob):
+            return True
+        # Check for ignore folder in patterns
+        if any(fnmatch.fnmatch(base_path, filter)
+                for filter in self.ignore_glob):
             return True
         return False
 
+    def should_copy_file(self, name: str) -> bool:
+        """Check if file should be copied."""
+        def match_pattern(name, pattern):
+            return fnmatch.fnmatch(name, pattern) or pattern in name
+
+        return any(match_pattern(name, pattern) for pattern in self.copy_glob)
+
     def is_hidden(self, filepath):
         """Return true if filepath is hidden."""
         def has_hidden_attribute(filepath):
@@ -133,38 +163,28 @@ def merge_docs(self, from_dir):
     def build_docs(self) -> list:
         """Build the docs directory from workspace files."""
         paths = []
-        included = self.get_included()
-        for item in included:
-            paths += filter(None, [self._process(item)])
-            for root, directories, files in os.walk(item):
-                for file in files:
-                    path = os.path.join(root, file)
-                    paths += filter(None, [self._process(path)])
-                directories[:] = [
-                    d for d in directories if not self.is_ignored(root, d)]
+        files = self.get_files()
+        for file in files:
+            if not os.path.isfile(file):
+                continue
+            from_dir = os.path.dirname(file)
+            name = os.path.basename(file)
+            build_prefix = os.path.normpath(
+                os.path.join(self.build_dir, from_dir))
+
+            if (self.try_copy_file(from_dir, name, build_prefix) or
+                    self.try_extract(from_dir, name, build_prefix)):
+                paths.append(file)
         return paths
 
-    def _process(self, file) -> str:
-        if not os.path.isfile(file):
-            return None
-        from_dir = os.path.dirname(file)
-        name = os.path.basename(file)
-        build_prefix = os.path.normpath(os.path.join(self.build_dir, from_dir))
-
-        if (self.try_copy_file(from_dir, name, build_prefix) or
-                self.try_extract(from_dir, name, build_prefix)):
-            return file
-        return None
-
     def try_extract(self, from_dir: str, name: str, to_dir: str) -> bool:
         """Extract content from file into destination.
 
         Returns the name of the file extracted if extractable.
         """
+        # Check if it's hidden
         path = os.path.join(from_dir, name)
-        if self.is_path_ignored(path):
-            utils.log.debug(
-                "mkdocs-simple-plugin: ignoring %s", path)
+        if self.ignore_hidden and self.is_hidden(path):
             return False
         extracted = False
         for item in self.semiliterate:
@@ -181,19 +201,15 @@ def try_copy_file(self, from_dir: str, name: str, to_dir: str) -> bool:
         original = os.path.join(from_dir, name)
         new_file = os.path.join(to_dir, name)
 
-        if not self.in_extensions(name):
-            utils.log.info(
-                "mkdocs-simple-plugin: skipping file extension %s", original)
-            return False
-        if self.is_path_ignored(original):
+        if not self.should_copy_file(name):
             utils.log.debug(
-                "mkdocs-simple-plugin: ignoring %s", original)
+                "mkdocs-simple-plugin: skip copying file %s", original)
             return False
         try:
             os.makedirs(to_dir, exist_ok=True)
             shutil.copy(original, new_file)
-            utils.log.debug("mkdocs-simple-plugin: %s --> %s",
-                            original, new_file)
+            utils.log.info("mkdocs-simple-plugin: %s --> %s",
+                           original, new_file)
             return True
         except (OSError, IOError, UnicodeDecodeError) as error:
             utils.log.warning(

tests/test_simple.py

@@ -43,6 +43,7 @@ def setUp(self) -> None:
     def test_is_hidden(self, os_stat):
         """Test is_hidden for correctness."""
         simple_test = simple.Simple(**self.default_settings)
+        simple_test.ignore_hidden = True
         os_stat.return_value.st_file_attributes = 0
         self.assertFalse(simple_test.is_hidden('test.md'))
         self.assertFalse(simple_test.is_hidden('./folder/test.md'))
@@ -63,42 +64,6 @@ def test_ignored_default(self):
             )
         )
 
-    def test_ignored_hidden(self):
-        """Test ignored files."""
-        simple_test = simple.Simple(**self.default_settings)
-
-        self.fs.create_file("directory/.hidden")
-        simple_test.ignore_hidden = True
-        self.assertTrue(
-            simple_test.is_ignored(
-                base_path="directory",
-                name=".hidden"
-            )
-        )
-        simple_test.ignore_hidden = False
-        self.assertFalse(
-            simple_test.is_ignored(
-                base_path="directory",
-                name=".hidden"
-            )
-        )
-
-        self.fs.create_file("directory/file")
-        simple_test.ignore_hidden = True
-        self.assertFalse(
-            simple_test.is_ignored(
-                base_path="directory",
-                name="file"
-            )
-        )
-        simple_test.ignore_hidden = False
-        self.assertFalse(
-            simple_test.is_ignored(
-                base_path="directory",
-                name="file"
-            )
-        )
-
     def test_ignored_paths(self):
         """Test ignored files."""
         simple_test = simple.Simple(**self.default_settings)
@@ -154,9 +119,9 @@ def test_ignored_mkdocsignore(self):
             simple_test.is_ignored(
                 base_path=".",
                 name="hello.md"))
-        self.assertIn("directory/*test*", simple_test.ignore_folders)
-        self.assertIn("*test*", simple_test.ignore_folders)
-        self.assertEqual(2, len(simple_test.ignore_folders))
+        self.assertIn("directory/*test*", simple_test.ignore_glob)
+        self.assertIn("*test*", simple_test.ignore_glob)
+        self.assertEqual(2, len(simple_test.ignore_glob))
 
     def test_ignored_mkdocsignore_empty(self):
         """Test empty mkdocsignore file."""
@@ -172,15 +137,56 @@ def test_ignored_mkdocsignore_empty(self):
             simple_test.is_ignored(
                 base_path=".",
                 name="hello.md"))
-        self.assertIn("directory/*", simple_test.ignore_folders)
-        self.assertEqual(1, len(simple_test.ignore_folders))
+        self.assertIn("directory/*", simple_test.ignore_glob)
+        self.assertEqual(1, len(simple_test.ignore_glob))
 
-    def test_in_extensions(self):
-        """Test extensions to search."""
-        self.default_settings["include_extensions"] = [".md"]
+    def test_should_copy(self):
+        """Test should_copy."""
         simple_test = simple.Simple(**self.default_settings)
-        self.assertTrue(simple_test.in_extensions(name="helloworld.md"))
-        self.assertFalse(simple_test.in_extensions(name="md.helloworld"))
+        simple_test.copy_glob = ["*.md"]
+        self.assertTrue(simple_test.should_copy_file(name="helloworld.md"))
+        self.assertFalse(simple_test.should_copy_file(name="md.helloworld"))
+
+        simple_test.copy_glob = [".pages"]
+        self.assertTrue(simple_test.should_copy_file(name=".pages"))
+
+    def test_get_files(self):
+        """Test getting all files."""
+        simple_test = simple.Simple(**self.default_settings)
+        # /foo
+        #  ├── baz.md
+        #  ├── .mkdocsignore
+        #  └── bar
+        #      ├── spam.md // ignored
+        #      ├── hello.txt
+        #      └── eggs.md
+        #  └── bat
+        #      ├── hello.md
+        #      └── world.md
+        # /goo
+        #  └── day.md
+        # boo.md
+        self.fs.create_file("/foo/baz.md")
+        self.fs.create_file("/foo/.mkdocsignore", contents="bar/spam.md*")
+        self.fs.create_file("/foo/bar/spam.md")
+        self.fs.create_file("/foo/bar/hello.txt")
+        self.fs.create_file("/foo/bar/eggs.md")
+        self.fs.create_file("/foo/bat/hello.md")
+        self.fs.create_file("/foo/bat/world.md")
+        self.fs.create_file("/goo/day.md")
+        self.fs.create_file("boo.md")
+
+        files = simple_test.get_files()
+        self.assertIn("foo/baz.md", files)
+        self.assertIn("foo/.mkdocsignore", files)
+        self.assertIn("foo/bar/hello.txt", files)
+        self.assertIn("foo/bar/eggs.md", files)
+        self.assertNotIn("foo/bar/spam.md", files)
+        self.assertIn("foo/bat/hello.md", files)
+        self.assertIn("foo/bat/world.md", files)
+        self.assertIn("goo/day.md", files)
+        self.assertIn("boo.md", files)
+        self.assertEqual(8, len(files))
 
     def test_build_docs(self):
         """Test build docs."""
@@ -189,8 +195,8 @@ def test_build_docs(self):
         # /foo
         #  ├── baz.md
         #  ├── .mkdocsignore //hidden + ignore settings
+        #  ├── .pages //include in copy
         #  └── bar
-        #      ├── .hidden  // hidden file
         #      ├── spam.md // ignored
         #      ├── hello.txt  // wrong extension
         #      └── eggs.md
@@ -202,21 +208,23 @@ def test_build_docs(self):
         # boo.md // not included
         self.fs.create_file("/foo/baz.md")
         self.fs.create_file("/foo/.mkdocsignore", contents="bar/spam.md*")
+        self.fs.create_file("/foo/.pages")
         self.fs.create_file("/foo/bar/spam.md")
         self.fs.create_file("/foo/bar/eggs.md")
-        self.fs.create_file("/foo/bar/.hidden")
         self.fs.create_file("/foo/bat/hello.md")
         self.fs.create_file("/foo/bat/world.md")
         self.fs.create_file("/goo/day.md")
         self.fs.create_file("boo.md")
 
-        simple_test.ignore_folders = set(["foo/bat/**"])
-        simple_test.include_folders = set(["foo/*"])
+        simple_test.ignore_glob = set(["foo/bat/**"])
+        simple_test.include_folders = set(["foo/"])
+        simple_test.copy_glob = set(["*.md", ".pages"])
 
         paths = simple_test.build_docs()
         self.assertIn("foo/baz.md", paths)
         self.assertIn("foo/bar/eggs.md", paths)
-        self.assertEqual(2, len(paths))
+        self.assertIn("foo/.pages", paths)
+        self.assertEqual(3, len(paths))
 
 
 if __name__ == '__main__':