Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix how file handling works so hidden files like .pages can be copied #394

Merged
merged 1 commit into from Sep 30, 2022
Merged

Fix how file handling works so hidden files like .pages can be copied #394

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion examples/ok-mkdocs-docs-include/mkdocs-test.yml
View file
Open in desktop
Expand Up @@ -2,4 +2,4 @@
site_name: ok-mkdocs-docs-include
plugins:
- simple:
include_folders: ["./subfolder**"]
include_folders: ["subfolder/**"]
126 changes: 71 additions & 55 deletions mkdocs_simple_plugin/simple.py
View file
Open in desktop
Expand Up @@ -4,7 +4,7 @@
import shutil
import stat
import sys
import glob
import pathlib

from mkdocs import utils
from mkdocs_simple_plugin.semiliterate import Semiliterate
Expand Down Expand Up @@ -49,29 +49,49 @@ def __init__(
ignore_paths: list,
semiliterate: dict,
**kwargs):
"""Initialize module instance with settings."""
"""Initialize module instance with settings.

Args:
build_docs_dir (str): Output directory for processed files
include_folders (list): Glob of folders to search for files
include_extensions (list): Glob of filenames to copy directly to
output
ignore_folders (list): Glob of paths to exclude
ignore_hidden (bool): Whether to ignore hidden files for processing
ignore_paths (list): Absolute filepaths to exclude
semiliterate (dict): Settings for processing file content in
Semiliterate

"""
self.build_dir = build_docs_dir
self.include_folders = set(include_folders)
self.include_extensions = set(include_extensions)
self.ignore_folders = set(ignore_folders)
self.copy_glob = set(include_extensions)
self.ignore_glob = set(ignore_folders)
self.ignore_hidden = ignore_hidden
self.hidden_prefix = set([".", "__"])
self.ignore_paths = set(ignore_paths)
self.semiliterate = []
for item in semiliterate:
self.semiliterate.append(Semiliterate(**item))

def get_included(self) -> list:
"""Get a list of folders and files to include."""
included = []
for pattern in self.include_folders:
included.extend(glob.glob(pattern))
# Return filtered list of included files
return [f for f in included if not self.is_path_ignored(f)]

def in_extensions(self, name: str) -> bool:
"""Check if file is in include extensions."""
return any(extension in name for extension in self.include_extensions)
def get_files(self) -> list:
"""Get a list of files to process, excluding ignored files."""
files = []
# Get all of the folders that match the include pattern.
entries = []
for entry in self.include_folders:
entries.extend(pathlib.Path().glob(entry))
# Get all of the files in the folder that aren't ignored.
for entry in entries:
# Add files to list
if entry.is_file() and not self.is_path_ignored(entry):
files.extend([os.path.normpath(entry)])
# Add all files in folders to list
for root, _, filenames in os.walk(entry):
files.extend([os.path.join(root, f)
for f in filenames if not self.is_ignored(root, f)]
)
return files

def is_ignored(self, base_path: str, name: str) -> bool:
"""Check if directory and filename should be ignored."""
Expand All @@ -81,12 +101,11 @@ def is_path_ignored(self, path: str = None) -> bool:
"""Check if path should be ignored."""
path = os.path.normpath(path)
base_path = os.path.dirname(path)
# Check if it's hidden
if self.ignore_hidden and self.is_hidden(path):
return True

# Check if its an internally required ignore path
if os.path.abspath(path) in self.ignore_paths:
return True
for ignored in self.ignore_paths:
if ignored in os.path.abspath(path):
return True

# Update ignore patterns from .mkdocsignore file
mkdocsignore = os.path.join(base_path, ".mkdocsignore")
Expand All @@ -98,14 +117,25 @@ def is_path_ignored(self, path: str = None) -> bool:
ignore_list = [x for x in ignore_list if not x.startswith('#')]
if not ignore_list:
ignore_list = ["*"]
self.ignore_folders.update(
self.ignore_glob.update(
set(os.path.join(base_path, filter) for filter in ignore_list))
# Check for ignore paths in patterns
if any(fnmatch.fnmatch(os.path.normpath(path), filter)
for filter in self.ignore_folders):
if any(fnmatch.fnmatch(path, filter)
for filter in self.ignore_glob):
return True
# Check for ignore folder in patterns
if any(fnmatch.fnmatch(base_path, filter)
for filter in self.ignore_glob):
return True
return False

def should_copy_file(self, name: str) -> bool:
"""Check if file should be copied."""
def match_pattern(name, pattern):
return fnmatch.fnmatch(name, pattern) or pattern in name

return any(match_pattern(name, pattern) for pattern in self.copy_glob)

def is_hidden(self, filepath):
"""Return true if filepath is hidden."""
def has_hidden_attribute(filepath):
Expand Down Expand Up @@ -133,38 +163,28 @@ def merge_docs(self, from_dir):
def build_docs(self) -> list:
"""Build the docs directory from workspace files."""
paths = []
included = self.get_included()
for item in included:
paths += filter(None, [self._process(item)])
for root, directories, files in os.walk(item):
for file in files:
path = os.path.join(root, file)
paths += filter(None, [self._process(path)])
directories[:] = [
d for d in directories if not self.is_ignored(root, d)]
files = self.get_files()
for file in files:
if not os.path.isfile(file):
continue
from_dir = os.path.dirname(file)
name = os.path.basename(file)
build_prefix = os.path.normpath(
os.path.join(self.build_dir, from_dir))

if (self.try_copy_file(from_dir, name, build_prefix) or
self.try_extract(from_dir, name, build_prefix)):
paths.append(file)
return paths

def _process(self, file) -> str:
if not os.path.isfile(file):
return None
from_dir = os.path.dirname(file)
name = os.path.basename(file)
build_prefix = os.path.normpath(os.path.join(self.build_dir, from_dir))

if (self.try_copy_file(from_dir, name, build_prefix) or
self.try_extract(from_dir, name, build_prefix)):
return file
return None

def try_extract(self, from_dir: str, name: str, to_dir: str) -> bool:
"""Extract content from file into destination.

Returns the name of the file extracted if extractable.
"""
# Check if it's hidden
path = os.path.join(from_dir, name)
if self.is_path_ignored(path):
utils.log.debug(
"mkdocs-simple-plugin: ignoring %s", path)
if self.ignore_hidden and self.is_hidden(path):
return False
extracted = False
for item in self.semiliterate:
Expand All @@ -181,19 +201,15 @@ def try_copy_file(self, from_dir: str, name: str, to_dir: str) -> bool:
original = os.path.join(from_dir, name)
new_file = os.path.join(to_dir, name)

if not self.in_extensions(name):
utils.log.info(
"mkdocs-simple-plugin: skipping file extension %s", original)
return False
if self.is_path_ignored(original):
if not self.should_copy_file(name):
utils.log.debug(
"mkdocs-simple-plugin: ignoring %s", original)
"mkdocs-simple-plugin: skip copying file %s", original)
return False
try:
os.makedirs(to_dir, exist_ok=True)
shutil.copy(original, new_file)
utils.log.debug("mkdocs-simple-plugin: %s --> %s",
original, new_file)
utils.log.info("mkdocs-simple-plugin: %s --> %s",
original, new_file)
return True
except (OSError, IOError, UnicodeDecodeError) as error:
utils.log.warning(
Expand Down
110 changes: 59 additions & 51 deletions tests/test_simple.py
View file
Open in desktop
Expand Up @@ -43,6 +43,7 @@ def setUp(self) -> None:
def test_is_hidden(self, os_stat):
"""Test is_hidden for correctness."""
simple_test = simple.Simple(**self.default_settings)
simple_test.ignore_hidden = True
os_stat.return_value.st_file_attributes = 0
self.assertFalse(simple_test.is_hidden('test.md'))
self.assertFalse(simple_test.is_hidden('./folder/test.md'))
Expand All @@ -63,42 +64,6 @@ def test_ignored_default(self):
)
)

def test_ignored_hidden(self):
"""Test ignored files."""
simple_test = simple.Simple(**self.default_settings)

self.fs.create_file("directory/.hidden")
simple_test.ignore_hidden = True
self.assertTrue(
simple_test.is_ignored(
base_path="directory",
name=".hidden"
)
)
simple_test.ignore_hidden = False
self.assertFalse(
simple_test.is_ignored(
base_path="directory",
name=".hidden"
)
)

self.fs.create_file("directory/file")
simple_test.ignore_hidden = True
self.assertFalse(
simple_test.is_ignored(
base_path="directory",
name="file"
)
)
simple_test.ignore_hidden = False
self.assertFalse(
simple_test.is_ignored(
base_path="directory",
name="file"
)
)

def test_ignored_paths(self):
"""Test ignored files."""
simple_test = simple.Simple(**self.default_settings)
Expand Down Expand Up @@ -154,9 +119,9 @@ def test_ignored_mkdocsignore(self):
simple_test.is_ignored(
base_path=".",
name="hello.md"))
self.assertIn("directory/*test*", simple_test.ignore_folders)
self.assertIn("*test*", simple_test.ignore_folders)
self.assertEqual(2, len(simple_test.ignore_folders))
self.assertIn("directory/*test*", simple_test.ignore_glob)
self.assertIn("*test*", simple_test.ignore_glob)
self.assertEqual(2, len(simple_test.ignore_glob))

def test_ignored_mkdocsignore_empty(self):
"""Test empty mkdocsignore file."""
Expand All @@ -172,15 +137,56 @@ def test_ignored_mkdocsignore_empty(self):
simple_test.is_ignored(
base_path=".",
name="hello.md"))
self.assertIn("directory/*", simple_test.ignore_folders)
self.assertEqual(1, len(simple_test.ignore_folders))
self.assertIn("directory/*", simple_test.ignore_glob)
self.assertEqual(1, len(simple_test.ignore_glob))

def test_in_extensions(self):
"""Test extensions to search."""
self.default_settings["include_extensions"] = [".md"]
def test_should_copy(self):
"""Test should_copy."""
simple_test = simple.Simple(**self.default_settings)
self.assertTrue(simple_test.in_extensions(name="helloworld.md"))
self.assertFalse(simple_test.in_extensions(name="md.helloworld"))
simple_test.copy_glob = ["*.md"]
self.assertTrue(simple_test.should_copy_file(name="helloworld.md"))
self.assertFalse(simple_test.should_copy_file(name="md.helloworld"))

simple_test.copy_glob = [".pages"]
self.assertTrue(simple_test.should_copy_file(name=".pages"))

def test_get_files(self):
"""Test getting all files."""
simple_test = simple.Simple(**self.default_settings)
# /foo
# ├── baz.md
# ├── .mkdocsignore
# └── bar
# ├── spam.md // ignored
# ├── hello.txt
# └── eggs.md
# └── bat
# ├── hello.md
# └── world.md
# /goo
# └── day.md
# boo.md
self.fs.create_file("/foo/baz.md")
self.fs.create_file("/foo/.mkdocsignore", contents="bar/spam.md*")
self.fs.create_file("/foo/bar/spam.md")
self.fs.create_file("/foo/bar/hello.txt")
self.fs.create_file("/foo/bar/eggs.md")
self.fs.create_file("/foo/bat/hello.md")
self.fs.create_file("/foo/bat/world.md")
self.fs.create_file("/goo/day.md")
self.fs.create_file("boo.md")

files = simple_test.get_files()
self.assertIn("foo/baz.md", files)
self.assertIn("foo/.mkdocsignore", files)
self.assertIn("foo/bar/hello.txt", files)
self.assertIn("foo/bar/eggs.md", files)
self.assertNotIn("foo/bar/spam.md", files)
self.assertIn("foo/bat/hello.md", files)
self.assertIn("foo/bat/world.md", files)
self.assertIn("goo/day.md", files)
self.assertIn("boo.md", files)
self.assertEqual(8, len(files))

def test_build_docs(self):
"""Test build docs."""
Expand All @@ -189,8 +195,8 @@ def test_build_docs(self):
# /foo
# ├── baz.md
# ├── .mkdocsignore //hidden + ignore settings
# ├── .pages //include in copy
# └── bar
# ├── .hidden // hidden file
# ├── spam.md // ignored
# ├── hello.txt // wrong extension
# └── eggs.md
Expand All @@ -202,21 +208,23 @@ def test_build_docs(self):
# boo.md // not included
self.fs.create_file("/foo/baz.md")
self.fs.create_file("/foo/.mkdocsignore", contents="bar/spam.md*")
self.fs.create_file("/foo/.pages")
self.fs.create_file("/foo/bar/spam.md")
self.fs.create_file("/foo/bar/eggs.md")
self.fs.create_file("/foo/bar/.hidden")
self.fs.create_file("/foo/bat/hello.md")
self.fs.create_file("/foo/bat/world.md")
self.fs.create_file("/goo/day.md")
self.fs.create_file("boo.md")

simple_test.ignore_folders = set(["foo/bat/**"])
simple_test.include_folders = set(["foo/*"])
simple_test.ignore_glob = set(["foo/bat/**"])
simple_test.include_folders = set(["foo/"])
simple_test.copy_glob = set(["*.md", ".pages"])

paths = simple_test.build_docs()
self.assertIn("foo/baz.md", paths)
self.assertIn("foo/bar/eggs.md", paths)
self.assertEqual(2, len(paths))
self.assertIn("foo/.pages", paths)
self.assertEqual(3, len(paths))


if __name__ == '__main__':
Expand Down