Adds a new binary_dirs key to mkdocs.yml. Listed directories bypass…

… file copy during serve by using junctions and symbolic links instead (mkdocs#2662)

mkdocs/commands/build.py

@@ -246,7 +246,13 @@ def _build_page(
         config._current_page = None
 
 
-def build(config: MkDocsConfig, *, serve_url: str | None = None, dirty: bool = False) -> None:
+def build(
+    config: MkDocsConfig,
+    *,
+    serve_url: str | None = None,
+    dirty: bool = False,
+    is_serve: bool = False,
+) -> None:
     """Perform a full site build."""
     logger = logging.getLogger('mkdocs')
 
@@ -322,7 +328,15 @@ def build(config: MkDocsConfig, *, serve_url: str | None = None, dirty: bool = F
         # with lower precedence get written first so that files with higher precedence can overwrite them.
 
         log.debug("Copying static assets.")
-        files.copy_static_files(dirty=dirty, inclusion=inclusion)
+        files.copy_static_files(
+            binary_dirs=config.binary_dirs,
+            site_dir=config.site_dir,
+            docs_dir=config.docs_dir,
+            use_directory_urls=config.use_directory_urls,
+            dirty=dirty,
+            inclusion=inclusion,
+            is_serve=is_serve,
+        )
 
         for template in config.theme.static_templates:
             _build_theme_template(template, env, files, config, nav)

mkdocs/commands/serve.py

@@ -64,7 +64,7 @@ def builder(config: MkDocsConfig | None = None):
             config = get_config()
             config.site_url = serve_url
 
-        build(config, serve_url=None if is_clean else serve_url, dirty=is_dirty)
+        build(config, serve_url=None if is_clean else serve_url, dirty=is_dirty, is_serve=True)
 
     server = LiveReloadServer(
         builder=builder, host=host, port=port, root=site_dir, mount_path=mount_path

mkdocs/config/defaults.py

@@ -79,6 +79,9 @@ class MkDocsConfig(base.Config):
     site_dir = c.SiteDir(default='site')
     """The directory where the site will be built to"""
 
+    binary_dirs = c.Optional(c.Type(list))
+    """The paths that store large binaries that should be symlinked instead of copied when running a server"""
+
     copyright = c.Optional(c.Type(str))
     """A copyright notice to add to the footer of documentation."""
 

mkdocs/contrib/search/__init__.py

@@ -117,4 +117,4 @@ def on_post_build(self, config: MkDocsConfig, **kwargs) -> None:
             for filename in files:
                 from_path = os.path.join(base_path, 'lunr-language', filename)
                 to_path = os.path.join(output_base_path, filename)
-                utils.copy_file(from_path, to_path)
+                utils.copy_file(from_path, to_path, config.binary_dirs, config.docs_dir)

mkdocs/structure/files.py

@@ -5,10 +5,11 @@
 import logging
 import os
 import posixpath
+import re
 import shutil
 import warnings
 from functools import cached_property
-from pathlib import PurePath, PurePosixPath
+from pathlib import Path, PurePath, PurePosixPath
 from typing import TYPE_CHECKING, Callable, Iterable, Iterator, Mapping, Sequence, overload
 from urllib.parse import quote as urlquote
 
@@ -112,14 +113,66 @@ def remove(self, file: File) -> None:
 
     def copy_static_files(
         self,
+        binary_dirs: list[str] | None,
+        site_dir: str,
+        docs_dir: str,
+        use_directory_urls: bool = False,
         dirty: bool = False,
+        is_serve: bool = False,
         *,
         inclusion: Callable[[InclusionLevel], bool] = InclusionLevel.is_included,
     ) -> None:
         """Copy static files from source to destination."""
         for file in self:
             if not file.is_documentation_page() and inclusion(file.inclusion):
-                file.copy_file(dirty)
+                file.copy_file(binary_dirs, docs_dir, dirty, is_serve)
+
+        # Create symbolic links to absolute dirs specified in the binary_dirs key,
+        # or copy their files if it's a build
+        if binary_dirs:
+            match = [re.search('^(?:/|~/|[A-Za-z]+:).*', x) for x in binary_dirs]
+
+            if any(match):
+                found_dirs = [x for x in match if x is not None]
+
+                for found_dir in found_dirs:
+                    found_dir_str = found_dir.group()
+
+                    # Get the last subdir name
+                    last_subdir = Path(found_dir_str).parts[-1]
+
+                    dir_source = Path(found_dir_str).expanduser()
+
+                    if is_serve:
+                        # Make a symbolic link to the directory
+                        dir_dest = Path(site_dir).joinpath(last_subdir)
+
+                        if not Path(dir_dest).exists():
+                            utils.create_symbolic_dir(dir_source, dir_dest)
+                    else:
+                        # Copy files
+                        if Path(dir_source).is_dir():
+                            log.info(f'Copying files from {dir_source!s} binary directory')
+                            for binary_dir_file in Path(dir_source).glob('**/*'):
+                                if not binary_dir_file.is_dir():
+                                    relative_binary_dir_file = (
+                                        str(binary_dir_file)
+                                        .replace(str(dir_source), '')
+                                        .strip('\\')
+                                        .strip('/')
+                                    )
+                                    binary_dest_dir = str(Path(site_dir).joinpath(dir_source.name))
+
+                                    binary_dir_file_obj = File(
+                                        relative_binary_dir_file,
+                                        str(dir_source),
+                                        binary_dest_dir,
+                                        use_directory_urls,
+                                    )
+
+                                    binary_dir_file_obj.copy_file([], docs_dir, dirty, is_serve)
+                        else:
+                            log.error(f'Can\'t copy files from binary directory {dir_source!s}, as it doesn\'t exist')
 
     def documentation_pages(
         self, *, inclusion: Callable[[InclusionLevel], bool] = InclusionLevel.is_included
@@ -464,7 +517,13 @@ def content_string(self, value: str):
         self._content = value
         self.abs_src_path = None
 
-    def copy_file(self, dirty: bool = False) -> None:
+    def copy_file(
+        self,
+        binary_dirs: list[str] | None = [],
+        docs_dir: str = '',
+        dirty: bool = False,
+        is_serve: bool = False,
+    ) -> None:
         """Copy source file to destination, ensuring parent directories exist."""
         if dirty and not self.is_modified():
             log.debug(f"Skip copying unmodified file: '{self.src_uri}'")
@@ -476,7 +535,9 @@ def copy_file(self, dirty: bool = False) -> None:
         if content is None:
             assert self.abs_src_path is not None
             try:
-                utils.copy_file(self.abs_src_path, output_path)
+                utils.copy_file(
+                    self.abs_src_path, self.abs_dest_path, binary_dirs, docs_dir, is_serve
+                )
             except shutil.SameFileError:
                 pass  # Let plugins write directly into site_dir.
         elif isinstance(content, str):

mkdocs/utils/__init__.py

@@ -16,7 +16,7 @@
 import warnings
 from collections import defaultdict
 from datetime import datetime, timezone
-from pathlib import PurePath
+from pathlib import Path, PurePath
 from typing import TYPE_CHECKING, Collection, Iterable, MutableSequence, TypeVar
 from urllib.parse import urlsplit
 
@@ -110,17 +110,89 @@ def insort(a: MutableSequence[T], x: T, *, key=lambda v: v) -> None:
         a.insert(i, x)
 
 
-def copy_file(source_path: str, output_path: str) -> None:
+def copy_file(
+    source_path: str,
+    output_path: str,
+    binary_dirs: list[str] | None = [],
+    docs_dir: str = '',
+    is_serve: bool = False,
+) -> None:
     """
     Copy source_path to output_path, making sure any parent directories exist.
 
     The output_path may be a directory.
     """
     output_dir = os.path.dirname(output_path)
-    os.makedirs(output_dir, exist_ok=True)
-    if os.path.isdir(output_path):
-        output_path = os.path.join(output_path, os.path.basename(source_path))
-    shutil.copyfile(source_path, output_path)
+    copy_files_fallback = False
+
+    # If the source_path contains a dir specified in the binary_dirs key, create symlinks instead
+    if binary_dirs and is_serve:
+        source_file_parent_posix = Path(source_path).parent.as_posix()
+        docs_dir_posix = Path(docs_dir).as_posix()
+
+        match = [
+            re.search(f'{docs_dir_posix}/{x.strip("/")}($|/)?', source_file_parent_posix)
+            for x in binary_dirs
+        ]
+
+        if any(match):
+            if not Path(output_dir).exists():
+                found_dir = (
+                    [x for x in match if x is not None][0]
+                    .group()
+                    .replace(docs_dir_posix, '')
+                    .strip('/')
+                )
+
+                # Find the first instance of the large binaries dir in the source path and set the source and
+                # destination accordingly
+                if found_dir in Path(source_path).parts:
+                    symlink_source = Path(
+                        *Path(source_path).parts[0 : Path(source_path).parts.index(found_dir) + 1]
+                    )
+                    symlink_dest = Path(
+                        *Path(output_path).parts[0 : Path(output_path).parts.index(found_dir) + 1]
+                    )
+
+                create_symbolic_dir(symlink_source, symlink_dest)
+        else:
+            copy_files_fallback = True
+    else:
+        copy_files_fallback = True
+
+    if copy_files_fallback:
+        os.makedirs(output_dir, exist_ok=True)
+
+        if os.path.isdir(output_path):
+            output_path = os.path.join(output_path, os.path.basename(source_path))
+
+        shutil.copyfile(source_path, output_path)
+
+
+def create_symbolic_dir(source_path: Path, output_path: Path) -> None:
+    """Create a symbolic directory in an output path."""
+    # Create a junction instead of a symlink in Windows, as junctions don't need elevated permissions
+    if os.name == 'nt':
+        import _winapi
+
+        try:
+            if Path(source_path).is_dir():
+                _winapi.CreateJunction(str(source_path), str(output_path))
+                log.info(f'Created junction to binary directory {source_path!s} at {output_path!s}')
+            else:
+                log.error(f'Can\'t create junction to binary directory {source_path!s}, as it doesn\'t exist')
+
+        except OSError:
+            log.error(f'Can\'t create junction to binary directory {output_path!s}')
+    elif os.name == 'posix':
+        try:
+            if Path(source_path).is_dir():
+                os.symlink(str(source_path), str(output_path))
+                log.info(f'Created symlink to binary directory {source_path!s} at {output_path!s}')
+            else:
+                log.error(f'Can\'t create symbolic link to binary directory {source_path!s}, as it doesn\'t exist')
+        except OSError:
+            log.error(f'Can\'t create symbolic link to binary directory {output_path!s}')
 
 
 def write_file(content: bytes, output_path: str) -> None: