Cache doctrees more efficiently

sphinx/environment/__init__.py

@@ -6,7 +6,7 @@
 import os
 import pickle
 from collections import defaultdict
-from copy import copy, deepcopy
+from copy import copy
 from datetime import datetime
 from os import path
 from typing import TYPE_CHECKING, Any, Callable, Generator, Iterator
@@ -178,6 +178,9 @@ def __init__(self, app: Sphinx):
         # docnames to re-read unconditionally on next build
         self.reread_always: set[str] = set()
 
+        # docname -> pickled doctree
+        self._pickled_doctree_cache: dict[str, bytes] = {}
+
         # File metadata
         # docname -> dict of metadata items
         self.metadata: dict[str, dict[str, Any]] = defaultdict(dict)
@@ -577,20 +580,22 @@ def get_domain(self, domainname: str) -> Domain:
 
     def get_doctree(self, docname: str) -> nodes.document:
         """Read the doctree for a file from the pickle and return it."""
-        doctreedir = self.doctreedir
-
-        @functools.lru_cache(maxsize=None)
-        def _load_doctree_from_disk(docname: str) -> nodes.document:
-            """Read the doctree for a file from the pickle and return it."""
-            filename = path.join(doctreedir, docname + '.doctree')
+        try:
+            serialised = self._pickled_doctree_cache[docname]
+        except KeyError:
+            filename = path.join(self.doctreedir, docname + '.doctree')
             with open(filename, 'rb') as f:
-                return pickle.load(f)
+                serialised = self._pickled_doctree_cache[docname] = f.read()
 
-        doctree = deepcopy(_load_doctree_from_disk(docname))
+        doctree = pickle.loads(serialised)
         doctree.settings.env = self
         doctree.reporter = LoggingReporter(self.doc2path(docname))
         return doctree
 
+    @functools.cached_property
+    def master_doctree(self) -> nodes.document:
+        return self.get_doctree(self.config.root_doc)
+
     def get_and_resolve_doctree(
         self,
         docname: str,

sphinx/environment/adapters/toctree.py

@@ -319,7 +319,7 @@ def get_toc_for(self, docname: str, builder: Builder) -> Node:
     def get_toctree_for(self, docname: str, builder: Builder, collapse: bool,
                         **kwargs: Any) -> Element | None:
         """Return the global TOC nodetree."""
-        doctree = self.env.get_doctree(self.env.config.root_doc)
+        doctree = self.env.master_doctree
         toctrees: list[Element] = []
         if 'includehidden' not in kwargs:
             kwargs['includehidden'] = True

sphinx/testing/util.py

@@ -156,6 +156,10 @@ def cleanup(self, doctrees: bool = False) -> None:
     def __repr__(self) -> str:
         return f'<{self.__class__.__name__} buildername={self.builder.name!r}>'
 
+    def build(self, force_all: bool = False, filenames: list[str] | None = None) -> None:
+        self.env._pickled_doctree_cache.clear()
+        super().build(force_all, filenames)
+
 
 class SphinxTestAppWrapperForSkipBuilding:
     """

tests/test_build_gettext.py

@@ -135,6 +135,7 @@ def msgid_getter(msgid):
                    'gettext_additional_targets': []})
 def test_gettext_disable_index_entries(app):
     # regression test for #976
+    app.env._pickled_doctree_cache.clear()  # clear cache
     app.builder.build(['index_entries'])
 
     _msgid_getter = re.compile(r'msgid "(.*)"').search
@@ -165,7 +166,7 @@ def msgid_getter(msgid):
 
 @pytest.mark.sphinx('gettext', testroot='intl', srcdir='gettext')
 def test_gettext_template(app):
-    app.builder.build_all()
+    app.build()
     assert (app.outdir / 'sphinx.pot').isfile()
 
     result = (app.outdir / 'sphinx.pot').read_text(encoding='utf8')

tests/test_build_html.py

@@ -665,7 +665,7 @@ def test_numfig_without_numbered_toctree_warn(app, warning):
     index = (app.srcdir / 'index.rst').read_text(encoding='utf8')
     index = re.sub(':numbered:.*', '', index)
     (app.srcdir / 'index.rst').write_text(index, encoding='utf8')
-    app.builder.build_all()
+    app.build()
 
     warnings = warning.getvalue()
     assert 'index.rst:47: WARNING: numfig is disabled. :numref: is ignored.' not in warnings

tests/test_build_latex.py

@@ -429,7 +429,7 @@ def test_numref_with_prefix2(app, status, warning):
     'latex', testroot='numfig',
     confoverrides={'numfig': True, 'language': 'ja'})
 def test_numref_with_language_ja(app, status, warning):
-    app.builder.build_all()
+    app.build()
     result = (app.outdir / 'python.tex').read_text(encoding='utf8')
     print(result)
     print(status.getvalue())

tests/test_ext_math.py

@@ -108,7 +108,7 @@ def test_mathjax_align(app, status, warning):
                     confoverrides={'math_number_all': True,
                                    'extensions': ['sphinx.ext.mathjax']})
 def test_math_number_all_mathjax(app, status, warning):
-    app.builder.build_all()
+    app.build()
 
     content = (app.outdir / 'index.html').read_text(encoding='utf8')
     html = (r'<div class="math notranslate nohighlight" id="equation-index-0">\s*'
@@ -119,7 +119,7 @@ def test_math_number_all_mathjax(app, status, warning):
 @pytest.mark.sphinx('latex', testroot='ext-math',
                     confoverrides={'extensions': ['sphinx.ext.mathjax']})
 def test_math_number_all_latex(app, status, warning):
-    app.builder.build_all()
+    app.build()
 
     content = (app.outdir / 'python.tex').read_text(encoding='utf8')
     macro = (r'\\begin{equation\*}\s*'