From d0a1890af211390e99c6f6ba1441a1b3261545bc Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 12 Mar 2019 18:16:26 +0100 Subject: [PATCH 1/6] Use latexmk command to build PDF files New versions of Sphinx use `latexmk` to build the PDF files. This command uses a file called `latexmkrc` (or `latexmkjarc` for Japanese) which contains all the proper commands that needs to be ran depending on different Sphinx configurations. `latexmk` will take care by itself on the amount of phases that need to be ran without us worrying about it. Currently, this is not considering LATEXMKOPTS and XINDYOPTS environment variables configured by Sphinx. This feature is implemented under a Feature flag so we can test it easily without breaking other working projects. References: - https://github.com/rtfd/readthedocs.org/issues/1556 - https://github.com/rtfd/readthedocs.org/issues/4454 - https://github.com/rtfd/readthedocs.org/pull/5405 - https://github.com/TECS-WG/tecs-docs/issues/7 - https://github.com/sphinx-doc/sphinx/blob/master/sphinx/texinputs/Makefile_t - https://www.sphinx-doc.org/en/master/usage/builders/index.html#sphinx.builders.latex.LaTeXBuilder --- readthedocs/doc_builder/backends/sphinx.py | 53 ++++++++++++++++++++++ readthedocs/projects/models.py | 2 + 2 files changed, 55 insertions(+) diff --git a/readthedocs/doc_builder/backends/sphinx.py b/readthedocs/doc_builder/backends/sphinx.py index 99592139433..c0257ef95db 100644 --- a/readthedocs/doc_builder/backends/sphinx.py +++ b/readthedocs/doc_builder/backends/sphinx.py @@ -6,12 +6,14 @@ .. _Sphinx: http://www.sphinx-doc.org/ """ import codecs +import itertools import logging import os import shutil import sys import zipfile from glob import glob +from pathlib import Path from django.conf import settings from django.template import loader as template_loader @@ -395,6 +397,57 @@ def build(self): raise BuildEnvironmentError('No TeX files were found') # Run LaTeX -> PDF conversions + if self.project.has_feature(Feature.USE_PDF_LATEXMK): + return self._build_latexmk(cwd, latex_cwd) + + return self._build_pdflatex(tex_files, latex_cwd) + + def _build_latexmk(self, cwd, latex_cwd): + # These steps are copied from the Makefile generated by Sphinx >= 1.6 + # https://github.com/sphinx-doc/sphinx/blob/master/sphinx/texinputs/Makefile_t + latex_path = Path(latex_cwd) + images = [] + for extension in ('png', 'gif', 'jpg', 'jpeg'): + images.extend(latex_path.glob(f'*.{extension}')) + + # FIXME: instead of checking by language here, what we want to check if + # ``latex_engine`` is ``platex`` + pdfs = [] + if self.project.language == 'ja': + pdfs = latex_path.glob('*.pdf') + + for image in itertools.chain(images, pdfs): + self.run( + 'extractbb', + image.name, + cwd=latex_cwd, + record=False, + ) + + rcfile = 'latexmkrc' + if self.project.language == 'ja': + rcfile = 'latexmkjarc' + + cmd_ret = self.run( + 'latexmk', + '-r', + rcfile, + + # FIXME: check for platex here as well + '-pdfdvi' if self.project.language == 'ja' else '-pdf', + + '-dvi-', + '-ps-', + f'-jobname={self.project.slug}', + warn_only=True, + cwd=latex_cwd, + ) + + self.pdf_file_name = f'{self.project.slug}.pdf' + + return True # :) + + def _build_pdflatex(self, tex_files, latex_cwd): pdflatex_cmds = [ ['pdflatex', '-interaction=nonstopmode', tex_file] for tex_file in tex_files diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 94f154861e1..0a23a333bb6 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -1314,10 +1314,12 @@ def add_features(sender, **kwargs): DONT_SHALLOW_CLONE = 'dont_shallow_clone' USE_TESTING_BUILD_IMAGE = 'use_testing_build_image' SHARE_SPHINX_DOCTREE = 'share_sphinx_doctree' + USE_PDF_LATEXMK = 'use_pdf_latexmk' FEATURES = ( (USE_SPHINX_LATEST, _('Use latest version of Sphinx')), (USE_SETUPTOOLS_LATEST, _('Use latest version of setuptools')), + (USE_PDF_LATEXMK, _('Use latexmk to build the PDF')), (ALLOW_DEPRECATED_WEBHOOKS, _('Allow deprecated webhook views')), (PIP_ALWAYS_UPGRADE, _('Always run pip install --upgrade')), (SKIP_SUBMODULES, _('Skip git submodule checkout')), From 24ba76222df361d6e284bf2c028dc74f0e0271d1 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 13 Mar 2019 18:52:39 +0100 Subject: [PATCH 2/6] Show latexrc file on build output --- readthedocs/doc_builder/backends/sphinx.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/readthedocs/doc_builder/backends/sphinx.py b/readthedocs/doc_builder/backends/sphinx.py index c0257ef95db..83074605847 100644 --- a/readthedocs/doc_builder/backends/sphinx.py +++ b/readthedocs/doc_builder/backends/sphinx.py @@ -428,6 +428,12 @@ def _build_latexmk(self, cwd, latex_cwd): if self.project.language == 'ja': rcfile = 'latexmkjarc' + self.run( + 'cat', + rcfile, + cwd=latex_cwd, + ) + cmd_ret = self.run( 'latexmk', '-r', From c2d68e55296a8e07f08521629e7bf3a3b725c9ac Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Sat, 16 Mar 2019 23:47:08 +0100 Subject: [PATCH 3/6] Return the proper successful command --- readthedocs/doc_builder/backends/sphinx.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/doc_builder/backends/sphinx.py b/readthedocs/doc_builder/backends/sphinx.py index 83074605847..2d1e0db7299 100644 --- a/readthedocs/doc_builder/backends/sphinx.py +++ b/readthedocs/doc_builder/backends/sphinx.py @@ -434,7 +434,7 @@ def _build_latexmk(self, cwd, latex_cwd): cwd=latex_cwd, ) - cmd_ret = self.run( + cmd = self.run( 'latexmk', '-r', rcfile, @@ -451,7 +451,7 @@ def _build_latexmk(self, cwd, latex_cwd): self.pdf_file_name = f'{self.project.slug}.pdf' - return True # :) + return cmd.successful def _build_pdflatex(self, tex_files, latex_cwd): pdflatex_cmds = [ From 36ed646882906bc37a76a0e3642bf612b7a611d4 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 18 Mar 2019 11:53:07 +0100 Subject: [PATCH 4/6] Append default Sphinx configurations for Chinese and Japanaese Automatically select `xelatex` for Chinese and `platex` for Japanese. These defaults can be overridden by the user. Force `latex_use_xindy=False` for now until we support it in our Docker image. --- readthedocs/doc_builder/backends/sphinx.py | 3 ++ .../templates/doc_builder/conf.py.tmpl | 35 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/readthedocs/doc_builder/backends/sphinx.py b/readthedocs/doc_builder/backends/sphinx.py index 2d1e0db7299..742067d7e86 100644 --- a/readthedocs/doc_builder/backends/sphinx.py +++ b/readthedocs/doc_builder/backends/sphinx.py @@ -153,6 +153,9 @@ def get_config_params(self): 'dont_overwrite_sphinx_context': self.project.has_feature( Feature.DONT_OVERWRITE_SPHINX_CONTEXT, ), + 'use_pdf_latexmk': self.project.has_feature( + Feature.USE_PDF_LATEXMK, + ), } finalize_sphinx_context_data.send( diff --git a/readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl b/readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl index 9a842eac0d6..c1e8b12f5db 100644 --- a/readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl +++ b/readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl @@ -140,3 +140,38 @@ if 'extensions' in globals(): extensions.insert(0, "readthedocs_ext.readthedocs") else: extensions = ["readthedocs_ext.readthedocs"] + +{% if use_pdf_latexmk %} +project_language = '{{ project.language }}' + +# User's Sphinx configurations +language_user = globals().get('language', None) +latex_engine_user = globals().get('latex_engine', None) +latex_elements_user = globals().get('latex_elements', None) + +chinese = any([ + language_user in ('zh_CN', 'zh_TW'), + project_language in ('zh_CN', 'zh_TW'), +]) + +japanase = any([ + language_user == 'ja', + project_language == 'ja', +]) + +if chinese: + latex_engine = latex_engine_user or 'xelatex' + + # Remove this once xindy gets installed in Docker image and XINDYOPS + # env variable is supported + # https://github.com/rtfd/readthedocs-docker-images/pull/98 + latex_use_xindy = False + + latex_elements_rtd = { + 'preamble': '\\usepackage[UTF8]{ctex}\n', + } + latex_elements = latex_elements_user or latex_elements_rtd +elif japanase: + latex_engine = latex_engine_user or 'platex' + latex_use_xindy = False +{% endif %} From 98f0f55a973569cd9b690bd5624b131be1ec36a9 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 18 Mar 2019 11:57:23 +0100 Subject: [PATCH 5/6] Document USE_PDF_LATEXMK feature flag --- docs/guides/feature-flags.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/guides/feature-flags.rst b/docs/guides/feature-flags.rst index 91ba814356b..7435e0ed806 100644 --- a/docs/guides/feature-flags.rst +++ b/docs/guides/feature-flags.rst @@ -12,6 +12,8 @@ or disable one or more of these featured flags for a particular project. Available Flags --------------- +``USE_PDF_LATEXMK``: :featureflags:`USE_PDF_LATEXMK` + ``USE_SPHINX_LATEST``: :featureflags:`USE_SPHINX_LATEST` ``USE_SETUPTOOLS_LATEST``: :featureflags:`USE_SETUPTOOLS_LATEST` From d9ccf27f4c47645dc8b031ec5b93955c8e41e774 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 18 Mar 2019 16:29:55 +0100 Subject: [PATCH 6/6] Add comment explaining one of the build steps --- readthedocs/doc_builder/backends/sphinx.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/readthedocs/doc_builder/backends/sphinx.py b/readthedocs/doc_builder/backends/sphinx.py index 742067d7e86..2938ce8d325 100644 --- a/readthedocs/doc_builder/backends/sphinx.py +++ b/readthedocs/doc_builder/backends/sphinx.py @@ -417,6 +417,10 @@ def _build_latexmk(self, cwd, latex_cwd): # ``latex_engine`` is ``platex`` pdfs = [] if self.project.language == 'ja': + # Japanese language is the only one that requires this extra + # step. I don't know exactly why but most of the documentation that + # I read differentiate this language from the others. I suppose + # it's because it mix kanji (Chinese) with its own symbols. pdfs = latex_path.glob('*.pdf') for image in itertools.chain(images, pdfs):