From b035e12e3c875de72d154f273499913ce23c98be Mon Sep 17 00:00:00 2001 From: jfbu <2589111+jfbu@users.noreply.github.com> Date: Sat, 18 Jun 2022 23:38:58 +0200 Subject: [PATCH 1/6] LaTeX: fix #8686 Text can fall out of code-block at end of page --- sphinx/texinputs/sphinxlatexliterals.sty | 107 +++++++++++++---------- 1 file changed, 60 insertions(+), 47 deletions(-) diff --git a/sphinx/texinputs/sphinxlatexliterals.sty b/sphinx/texinputs/sphinxlatexliterals.sty index cc768c25ba1..7c05bdef701 100644 --- a/sphinx/texinputs/sphinxlatexliterals.sty +++ b/sphinx/texinputs/sphinxlatexliterals.sty @@ -1,7 +1,7 @@ %% LITERAL BLOCKS % % change this info string if making any custom modification -\ProvidesFile{sphinxlatexliterals.sty}[2021/12/06 code-blocks and parsed literals] +\ProvidesFile{sphinxlatexliterals.sty}[2022/06/18 code-blocks and parsed literals] % Provides support for this output mark-up from Sphinx latex writer: % @@ -50,8 +50,8 @@ % 4- while still allowing expansion of Pygments latex mark-up % Other aspects such as framing, caption handling, codeline wrapping are % added on top of it. We should stop using fancyvrb and implement -% 1, 2, 3, 4 by own Sphinx fully native Verbatim. This would allow to solve -% limitations with wrapped long code line not allowing page break. +% 1, 2, 3, 4 by own Sphinx fully native Verbatim. This would greatly simply +% in particular wrapping long code lines in a way allowing page breaks. \RequirePackage{fancyvrb} % For parsed-literal blocks. \RequirePackage{alltt} @@ -60,6 +60,9 @@ % Skip to next page if not enough space at bottom \RequirePackage{needspace} +\newbox\spx@verb@tempboxa +\newbox\spx@verb@tempboxb + % Based on use of "fancyvrb.sty"'s Verbatim. % - with framing allowing page breaks ("framed.sty") % - with breaking of long lines (exploits Pygments mark-up), @@ -267,19 +270,42 @@ {\sphinxrestorefboxsep\sphinxVerbatimFormatLine{#1}}% % no need to restore \fboxsep here, as this ends up in a \hbox from fancyvrb }% -% \sphinxVerbatimFormatLine will be set locally to one of those two: -\newcommand\sphinxVerbatimFormatLineWrap{% - \hsize\linewidth - \ifspx@opt@verbatimforcewraps - \expandafter\spx@verb@FormatLineForceWrap - \else\expandafter\spx@verb@FormatLineWrap - \fi +\newcommand\sphinxVerbatimFormatLine[1]{\hb@xt@\linewidth{\strut #1\hss}}% +% The next two macros are a deep hack of fancyvrb.sty core line processing +% in order to wrap too long lines, either at spaces and natural break-points, +% (soft wrap) or optionally at any character (hard wrap). This requires +% deep hack to work around the \hbox'es wrappers of fancyverb.sty which +% prevents pagebreaks. +\def\spx@verb@@ProcessLines{% + \unskip + \unpenalty + \setbox\spx@verb@tempboxb\lastbox +\ifvoid\spx@verb@tempboxb\else + {\spx@verb@@ProcessLines}% + \FV@ProcessLine{\box\spx@verb@tempboxb\strut}% + \global\let\FV@ProcessLine\FV@ProcessLine + \global\setbox\@tempboxa=\box\@tempboxa +\fi }% -\newcommand\sphinxVerbatimFormatLineNoWrap[1]{\hb@xt@\linewidth{\strut #1\hss}}% -\long\def\spx@verb@FormatLineWrap#1{% - \vtop{\raggedright\hyphenpenalty\z@\exhyphenpenalty\z@ +% This will replace \FV@@PreProcessLine +\def\spx@verb@@PreProcessLine{% + \FV@StepLineNo + \FV@Gobble + \def\spx@verb@FV@Line{\FV@Line}% + \ifspx@opt@verbatimforcewraps + \spx@verb@DecideIfWillForceWrap + \fi + \setbox\spx@verb@tempboxa=\vtop{\hsize\linewidth + \raggedright\hyphenpenalty\z@\exhyphenpenalty\z@ \doublehyphendemerits\z@\finalhyphendemerits\z@ - \strut #1\strut}% + \strut\spx@verb@FV@Line\strut}% + \setbox\spx@verb@tempboxa=\vtop{\unvbox\spx@verb@tempboxa + \setbox\spx@verb@tempboxb\lastbox + {\spx@verb@@ProcessLines}% + \FV@ProcessLine{\box\spx@verb@tempboxb\strut}% + \global\let\FV@ProcessLine\FV@ProcessLine + \global\setbox\@tempboxa=\box\@tempboxa}% + \unvbox\spx@verb@tempboxa }% % % The normal line wrapping allows breaks at spaces and ascii non @@ -342,52 +368,49 @@ % character widths, or if the min width plus verbatimmaxunderfull character % widths is inferior to linewidth, then we apply the "force wrapping" with % potential line break at each character, else we don't. -\long\def\spx@verb@FormatLineForceWrap#1{% - % \spx@image@box is a scratch box register that we can use here +\long\def\spx@verb@DecideIfWillForceWrap{% \global\let\spx@verb@maxwidth\z@ \global\let\spx@verb@minwidth\linewidth - \setbox\spx@image@box + \setbox\spx@verb@tempboxa \vtop{\raggedright\hyphenpenalty\z@\exhyphenpenalty\z@ \doublehyphendemerits\z@\finalhyphendemerits\z@ - \spx@everypar{}\noindent\strut #1\strut\spx@par + \spx@everypar{}\noindent\strut\FV@Line\strut\spx@par \spx@verb@getwidths}% \ifdim\spx@verb@maxwidth> \dimexpr\linewidth+\spx@opt@verbatimmaxoverfull\fontcharwd\font`X \relax - \spx@verb@FormatLineWrap{\spx@verb@wrapPYG #1\spx@verb@wrapPYG}% + \def\spx@verb@FV@Line{\expandafter\spx@verb@wrapPYG\FV@Line\spx@verb@wrapPYG}% \else \ifdim\spx@verb@minwidth< \dimexpr\linewidth-\spx@opt@verbatimmaxunderfull\fontcharwd\font`X \relax - \spx@verb@FormatLineWrap{\spx@verb@wrapPYG #1\spx@verb@wrapPYG}% - \else - \spx@verb@FormatLineWrap{#1}% - \fi\fi + \def\spx@verb@FV@Line{\expandafter\spx@verb@wrapPYG\FV@Line\spx@verb@wrapPYG}% + \fi + \fi }% % auxiliary paragraph dissector to get max and min widths % but minwidth must not take into account the last line -\newbox\spx@scratchbox \def\spx@verb@getwidths {% \unskip\unpenalty - \setbox\spx@scratchbox\lastbox - \ifvoid\spx@scratchbox + \setbox\spx@verb@tempboxb\lastbox + \ifvoid\spx@verb@tempboxb \else - \setbox\spx@scratchbox\hbox{\unhbox\spx@scratchbox}% - \ifdim\spx@verb@maxwidth<\wd\spx@scratchbox - \xdef\spx@verb@maxwidth{\number\wd\spx@scratchbox sp}% + \setbox\spx@verb@tempboxb\hbox{\unhbox\spx@verb@tempboxb}% + \ifdim\spx@verb@maxwidth<\wd\spx@verb@tempboxb + \xdef\spx@verb@maxwidth{\number\wd\spx@verb@tempboxb sp}% \fi \expandafter\spx@verb@getwidths@loop \fi }% \def\spx@verb@getwidths@loop {% \unskip\unpenalty - \setbox\spx@scratchbox\lastbox - \ifvoid\spx@scratchbox + \setbox\spx@verb@tempboxb\lastbox + \ifvoid\spx@verb@tempboxb \else - \setbox\spx@scratchbox\hbox{\unhbox\spx@scratchbox}% - \ifdim\spx@verb@maxwidth<\wd\spx@scratchbox - \xdef\spx@verb@maxwidth{\number\wd\spx@scratchbox sp}% + \setbox\spx@verb@tempboxb\hbox{\unhbox\spx@verb@tempboxb}% + \ifdim\spx@verb@maxwidth<\wd\spx@verb@tempboxb + \xdef\spx@verb@maxwidth{\number\wd\spx@verb@tempboxb sp}% \fi - \ifdim\spx@verb@minwidth>\wd\spx@scratchbox - \xdef\spx@verb@minwidth{\number\wd\spx@scratchbox sp}% + \ifdim\spx@verb@minwidth>\wd\spx@verb@tempboxb + \xdef\spx@verb@minwidth{\number\wd\spx@verb@tempboxb sp}% \fi \expandafter\spx@verb@getwidths@loop \fi @@ -544,22 +567,12 @@ \let\sphinxVerbatim@Continues\@empty \fi \ifspx@opt@verbatimwrapslines - % fancyvrb's Verbatim puts each input line in (unbreakable) horizontal boxes. - % This customization wraps each line from the input in a \vtop, thus - % allowing it to wrap and display on two or more lines in the latex output. - % - The codeline counter will be increased only once. - % - The wrapped material will not break across pages, it is impossible - % to achieve this without extensive rewrite of fancyvrb. - % - The (not used in sphinx) obeytabs option to Verbatim is - % broken by this change (showtabs and tabspace work). - \let\sphinxVerbatimFormatLine\sphinxVerbatimFormatLineWrap + \let\FV@@PreProcessLine\spx@verb@@PreProcessLine \let\FV@Space\spx@verbatim@space % Allow breaks at special characters using \PYG... macros. \sphinxbreaksatspecials % Breaks at punctuation characters . , ; ? ! and / (needs catcode activation) \fvset{codes*=\sphinxbreaksviaactive}% - \else % end of conditional code for wrapping long code lines - \let\sphinxVerbatimFormatLine\sphinxVerbatimFormatLineNoWrap \fi \let\FancyVerbFormatLine\sphinxFancyVerbFormatLine \VerbatimEnvironment From 9622060b228837a99450efa9923d887b0268114b Mon Sep 17 00:00:00 2001 From: jfbu <2589111+jfbu@users.noreply.github.com> Date: Sun, 19 Jun 2022 08:53:00 +0200 Subject: [PATCH 2/6] LaTeX comments about fancyvrb obeytabs option modified: sphinx/texinputs/sphinxlatexliterals.sty --- sphinx/texinputs/sphinxlatexliterals.sty | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/sphinx/texinputs/sphinxlatexliterals.sty b/sphinx/texinputs/sphinxlatexliterals.sty index 7c05bdef701..15ecb490d60 100644 --- a/sphinx/texinputs/sphinxlatexliterals.sty +++ b/sphinx/texinputs/sphinxlatexliterals.sty @@ -270,6 +270,10 @@ {\sphinxrestorefboxsep\sphinxVerbatimFormatLine{#1}}% % no need to restore \fboxsep here, as this ends up in a \hbox from fancyvrb }% +% MEMO: fancyvrb has options obeytabs and tabsize. Anyhow tab characters +% do not make it to the tex file, they have been converted to spaces earlier. +% But, if this was not the case, the support would be implemented here via +% \newcommand\sphinxVerbatimFormatLine[1]{\hb@xt@\linewidth{\FV@ObeyTabs{\strut #1}\hss}}% \newcommand\sphinxVerbatimFormatLine[1]{\hb@xt@\linewidth{\strut #1\hss}}% % The next two macros are a deep hack of fancyvrb.sty core line processing % in order to wrap too long lines, either at spaces and natural break-points, @@ -293,18 +297,26 @@ \FV@Gobble \def\spx@verb@FV@Line{\FV@Line}% \ifspx@opt@verbatimforcewraps - \spx@verb@DecideIfWillForceWrap + \spx@verb@DecideIfWillDoForceWrap \fi \setbox\spx@verb@tempboxa=\vtop{\hsize\linewidth \raggedright\hyphenpenalty\z@\exhyphenpenalty\z@ \doublehyphendemerits\z@\finalhyphendemerits\z@ - \strut\spx@verb@FV@Line\strut}% +% MEMO: fancyvrb has options obeytabs and tabsize. Anyhow tab characters +% do not make it to the tex file, they have been converted to spaces earlier. +% But, if this was not the case, the support would be implemented here via +% \FV@ObeyTabs{\strut\spx@verb@FV@Line\strut}% +% And one would need a similar change in the measuring phase done by +% \spx@verb@DecideIfWillDoForceWrap + \strut\spx@verb@FV@Line\strut + }% \setbox\spx@verb@tempboxa=\vtop{\unvbox\spx@verb@tempboxa \setbox\spx@verb@tempboxb\lastbox {\spx@verb@@ProcessLines}% \FV@ProcessLine{\box\spx@verb@tempboxb\strut}% \global\let\FV@ProcessLine\FV@ProcessLine - \global\setbox\@tempboxa=\box\@tempboxa}% + \global\setbox\@tempboxa=\box\@tempboxa + }% \unvbox\spx@verb@tempboxa }% % @@ -368,7 +380,7 @@ % character widths, or if the min width plus verbatimmaxunderfull character % widths is inferior to linewidth, then we apply the "force wrapping" with % potential line break at each character, else we don't. -\long\def\spx@verb@DecideIfWillForceWrap{% +\long\def\spx@verb@DecideIfWillDoForceWrap{% \global\let\spx@verb@maxwidth\z@ \global\let\spx@verb@minwidth\linewidth \setbox\spx@verb@tempboxa From 24e2c9d5a3e32b591fd2f6234e971d73ac4f34ac Mon Sep 17 00:00:00 2001 From: jfbu <2589111+jfbu@users.noreply.github.com> Date: Sun, 19 Jun 2022 09:21:25 +0200 Subject: [PATCH 3/6] LaTeX code comments related to SphinxVerbatim versus fancyvrb modified: sphinx/texinputs/sphinxlatexliterals.sty --- sphinx/texinputs/sphinxlatexliterals.sty | 43 +++++++++++++++++++----- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/sphinx/texinputs/sphinxlatexliterals.sty b/sphinx/texinputs/sphinxlatexliterals.sty index 15ecb490d60..a45d33aae49 100644 --- a/sphinx/texinputs/sphinxlatexliterals.sty +++ b/sphinx/texinputs/sphinxlatexliterals.sty @@ -254,7 +254,7 @@ \def\spx@abovecaptionskip{\sphinxverbatimsmallskipamount}% } \newcommand*{\sphinxverbatimsmallskipamount}{\smallskipamount} -% serves to implement line highlighting and line wrapping +% serves to implement line highlighting \newcommand\sphinxFancyVerbFormatLine[1]{% \expandafter\sphinx@verbatim@checkifhl\expandafter{\the\FV@CodeLineNo}% \ifin@ @@ -275,11 +275,21 @@ % But, if this was not the case, the support would be implemented here via % \newcommand\sphinxVerbatimFormatLine[1]{\hb@xt@\linewidth{\FV@ObeyTabs{\strut #1}\hss}}% \newcommand\sphinxVerbatimFormatLine[1]{\hb@xt@\linewidth{\strut #1\hss}}% -% The next two macros are a deep hack of fancyvrb.sty core line processing -% in order to wrap too long lines, either at spaces and natural break-points, -% (soft wrap) or optionally at any character (hard wrap). This requires -% deep hack to work around the \hbox'es wrappers of fancyverb.sty which -% prevents pagebreaks. +% The next two macros are a deep hack of fancyvrb.sty core line processing in +% order to wrap too long lines, either at spaces and natural break-points, +% (soft wrap) or optionally at any character (hard wrap). This requires deep +% hack to work around the \hbox'es wrappers of fancyvrb.sty as they would +% prevent page breaks. Formerly Sphinx obtained wrapping by inserting the +% material into a vertical box (which was later again boxed -- twice -- by +% fancyvrb thinking it was a single line...) but this was incompatible with +% allowing page breaks (refs: #8686). +% We use core TeX techniques to pre-process a paragraph then recover its +% constituents lines (as boxes, not as tokens) and hand them over to original +% fancyvrb line process. It is mandatory to update \FV@ProcessLine and +% \@tempboxa globally to get fancyvrb internals into working to our +% satisfaction. +% This will get disrupted if anything adding vertical penalties or glues +% is activated via some \vadjust from inside the Pygmentized code lines. \def\spx@verb@@ProcessLines{% \unskip \unpenalty @@ -291,7 +301,18 @@ \global\setbox\@tempboxa=\box\@tempboxa \fi }% -% This will replace \FV@@PreProcessLine +% This will replace fancyvrb's \FV@@PreProcessLine +% Instead of boxing \FV@Line (which contains the Pygmentized line tokens), we +% first typeset it in a vertical box of the suitable width (taking into +% account nested lists) to activate the TeX built-in paragraph builder, then +% we recover individual lines as horizontal boxes and feed them to fancyvrb +% native line processing (which may add line numbers). The interline +% penalties and vertical glue to maintain baseline distance will be added +% again by this process so in recursive \spx@verb@@ProcessLines which starts +% from bottom and makes its way up to first part of the wrapped line we do not +% need to worry about them. An additional initial measuring step is needed if +% user issued verbatimforcewraps=true, which elaborates on the same technique. +% If hard wraps get activated, they get implemented via hacked \PYG macros. \def\spx@verb@@PreProcessLine{% \FV@StepLineNo \FV@Gobble @@ -390,6 +411,8 @@ \spx@verb@getwidths}% \ifdim\spx@verb@maxwidth> \dimexpr\linewidth+\spx@opt@verbatimmaxoverfull\fontcharwd\font`X \relax +% The \expandafter is due to \spx@verb@wrapPYG requiring to "see" the TeX tokens +% from the pygmentize output. \def\spx@verb@FV@Line{\expandafter\spx@verb@wrapPYG\FV@Line\spx@verb@wrapPYG}% \else \ifdim\spx@verb@minwidth< @@ -579,11 +602,13 @@ \let\sphinxVerbatim@Continues\@empty \fi \ifspx@opt@verbatimwrapslines + % deep hack into fancyvrb's internal processing of input lines \let\FV@@PreProcessLine\spx@verb@@PreProcessLine + % space character will allow line breaks \let\FV@Space\spx@verbatim@space - % Allow breaks at special characters using \PYG... macros. + % allow breaks at special characters using \PYG... macros. \sphinxbreaksatspecials - % Breaks at punctuation characters . , ; ? ! and / (needs catcode activation) + % breaks at punctuation characters . , ; ? ! and / (needs catcode activation) \fvset{codes*=\sphinxbreaksviaactive}% \fi \let\FancyVerbFormatLine\sphinxFancyVerbFormatLine From 33a6b438d11be664563198174c575bed67507af9 Mon Sep 17 00:00:00 2001 From: jfbu <2589111+jfbu@users.noreply.github.com> Date: Sun, 19 Jun 2022 09:56:41 +0200 Subject: [PATCH 4/6] LaTeX fix #8686: get line number to be printed only once when wrapping This maintains existing behavior. --- sphinx/texinputs/sphinxlatexliterals.sty | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sphinx/texinputs/sphinxlatexliterals.sty b/sphinx/texinputs/sphinxlatexliterals.sty index a45d33aae49..37c146809ba 100644 --- a/sphinx/texinputs/sphinxlatexliterals.sty +++ b/sphinx/texinputs/sphinxlatexliterals.sty @@ -299,8 +299,13 @@ \FV@ProcessLine{\box\spx@verb@tempboxb\strut}% \global\let\FV@ProcessLine\FV@ProcessLine \global\setbox\@tempboxa=\box\@tempboxa + \aftergroup\spx@verb@@InhibitLineNumber \fi }% +\def\spx@verb@@InhibitLineNumber{% + \let\FV@LeftListNumber\relax + \let\FV@RightListNumber\relax +}% % This will replace fancyvrb's \FV@@PreProcessLine % Instead of boxing \FV@Line (which contains the Pygmentized line tokens), we % first typeset it in a vertical box of the suitable width (taking into From dc013c75aff336db43893f55e72a8588f29adcf3 Mon Sep 17 00:00:00 2001 From: jfbu <2589111+jfbu@users.noreply.github.com> Date: Sun, 19 Jun 2022 10:01:10 +0200 Subject: [PATCH 5/6] Update CHANGES for PR #10577 --- CHANGES | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES b/CHANGES index fad26e53624..cf7fb9cc45f 100644 --- a/CHANGES +++ b/CHANGES @@ -29,6 +29,8 @@ Bugs fixed * #10460: logging: Always show node source locations as absolute paths. * #10520: HTML Theme: Fix use of sidebar classes in ``agogo.css_t``. * #6679: HTML Theme: Fix inclusion of hidden toctrees in the agogo theme. +* #8686: LaTeX: Text can fall out of code-block at end of page and leave artifact + on next page Testing -------- From 3694b6991da3644135b03945041fbc0114ca7f8f Mon Sep 17 00:00:00 2001 From: jfbu <2589111+jfbu@users.noreply.github.com> Date: Sun, 19 Jun 2022 10:08:17 +0200 Subject: [PATCH 6/6] LaTeX typo in a comment --- sphinx/texinputs/sphinxlatexliterals.sty | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/texinputs/sphinxlatexliterals.sty b/sphinx/texinputs/sphinxlatexliterals.sty index 37c146809ba..4e9d22f9a04 100644 --- a/sphinx/texinputs/sphinxlatexliterals.sty +++ b/sphinx/texinputs/sphinxlatexliterals.sty @@ -50,7 +50,7 @@ % 4- while still allowing expansion of Pygments latex mark-up % Other aspects such as framing, caption handling, codeline wrapping are % added on top of it. We should stop using fancyvrb and implement -% 1, 2, 3, 4 by own Sphinx fully native Verbatim. This would greatly simply +% 1, 2, 3, 4 by own Sphinx fully native Verbatim. This would greatly simplify % in particular wrapping long code lines in a way allowing page breaks. \RequirePackage{fancyvrb} % For parsed-literal blocks.