TeX-escape the - char to \sphinxhyphen{} for non-Unicode engines

The commit is careful not to break ids and URIs.

sphinx/texinputs/sphinx.sty

@@ -1078,7 +1078,7 @@
 }
 
 \def\sphinx@verbatim@nolig@list {\do \`}%
-% Some characters . , ; ? ! / are not pygmentized.
+% Some characters . , ; ? ! / are neither pygmentized nor "tex-escaped".
 % This macro makes them "active" and they will insert potential linebreaks.
 % Not compatible with math mode (cf \sphinxunactivateextras).
 \newcommand*\sphinxbreaksbeforeactivelist {}% none
@@ -1369,7 +1369,6 @@
 }
 \newcommand*\sphinxbreaksviaactiveinparsedliteral{%
   \sphinxbreaksviaactive % by default handles . , ; ? ! /
-  \do\-% we need also the hyphen character (ends up "as is" in parsed-literal)
   \lccode`\~`\~ %
   % update \dospecials as it is used by \url
   % but deactivation will already have been done hence this is unneeded:
@@ -1380,7 +1379,7 @@
   \lccode`~32 \lowercase{\let~}\spx@verbatim@space\lccode`\~`\~
 }
 \newcommand*{\sphinxunactivateextras}{\let\do\@makeother
-      \sphinxbreaksbeforeactivelist\sphinxbreaksafteractivelist\do\-}%
+      \sphinxbreaksbeforeactivelist\sphinxbreaksafteractivelist}%
 % the \catcode13=5\relax (deactivate end of input lines) is left to callers
 \newcommand*{\sphinxunactivateextrasandspace}{\catcode32=10\relax
       \sphinxunactivateextras}%
@@ -1805,12 +1804,20 @@
       % break also at \
       \let\sphinx@textbackslash\textbackslash
       \let\textbackslash\sphinxtextbackslash
-      % do not typeset a continuation symbol on next line
+      % by default, no continuation symbol on next line but may be added
       \let\sphinxafterbreak\sphinxafterbreakofinlineliteral
       % do not overwrite the comma set-up
       \let\verbatim@nolig@list\sphinx@literal@nolig@list
    \fi
    % fix a space-gobbling issue due to LaTeX's original \do@noligs
+% TODO: using \@noligs as patched by upquote.sty is now unneeded because
+% either ` and ' are escaped (non-unicode engines) or they don't build
+% ligatures (unicode engines). Thus remove this and unify handling of `, <, >, 
+% ' and - with the characters . , ; ? ! / as handled via
+% \sphinxbreaksviaactive.
+% Hence \sphinx@do@noligs will be removed, or rather replaced  with code
+% inserting discretionaries, as they allow a continuation symbol on start of
+% next line to achieve common design with code-blocks.
    \let\do@noligs\sphinx@do@noligs
    \@noligs\endlinechar\m@ne\everyeof{}% (<- in case inside \sphinxhref)
    \expandafter\scantokens
@@ -1866,6 +1873,7 @@
 % reduce hyperref "Token not allowed in a PDF string" warnings on PDF builds
 \AtBeginDocument{\pdfstringdefDisableCommands{%
 % all "protected" macros possibly ending up in section titles should be here
+% TODO: examine if \sphinxhref, \sphinxurl, \sphinnolinkurl should be handled
     \let\sphinxstyleemphasis        \@firstofone
     \let\sphinxstyleliteralemphasis \@firstofone
     \let\sphinxstylestrong          \@firstofone
@@ -1879,8 +1887,18 @@
     \let\sphinxemail    \@firstofone
     \let\sphinxcrossref \@firstofone
     \let\sphinxtermref  \@firstofone
+    \def\sphinxhyphen#1{-}% the #1 is there to gobble the {}
 }}
 
+% Special characters
+% This definition prevents ligatures with non-Unicode engines
+% (the escaping is not done for Unicode engines, as ligatures are prevented
+% otherwise).
+% It also allows in general text a breakpoint after the hyphen. This is
+% to keep in sync with behavior in code-blocks, parsed and inline literals.
+% For a breakpoint before the hyphen modify this to \leavevmode\kern\z@-
+\protected\def\sphinxhyphen#1{-\kern\z@}
+
 % For curly braces inside \index macro
 \def\sphinxleftcurlybrace{\{}
 \def\sphinxrightcurlybrace{\}}

sphinx/texinputs/sphinx.xdy

@@ -115,6 +115,7 @@
 (merge-rule "\textbackslash{}" "\" :string)  ; " for Emacs syntax highlighting
 (merge-rule "\textasciitilde{}" "~~" :string); the ~~ escape is needed here
 (merge-rule "\textasciicircum{}" "^" :string)
+(merge-rule "\sphinxhyphen{}" "-" :string)
 (merge-rule "\textquotesingle{}" "'" :string)
 (merge-rule "\textasciigrave{}" "`" :string)
 (merge-rule "\textless{}" "<" :string)

sphinx/util/texescape.py

@@ -49,8 +49,8 @@
     # Note: the " renders curly in OT1 encoding but straight in T1, T2A, LY1...
     #       escaping it to \textquotedbl would break documents using OT1
     #       Sphinx does \shorthandoff{"} to avoid problems with some languages
-    # Can't replace - by {}- because it breaks URLs
-    # ('-', '{}-'),  # -- and --- are TeX ligatures
+    # There is no \text... LaTeX escape for the hyphen character -
+    ('-', r'\sphinxhyphen{}'),  # -- and --- are TeX ligatures
     # ,, is a TeX ligature in T1 encoding, but escaping the comma adds
     # complications (whether by {}, or a macro) and is not done
     # the next two require textcomp package
@@ -96,6 +96,11 @@
     ('₉', r'\(\sb{\text{9}}\)'),
 ]
 
+# TODO: this should be called tex_idescape_map because its only use is in
+#       sphinx.writers.latex.LaTeXTranslator.idescape()
+# %, {, }, \, #, and ~ are the only ones which must be replaced by _ character
+# It would be simpler to define it entirely here rather than in init().
+# Unicode replacements are superfluous, as idescape() uses backslashreplace
 tex_replace_map = {}  # type: Dict[int, str]
 
 _tex_escape_map = {}  # type: Dict[int, str]
@@ -143,10 +148,10 @@ def init() -> None:
 
     for a, b in ascii_tex_replacements:
         _tex_escape_map[ord(a)] = b
-        tex_replace_map[ord(a)] = '_'
 
     for a, b in unicode_tex_replacements:
         _tex_escape_map[ord(a)] = b
+        #  This is actually unneeded:
         tex_replace_map[ord(a)] = '_'
 
     for a, b in tex_replacements:

sphinx/writers/latex.py

@@ -487,7 +487,7 @@ def __init__(self, document: nodes.document, builder: "LaTeXBuilder") -> None:
         self.first_document = 1
         self.this_is_the_title = 1
         self.literal_whitespace = 0
-        self.no_contractions = 0
+        self.no_contractions = 0  # not used
         self.in_parsed_literal = 0
         self.compact_list = 0
         self.first_param = 0
@@ -991,13 +991,11 @@ def depart_desc_returns(self, node: Element) -> None:
 
     def visit_desc_name(self, node: Element) -> None:
         self.body.append(r'\sphinxbfcode{\sphinxupquote{')
-        self.no_contractions += 1
         self.literal_whitespace += 1
 
     def depart_desc_name(self, node: Element) -> None:
         self.body.append('}}')
         self.literal_whitespace -= 1
-        self.no_contractions -= 1
 
     def visit_desc_parameterlist(self, node: Element) -> None:
         # close name, open parameterlist
@@ -1843,11 +1841,9 @@ def depart_emphasis(self, node: Element) -> None:
 
     def visit_literal_emphasis(self, node: Element) -> None:
         self.body.append(r'\sphinxstyleliteralemphasis{\sphinxupquote{')
-        self.no_contractions += 1
 
     def depart_literal_emphasis(self, node: Element) -> None:
         self.body.append('}}')
-        self.no_contractions -= 1
 
     def visit_strong(self, node: Element) -> None:
         self.body.append(r'\sphinxstylestrong{')
@@ -1857,11 +1853,9 @@ def depart_strong(self, node: Element) -> None:
 
     def visit_literal_strong(self, node: Element) -> None:
         self.body.append(r'\sphinxstyleliteralstrong{\sphinxupquote{')
-        self.no_contractions += 1
 
     def depart_literal_strong(self, node: Element) -> None:
         self.body.append('}}')
-        self.no_contractions -= 1
 
     def visit_abbreviation(self, node: Element) -> None:
         abbr = node.astext()
@@ -1921,14 +1915,12 @@ def depart_citation_reference(self, node: Element) -> None:
         pass
 
     def visit_literal(self, node: Element) -> None:
-        self.no_contractions += 1
         if self.in_title:
             self.body.append(r'\sphinxstyleliteralintitle{\sphinxupquote{')
         else:
             self.body.append(r'\sphinxcode{\sphinxupquote{')
 
     def depart_literal(self, node: Element) -> None:
-        self.no_contractions -= 1
         self.body.append('}}')
 
     def visit_footnote_reference(self, node: Element) -> None:
@@ -2102,9 +2094,7 @@ def depart_option_list_item(self, node: Element) -> None:
 
     def visit_option_string(self, node: Element) -> None:
         ostring = node.astext()
-        self.no_contractions += 1
         self.body.append(self.encode(ostring))
-        self.no_contractions -= 1
         raise nodes.SkipNode
 
     def visit_description(self, node: Element) -> None:
@@ -2188,14 +2178,15 @@ def encode(self, text: str) -> str:
             # Insert a blank before the newline, to avoid
             # ! LaTeX Error: There's no line here to end.
             text = text.replace('\n', '~\\\\\n').replace(' ', '~')
-        if self.no_contractions:
-            text = text.replace('--', '-{-}')
-            text = text.replace("''", "'{'}")
         return text
 
     def encode_uri(self, text: str) -> str:
+        # TODO: it is probably wrong that this uses texescape.escape()
+        #       this must be checked against hyperref package exact dealings
+        #       mainly, %, #, {, } and \ need escaping via a \ escape
         # in \href, the tilde is allowed and must be represented literally
-        return self.encode(text).replace('\\textasciitilde{}', '~')
+        return self.encode(text).replace('\\textasciitilde{}', '~').\
+            replace('\\sphinxhyphen{}', '-')
 
     def visit_Text(self, node: Text) -> None:
         text = self.encode(node.astext())

tests/roots/test-latex-table/expects/complex_spanning_cell.tex

@@ -2,10 +2,10 @@
 table having …
 \begin{itemize}
 \item {} 
-consecutive multirow at top of row (1-1 and 1-2)
+consecutive multirow at top of row (1\sphinxhyphen{}1 and 1\sphinxhyphen{}2)
 
 \item {} 
-consecutive multirow at end of row (1-4 and 1-5)
+consecutive multirow at end of row (1\sphinxhyphen{}4 and 1\sphinxhyphen{}5)
 
 \end{itemize}
 
@@ -16,40 +16,40 @@
 \hline
 \sphinxmultirow{3}{1}{%
 \begin{varwidth}[t]{\sphinxcolwidth{1}{5}}
-cell1-1
+cell1\sphinxhyphen{}1
 \par
 \vskip-\baselineskip\vbox{\hbox{\strut}}\end{varwidth}%
 }%
 &\sphinxmultirow{3}{2}{%
 \begin{varwidth}[t]{\sphinxcolwidth{1}{5}}
-cell1-2
+cell1\sphinxhyphen{}2
 \par
 \vskip-\baselineskip\vbox{\hbox{\strut}}\end{varwidth}%
 }%
 &
-cell1-3
+cell1\sphinxhyphen{}3
 &\sphinxmultirow{3}{4}{%
 \begin{varwidth}[t]{\sphinxcolwidth{1}{5}}
-cell1-4
+cell1\sphinxhyphen{}4
 \par
 \vskip-\baselineskip\vbox{\hbox{\strut}}\end{varwidth}%
 }%
 &\sphinxmultirow{2}{5}{%
 \begin{varwidth}[t]{\sphinxcolwidth{1}{5}}
-cell1-5
+cell1\sphinxhyphen{}5
 \par
 \vskip-\baselineskip\vbox{\hbox{\strut}}\end{varwidth}%
 }%
 \\
 \cline{3-3}\sphinxtablestrut{1}&\sphinxtablestrut{2}&\sphinxmultirow{2}{6}{%
 \begin{varwidth}[t]{\sphinxcolwidth{1}{5}}
-cell2-3
+cell2\sphinxhyphen{}3
 \par
 \vskip-\baselineskip\vbox{\hbox{\strut}}\end{varwidth}%
 }%
 &\sphinxtablestrut{4}&\sphinxtablestrut{5}\\
 \cline{5-5}\sphinxtablestrut{1}&\sphinxtablestrut{2}&\sphinxtablestrut{6}&\sphinxtablestrut{4}&
-cell3-5
+cell3\sphinxhyphen{}5
 \\
 \hline
 \end{tabulary}

tests/roots/test-latex-table/expects/gridtable.tex

@@ -12,40 +12,40 @@
 header3
 \\
 \hline
-cell1-1
+cell1\sphinxhyphen{}1
 &\sphinxmultirow{2}{5}{%
 \begin{varwidth}[t]{\sphinxcolwidth{1}{3}}
-cell1-2
+cell1\sphinxhyphen{}2
 \par
 \vskip-\baselineskip\vbox{\hbox{\strut}}\end{varwidth}%
 }%
 &
-cell1-3
+cell1\sphinxhyphen{}3
 \\
 \cline{1-1}\cline{3-3}\sphinxmultirow{2}{7}{%
 \begin{varwidth}[t]{\sphinxcolwidth{1}{3}}
-cell2-1
+cell2\sphinxhyphen{}1
 \par
 \vskip-\baselineskip\vbox{\hbox{\strut}}\end{varwidth}%
 }%
 &\sphinxtablestrut{5}&
-cell2-3
+cell2\sphinxhyphen{}3
 \\
 \cline{2-3}\sphinxtablestrut{7}&\sphinxstartmulticolumn{2}%
 \sphinxmultirow{2}{9}{%
 \begin{varwidth}[t]{\sphinxcolwidth{2}{3}}
-cell3-2
+cell3\sphinxhyphen{}2
 \par
 \vskip-\baselineskip\vbox{\hbox{\strut}}\end{varwidth}%
 }%
 \sphinxstopmulticolumn
 \\
 \cline{1-1}
-cell4-1
+cell4\sphinxhyphen{}1
 &\multicolumn{2}{l|}{\sphinxtablestrut{9}}\\
 \hline\sphinxstartmulticolumn{3}%
 \begin{varwidth}[t]{\sphinxcolwidth{3}{3}}
-cell5-1
+cell5\sphinxhyphen{}1
 \par
 \vskip-\baselineskip\vbox{\hbox{\strut}}\end{varwidth}%
 \sphinxstopmulticolumn

tests/roots/test-latex-table/expects/longtable.tex

@@ -27,19 +27,19 @@
 
 \endlastfoot
 
-cell1-1
+cell1\sphinxhyphen{}1
 &
-cell1-2
+cell1\sphinxhyphen{}2
 \\
 \hline
-cell2-1
+cell2\sphinxhyphen{}1
 &
-cell2-2
+cell2\sphinxhyphen{}2
 \\
 \hline
-cell3-1
+cell3\sphinxhyphen{}1
 &
-cell3-2
+cell3\sphinxhyphen{}2
 \\
 \hline
 \end{longtable}\sphinxatlongtableend\end{savenotes}

tests/roots/test-latex-table/expects/longtable_having_align.tex

@@ -27,19 +27,19 @@
 
 \endlastfoot
 
-cell1-1
+cell1\sphinxhyphen{}1
 &
-cell1-2
+cell1\sphinxhyphen{}2
 \\
 \hline
-cell2-1
+cell2\sphinxhyphen{}1
 &
-cell2-2
+cell2\sphinxhyphen{}2
 \\
 \hline
-cell3-1
+cell3\sphinxhyphen{}1
 &
-cell3-2
+cell3\sphinxhyphen{}2
 \\
 \hline
 \end{longtable}\sphinxatlongtableend\end{savenotes}

tests/roots/test-latex-table/expects/longtable_having_caption.tex

@@ -29,19 +29,19 @@
 
 \endlastfoot
 
-cell1-1
+cell1\sphinxhyphen{}1
 &
-cell1-2
+cell1\sphinxhyphen{}2
 \\
 \hline
-cell2-1
+cell2\sphinxhyphen{}1
 &
-cell2-2
+cell2\sphinxhyphen{}2
 \\
 \hline
-cell3-1
+cell3\sphinxhyphen{}1
 &
-cell3-2
+cell3\sphinxhyphen{}2
 \\
 \hline
 \end{longtable}\sphinxatlongtableend\end{savenotes}