Make code role highlighting consistent with code-block directive

Fixes #5157

This is factored out of the sphinx-immaterial theme:
https://github.com/jbms/sphinx-immaterial/blob/1ef121a612d4f5afc2a9ca9c4e3f20fca89065e8/sphinx_immaterial/inlinesyntaxhighlight.py#L1

See also:
#6916

sphinx/roles.py

@@ -11,6 +11,9 @@
 import re
 from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Type
 
+import docutils.parsers.rst.directives
+import docutils.parsers.rst.roles
+import docutils.parsers.rst.states
 from docutils import nodes, utils
 from docutils.nodes import Element, Node, TextElement, system_message
 
@@ -341,6 +344,53 @@ def run(self) -> Tuple[List[Node], List[system_message]]:
         return [nodes.abbreviation(self.rawtext, text, **options)], []
 
 
+# Sphinx provides the `code-block` directive for highlighting code blocks.
+# Docutils provides the `code` role which in theory can be used similarly by
+# defining a custom role for a given programming language:
+#
+#     .. .. role:: python(code)
+#          :language: python
+#          :class: highlight
+#
+# In practice this does not produce correct highlighting because it uses a
+# separate highlighting mechanism that results in the "long" pygments class
+# names rather than "short" pygments class names produced by the Sphinx
+# `code-block` directive and for which this extension contains CSS rules.
+#
+# In addition, even if that issue is fixed, because the highlighting
+# implementation in docutils, despite being based on pygments, differs from that
+# used by Sphinx, the output does not exactly match that produced by the Sphinx
+# `code-block` directive.
+#
+# This issue is noted here: //github.com/sphinx-doc/sphinx/issues/5157
+#
+# This overrides the docutils `code` role to perform highlighting in the same
+# way as the Sphinx `code-block` directive.
+def code_role(name: str, rawtext: str, text: str, lineno: int,
+              inliner: docutils.parsers.rst.states.Inliner,
+              options: Dict = {}, content: List[str] = []
+              ) -> Tuple[List[Node], List[system_message]]:
+    options = options.copy()
+    docutils.parsers.rst.roles.set_classes(options)
+    language = options.get('language', '')
+    classes = ['code']
+    if 'classes' in options:
+        classes.extend(options['classes'])
+
+    if language and language not in classes:
+        classes.append(language)
+
+    node = nodes.literal(rawtext, text, classes=classes, language=language)
+
+    return [node], []
+
+
+code_role.options = {  # type: ignore
+    'class': docutils.parsers.rst.directives.class_option,
+    'language': docutils.parsers.rst.directives.unchanged,
+}
+
+
 specific_docroles: Dict[str, RoleFunction] = {
     # links to download references
     'download': XRefRole(nodeclass=addnodes.download_reference),
@@ -368,6 +418,10 @@ def setup(app: "Sphinx") -> Dict[str, Any]:
     for rolename, func in specific_docroles.items():
         roles.register_local_role(rolename, func)
 
+    # Since docutils registers it as a canonical role, override it as a
+    # canonical role as well.
+    roles.register_canonical_role('code', code_role)
+
     return {
         'version': 'builtin',
         'parallel_read_safe': True,

sphinx/writers/html.py

@@ -497,10 +497,31 @@ def visit_literal(self, node: Element) -> None:
         if 'kbd' in node['classes']:
             self.body.append(self.starttag(node, 'kbd', '',
                                            CLASS='docutils literal notranslate'))
-        else:
+            return
+        lang = node.get("language", None)
+        if 'code' not in node['classes'] or not lang:
             self.body.append(self.starttag(node, 'code', '',
                                            CLASS='docutils literal notranslate'))
             self.protect_literal_text += 1
+            return
+
+        def warner(msg):
+            logger.warning(msg, location=node)
+
+        highlight_args = dict(node.get("highlight_args", {}), nowrap=True)
+        opts = self.config.highlight_options.get(lang, {})
+        highlighted = self.highlighter.highlight_block(
+            node.astext(), lang, opts=opts, warn=warner, location=node,
+            **highlight_args
+        )
+        starttag = self.starttag(
+            node,
+            "code",
+            suffix="",
+            CLASS="docutils literal highlight highlight-%s" % lang,
+        )
+        self.body.append(starttag + highlighted.strip() + "</code>")
+        raise nodes.SkipNode
 
     def depart_literal(self, node: Element) -> None:
         if 'kbd' in node['classes']:

sphinx/writers/latex.py

@@ -1747,10 +1747,31 @@ def depart_citation_reference(self, node: Element) -> None:
     def visit_literal(self, node: Element) -> None:
         if self.in_title:
             self.body.append(r'\sphinxstyleliteralintitle{\sphinxupquote{')
+            return
         elif 'kbd' in node['classes']:
             self.body.append(r'\sphinxkeyboard{\sphinxupquote{')
-        else:
+            return
+        lang = node.get("language", None)
+        if 'code' not in node['classes'] or not lang:
             self.body.append(r'\sphinxcode{\sphinxupquote{')
+            return
+
+        def warner(msg):
+            logger.warning(msg, location=node)
+
+        highlight_args = dict(node.get("highlight_args", {}), nowrap=True)
+        opts = self.config.highlight_options.get(lang, {})
+        hlcode = self.highlighter.highlight_block(
+            node.astext(), lang, opts=opts,
+            location=node, **highlight_args
+        )
+        hlcode = hlcode.replace(r'\begin{Verbatim}[commandchars=\\\{\}]',
+                                r'\sphinxcode{\sphinxupquote{')
+        # get consistent trailer
+        hlcode = hlcode.rstrip()[:-14]  # strip \end{Verbatim}
+        self.body.append(hlcode)
+        self.body.append('}}')
+        raise nodes.SkipNode
 
     def depart_literal(self, node: Element) -> None:
         self.body.append('}}')

tests/roots/test-reST-code-role/index.rst

@@ -0,0 +1,9 @@
+.. role:: python(code)
+   :language: python
+   :class: highlight
+
+Inline :python:`def foo(1 + 2 + None + "abc"): pass` code block
+
+.. code-block:: python
+
+   def foo(1 + 2 + None + "abc"): pass

tests/test_build_html.py

@@ -1718,3 +1718,27 @@ def test_html_signaturereturn_icon(app):
     content = (app.outdir / 'index.html').read_text()
 
     assert ('<span class="sig-return-icon">&#x2192;</span>' in content)
+
+
+@pytest.mark.sphinx('html', testroot='reST-code-role')
+def test_html_code_role(app):
+    app.build()
+    content = (app.outdir / 'index.html').read_text()
+
+    common_content = (
+        '<span class="k">def</span> <span class="nf">foo</span>'
+        '<span class="p">(</span>'
+        '<span class="mi">1</span> '
+        '<span class="o">+</span> '
+        '<span class="mi">2</span> '
+        '<span class="o">+</span> '
+        '<span class="kc">None</span> '
+        '<span class="o">+</span> '
+        '<span class="s2">&quot;abc&quot;</span>'
+        '<span class="p">):</span> '
+        '<span class="k">pass</span>')
+    assert ('<p>Inline <code class="code highlight python docutils literal highlight-python">' +
+            common_content + '</code> code block</p>') in content
+    assert ('<div class="highlight-python notranslate">' +
+            '<div class="highlight"><pre><span></span>' +
+            common_content) in content

tests/test_build_latex.py

@@ -1628,3 +1628,30 @@ def test_latex_container(app, status, warning):
     result = (app.outdir / 'python.tex').read_text()
     assert r'\begin{sphinxuseclass}{classname}' in result
     assert r'\end{sphinxuseclass}' in result
+
+
+@pytest.mark.sphinx('latex', testroot='reST-code-role')
+def test_latex_code_role(app):
+    app.build()
+    content = (app.outdir / 'python.tex').read_text()
+
+    common_content = (
+        r'\PYG{k}{def} '
+        r'\PYG{n+nf}{foo}'
+        r'\PYG{p}{(}'
+        r'\PYG{l+m+mi}{1} '
+        r'\PYG{o}{+} '
+        r'\PYG{l+m+mi}{2} '
+        r'\PYG{o}{+} '
+        r'\PYG{k+kc}{None} '
+        r'\PYG{o}{+} '
+        r'\PYG{l+s+s2}{\PYGZdq{}}'
+        r'\PYG{l+s+s2}{abc}'
+        r'\PYG{l+s+s2}{\PYGZdq{}}'
+        r'\PYG{p}{)}'
+        r'\PYG{p}{:} '
+        r'\PYG{k}{pass}')
+    assert (r'Inline \sphinxcode{\sphinxupquote{' + '\n' +
+            common_content + '\n}} code block') in content
+    assert (r'\begin{sphinxVerbatim}[commandchars=\\\{\}]' +
+            '\n' + common_content + '\n' + r'\end{sphinxVerbatim}') in content