🐛 FIX: front-matter rendering with docutils (#477)

For the docutils base HTML writer, it is expected that `field_body` nodes contain block elements (e.g. `paragraph`) as children,
rather than the `Text` node previously created by `DocutilsRenderer.dict_to_fm_field_list`.

.github/workflows/tests.yml

@@ -83,7 +83,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install .
-        pip install docutils==${{ matrix.docutils-version }}
+        pip install pytest~=6.2 docutils==${{ matrix.docutils-version }}
     - name: ensure sphinx is not installed
       run: |
         python -c "\
@@ -93,6 +93,8 @@ jobs:
             pass
         else:
             raise AssertionError()"
+    - name: Run pytest for docutils-only tests
+      run: pytest tests/test_docutils.py tests/test_renderers/test_fixtures_docutils.py
     - name: Run docutils CLI
       run: echo "test" | myst-docutils-html
 

myst_parser/docutils_renderer.py

@@ -730,6 +730,7 @@ def dict_to_fm_field_list(
 
         """
         field_list = nodes.field_list()
+        field_list.source, field_list.line = self.document["source"], line
 
         bibliofields = get_language(language_code).bibliographic_fields
         state_machine = MockStateMachine(self, line)
@@ -741,9 +742,12 @@ def dict_to_fm_field_list(
             value = str(value)
             if key in bibliofields:
                 para_nodes, _ = state.inline_text(value, line)
-                body_children = [nodes.paragraph("", "", *para_nodes)]
             else:
-                body_children = [nodes.Text(value, value)]
+                para_nodes = [nodes.Text(value, value)]
+
+            body_children = [nodes.paragraph("", "", *para_nodes)]
+            body_children[0].source = self.document["source"]
+            body_children[0].line = 0
 
             field_node = nodes.field()
             field_node.source = value

tests/conftest.py

@@ -1 +1,8 @@
-pytest_plugins = "sphinx.testing.fixtures"
+"""Top-level configuration for pytest."""
+try:
+    import sphinx  # noqa: F401
+except ImportError:
+    pass
+else:
+    # only use when Sphinx is installed, to allow testing myst-docutils
+    pytest_plugins = "sphinx.testing.fixtures"

tests/test_renderers/fixtures/syntax_elements.md

@@ -631,17 +631,20 @@ c:
             <field_name>
                 a
             <field_body>
-                1
+                <paragraph>
+                    1
         <field>
             <field_name>
                 b
             <field_body>
-                foo
+                <paragraph>
+                    foo
         <field>
             <field_name>
                 c
             <field_body>
-                {"d": 2}
+                <paragraph>
+                    {"d": 2}
 .
 
 --------------------------
@@ -758,7 +761,8 @@ other: Something else
             <field_name>
                 other
             <field_body>
-                Something else
+                <paragraph>
+                    Something else
 .
 
 --------------------------
@@ -859,7 +863,8 @@ a = 1
             <field_name>
                 a
             <field_body>
-                1
+                <paragraph>
+                    1
     <target ids="target" names="target">
     <section ids="header-1" names="header\ 1">
         <title>

tests/test_renderers/test_fixtures_docutils.py

@@ -0,0 +1,58 @@
+"""Test fixture files, using the ``DocutilsRenderer``.
+
+Note, the output AST is before any transforms are applied.
+"""
+from pathlib import Path
+
+import pytest
+from markdown_it.utils import read_fixture_file
+
+from myst_parser.docutils_renderer import DocutilsRenderer, make_document
+from myst_parser.main import MdParserConfig, create_md_parser
+
+FIXTURE_PATH = Path(__file__).parent.joinpath("fixtures")
+
+
+@pytest.mark.parametrize(
+    "line,title,input,expected",
+    read_fixture_file(FIXTURE_PATH.joinpath("docutil_roles.md")),
+    ids=[
+        f"{i[0]}-{i[1]}" for i in read_fixture_file(FIXTURE_PATH / "docutil_roles.md")
+    ],
+)
+def test_docutils_roles(line, title, input, expected):
+    """Test output of docutils roles."""
+    parser = create_md_parser(MdParserConfig(), DocutilsRenderer)
+    parser.options["document"] = document = make_document()
+    parser.render(input)
+    try:
+        assert "\n".join(
+            [ll.rstrip() for ll in document.pformat().splitlines()]
+        ) == "\n".join([ll.rstrip() for ll in expected.splitlines()])
+    except AssertionError:
+        print(document.pformat())
+        raise
+
+
+@pytest.mark.parametrize(
+    "line,title,input,expected",
+    read_fixture_file(FIXTURE_PATH.joinpath("docutil_directives.md")),
+    ids=[
+        f"{i[0]}-{i[1]}"
+        for i in read_fixture_file(FIXTURE_PATH / "docutil_directives.md")
+    ],
+)
+def test_docutils_directives(line, title, input, expected):
+    """Test output of docutils directives."""
+    if title.startswith("SKIP"):  # line-block directive not yet supported
+        pytest.skip(title)
+    parser = create_md_parser(MdParserConfig(), DocutilsRenderer)
+    parser.options["document"] = document = make_document()
+    parser.render(input)
+    try:
+        assert "\n".join(
+            [ll.rstrip() for ll in document.pformat().splitlines()]
+        ) == "\n".join([ll.rstrip() for ll in expected.splitlines()])
+    except AssertionError:
+        print(document.pformat())
+        raise

tests/test_renderers/test_fixtures.py → tests/test_renderers/test_fixtures_sphinx.py

@@ -1,3 +1,7 @@
+"""Test fixture files, using the ``SphinxRenderer``.
+
+Note, the output AST is before any transforms are applied.
+"""
 import re
 from pathlib import Path
 
@@ -60,41 +64,6 @@ def test_directive_options(line, title, input, expected):
     ) == "\n".join([ll.rstrip() for ll in expected.splitlines()])
 
 
-@pytest.mark.parametrize(
-    "line,title,input,expected",
-    read_fixture_file(FIXTURE_PATH.joinpath("docutil_roles.md")),
-    ids=[
-        f"{i[0]}-{i[1]}" for i in read_fixture_file(FIXTURE_PATH / "docutil_roles.md")
-    ],
-)
-def test_docutils_roles(line, title, input, expected):
-    document = to_docutils(input)
-    print(document.pformat())
-    assert "\n".join(
-        [ll.rstrip() for ll in document.pformat().splitlines()]
-    ) == "\n".join([ll.rstrip() for ll in expected.splitlines()])
-
-
-@pytest.mark.parametrize(
-    "line,title,input,expected",
-    read_fixture_file(FIXTURE_PATH.joinpath("docutil_directives.md")),
-    ids=[
-        f"{i[0]}-{i[1]}"
-        for i in read_fixture_file(FIXTURE_PATH / "docutil_directives.md")
-    ],
-)
-def test_docutils_directives(line, title, input, expected):
-    # TODO fix skipped directives
-    # TODO test domain directives
-    if title.startswith("SKIP"):
-        pytest.skip(title)
-    document = to_docutils(input)
-    print(document.pformat())
-    assert "\n".join(
-        [ll.rstrip() for ll in document.pformat().splitlines()]
-    ) == "\n".join([ll.rstrip() for ll in expected.splitlines()])
-
-
 @pytest.mark.parametrize(
     "line,title,input,expected",
     read_fixture_file(FIXTURE_PATH.joinpath("sphinx_directives.md")),