From 06e409d94a8eae8cfed34e496f1413e05bb2e21d Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Tue, 4 Jan 2022 18:29:23 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20Source=20line=20reporting?= =?UTF-8?q?=20for=20nested=20parsing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, nested parsing of content within directives would not correctly propagate the starting line number of the content. --- myst_parser/docutils_renderer.py | 15 ++++++++++++--- myst_parser/mocking.py | 4 ++-- myst_parser/parse_directives.py | 15 +++++++++------ .../fixtures/mock_include_errors.md | 2 +- .../test_renderers/fixtures/reporter_warnings.md | 16 +++++++++++++++- tests/test_renderers/test_parse_directives.py | 16 +++++++++++++--- .../test_parsing_Note___class__name_n_na_.yml | 7 +++++++ .../test_parsing_Note__a_.yml | 1 + .../test_parsing_Note_a__.yml | 1 + 9 files changed, 61 insertions(+), 16 deletions(-) create mode 100644 tests/test_renderers/test_parse_directives/test_parsing_Note___class__name_n_na_.yml diff --git a/myst_parser/docutils_renderer.py b/myst_parser/docutils_renderer.py index 0207ac21..9caf091c 100644 --- a/myst_parser/docutils_renderer.py +++ b/myst_parser/docutils_renderer.py @@ -262,10 +262,19 @@ def add_document_wordcount(self) -> None: self.document.note_substitution_def(substitution_node, f"wordcount-{key}") def nested_render_text(self, text: str, lineno: int) -> None: - """Render unparsed text.""" + """Render unparsed text. + + :param text: the text to render + :param lineno: the starting line number of the text, within the full source + """ tokens = self.md.parse(text + "\n", self.md_env) + # update the line numbers + for token in tokens: + if token.map: + token.map = [token.map[0] + lineno, token.map[1] + lineno] + # remove front matter if tokens and tokens[0].type == "front_matter": tokens.pop(0) @@ -1139,7 +1148,7 @@ def run_directive( directive_class.option_spec["relative-docs"] = directives.path try: - arguments, options, body_lines = parse_directive_text( + arguments, options, body_lines, content_offset = parse_directive_text( directive_class, first_line, content ) except DirectiveParsingError as error: @@ -1175,7 +1184,7 @@ def run_directive( # the absolute line number of the first line of the directive lineno=position, # the line offset of the first line of the content - content_offset=0, # TODO get content offset from `parse_directive_text` + content_offset=content_offset, # a string containing the entire directive block_text="\n".join(body_lines), state=state, diff --git a/myst_parser/mocking.py b/myst_parser/mocking.py index 382596e2..67ad4192 100644 --- a/myst_parser/mocking.py +++ b/myst_parser/mocking.py @@ -114,14 +114,14 @@ def parse_directive_block( if option_presets: raise MockingError("parse_directive_block: option_presets not implemented") # TODO should argument_str always be ""? - arguments, options, body_lines = parse_directive_text( + arguments, options, body_lines, content_offset = parse_directive_text( directive, "", "\n".join(content) ) return ( arguments, options, StringList(body_lines, source=content.source), - line_offset + len(content) - len(body_lines), + line_offset + content_offset, ) def nested_parse( diff --git a/myst_parser/parse_directives.py b/myst_parser/parse_directives.py index 64d0e381..b053deae 100644 --- a/myst_parser/parse_directives.py +++ b/myst_parser/parse_directives.py @@ -36,7 +36,7 @@ import datetime import re from textwrap import dedent -from typing import Any, Callable, Dict, Type +from typing import Any, Callable, Dict, List, Tuple, Type import yaml from docutils.parsers.rst import Directive @@ -54,7 +54,7 @@ def parse_directive_text( first_line: str, content: str, validate_options: bool = True, -): +) -> Tuple[List[str], dict, List[str], int]: """Parse (and validate) the full directive text. :param first_line: The text on the same line as the directive name. @@ -62,17 +62,19 @@ def parse_directive_text( :param content: All text after the first line. Can include options. :param validate_options: Whether to validate the values of options + :returns: (arguments, options, body_lines, content_offset) """ if directive_class.option_spec: body, options = parse_directive_options( content, directive_class, validate=validate_options ) + body_lines = body.splitlines() + content_offset = len(content.splitlines()) - len(body_lines) else: # If there are no possible options, we do not look for a YAML block options = {} - body = content - - body_lines = body.splitlines() + body_lines = content.splitlines() + content_offset = 0 if not ( directive_class.required_arguments @@ -91,12 +93,13 @@ def parse_directive_text( # this is to allow space between the options and the content if body_lines and not body_lines[0].strip(): body_lines = body_lines[1:] + content_offset += 1 # check for body content if body_lines and not directive_class.has_content: raise DirectiveParsingError("No content permitted") - return arguments, options, body_lines + return arguments, options, body_lines, content_offset def parse_directive_options( diff --git a/tests/test_renderers/fixtures/mock_include_errors.md b/tests/test_renderers/fixtures/mock_include_errors.md index de35edd5..1e5c2e9b 100644 --- a/tests/test_renderers/fixtures/mock_include_errors.md +++ b/tests/test_renderers/fixtures/mock_include_errors.md @@ -20,5 +20,5 @@ Error in include file: ```{include} bad.md ``` . -tmpdir/bad.md:1: (ERROR/3) Unknown interpreted text role "a". +tmpdir/bad.md:2: (ERROR/3) Unknown interpreted text role "a". . diff --git a/tests/test_renderers/fixtures/reporter_warnings.md b/tests/test_renderers/fixtures/reporter_warnings.md index bda330d7..2fd6d677 100644 --- a/tests/test_renderers/fixtures/reporter_warnings.md +++ b/tests/test_renderers/fixtures/reporter_warnings.md @@ -136,5 +136,19 @@ header nested in admonition # Header ``` . -:1: (WARNING/2) Header nested in this element can lead to unexpected outcomes +:2: (WARNING/2) Header nested in this element can lead to unexpected outcomes +. + +nested parse warning +. +Paragraph + +```{note} +:class: abc +:name: xyz + +{unknown}`a` +``` +. +:7: (ERROR/3) Unknown interpreted text role "unknown". . diff --git a/tests/test_renderers/test_parse_directives.py b/tests/test_renderers/test_parse_directives.py index 07e4c0f2..c85e6637 100644 --- a/tests/test_renderers/test_parse_directives.py +++ b/tests/test_renderers/test_parse_directives.py @@ -6,11 +6,21 @@ from myst_parser.parse_directives import DirectiveParsingError, parse_directive_text -@pytest.mark.parametrize("klass,arguments,content", [(Note, "", "a"), (Note, "a", "")]) +@pytest.mark.parametrize( + "klass,arguments,content", + [(Note, "", "a"), (Note, "a", ""), (Note, "", ":class: name\n\na")], +) def test_parsing(klass, arguments, content, data_regression): - arguments, options, body_lines = parse_directive_text(klass, arguments, content) + arguments, options, body_lines, content_offset = parse_directive_text( + klass, arguments, content + ) data_regression.check( - {"arguments": arguments, "options": options, "body": body_lines} + { + "arguments": arguments, + "options": options, + "body": body_lines, + "content_offset": content_offset, + } ) diff --git a/tests/test_renderers/test_parse_directives/test_parsing_Note___class__name_n_na_.yml b/tests/test_renderers/test_parse_directives/test_parsing_Note___class__name_n_na_.yml new file mode 100644 index 00000000..8b9bbdf3 --- /dev/null +++ b/tests/test_renderers/test_parse_directives/test_parsing_Note___class__name_n_na_.yml @@ -0,0 +1,7 @@ +arguments: [] +body: +- a +content_offset: 2 +options: + class: + - name diff --git a/tests/test_renderers/test_parse_directives/test_parsing_Note__a_.yml b/tests/test_renderers/test_parse_directives/test_parsing_Note__a_.yml index f4a55171..fd1e5585 100644 --- a/tests/test_renderers/test_parse_directives/test_parsing_Note__a_.yml +++ b/tests/test_renderers/test_parse_directives/test_parsing_Note__a_.yml @@ -1,4 +1,5 @@ arguments: [] body: - a +content_offset: 0 options: {} diff --git a/tests/test_renderers/test_parse_directives/test_parsing_Note_a__.yml b/tests/test_renderers/test_parse_directives/test_parsing_Note_a__.yml index f4a55171..fd1e5585 100644 --- a/tests/test_renderers/test_parse_directives/test_parsing_Note_a__.yml +++ b/tests/test_renderers/test_parse_directives/test_parsing_Note_a__.yml @@ -1,4 +1,5 @@ arguments: [] body: - a +content_offset: 0 options: {}