From d06903fa8059fecaeabedc9dd126f8e62e0d07d0 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Wed, 29 Dec 2021 02:28:31 +0100 Subject: [PATCH 1/5] =?UTF-8?q?=F0=9F=91=8C=20IMPROVE:=20Code=20block=20hi?= =?UTF-8?q?ghlighting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/conf.py | 1 + docs/sphinx/reference.md | 3 + docs/syntax/reference.md | 8 +- docs/syntax/syntax.md | 89 +- myst_parser/__init__.py | 11 +- myst_parser/docutils_.py | 1 - myst_parser/docutils_renderer.py | 103 ++- myst_parser/main.py | 27 +- .../fixtures/docutil_syntax_elements.md | 864 ++++++++++++++++++ ..._elements.md => sphinx_syntax_elements.md} | 0 .../test_renderers/test_fixtures_docutils.py | 23 + tests/test_renderers/test_fixtures_sphinx.py | 5 +- 12 files changed, 1078 insertions(+), 57 deletions(-) create mode 100644 tests/test_renderers/fixtures/docutil_syntax_elements.md rename tests/test_renderers/fixtures/{syntax_elements.md => sphinx_syntax_elements.md} (100%) diff --git a/docs/conf.py b/docs/conf.py index 9f244fd8..f054c054 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -88,6 +88,7 @@ "substitution", "tasklist", ] +myst_number_code_blocks = ["typescript"] myst_heading_anchors = 2 myst_footnote_transition = True myst_dmath_double_inline = True diff --git a/docs/sphinx/reference.md b/docs/sphinx/reference.md index 1ba6ee5a..5d3235c8 100644 --- a/docs/sphinx/reference.md +++ b/docs/sphinx/reference.md @@ -38,6 +38,9 @@ To do so, use the keywords beginning `myst_`. * - `myst_heading_slug_func` - `None` - Use the specified function to auto-generate heading anchors, [see here](syntax/header-anchors) for details. +* - `myst_number_code_blocks` + - `()` + - Add line numbers to code blocks with these languages, [see here](syntax/code-blocks) for details. * - `myst_substitutions` - `{}` - A mapping of keys to substitutions, used globally for all MyST documents when the "substitution" extension is enabled. diff --git a/docs/syntax/reference.md b/docs/syntax/reference.md index c03226ba..91ff7b8d 100644 --- a/docs/syntax/reference.md +++ b/docs/syntax/reference.md @@ -120,12 +120,13 @@ we have shown equivalent rST syntax for many MyST markdown features below. ====== ``` * - Quote - - quoted text + - Quoted text - ```md > this is a quote ``` * - CodeFence - - enclosed in 3 or more backticks with an optional language name + - Enclosed in 3 or more `` ` `` or `~` with an optional language name. + See {ref}`syntax/code-blocks` for more information. - ````md ```python print('this is python') @@ -176,8 +177,7 @@ In addition to these summaries of inline syntax, see {ref}`extra-markdown-syntax - Description - Example * - Role - - See {ref}`syntax/roles` for more - information. + - See {ref}`syntax/roles` for more information. - ```md {rolename}`interpreted text` ``` diff --git a/docs/syntax/syntax.md b/docs/syntax/syntax.md index e92b2ba5..df27ea8c 100644 --- a/docs/syntax/syntax.md +++ b/docs/syntax/syntax.md @@ -610,6 +610,65 @@ leave the "text" section of the markdown link empty. For example, this markdown: `[](syntax.md)` will result in: [](syntax.md). ``` +(syntax/code-blocks)= +## Code blocks + +Code blocks contain a language identifier, which is used to determine the language of the code. +This language is used to determine the syntax highlighting, using an available [pygments lexer](https://pygments.org/docs/lexers/). + +````markdown +```python +from a import b +c = "string" +``` +```` + +```python +from a import b +c = "string" +``` + +You can create and register your own lexer, using the [`pygments.lexers` entry point](https://pygments.org/docs/plugins/#register-plugins), +or within a sphinx extension, with the [`app.add_lexer` method](sphinx:sphinx.application.Sphinx.add_lexer). + +Using the `myst_number_code_blocks` configuration option, you can also control whether code blocks are numbered by line. +For example, using `myst_number_code_blocks = ["typescript"]`: + +```typescript +type MyBool = true | false; + +interface User { + name: string; + id: number; +} +``` + +### Show backticks inside raw markdown blocks + +If you'd like to show backticks inside of your markdown, you can do so by nesting them +in backticks of a greater length. Markdown will treat the outer-most backticks as the +edges of the "raw" block and everything inside will show up. For example: + +``` `` `hi` `` ``` will be rendered as: `` `hi` `` + +and + +````` +```` +``` +hi +``` +```` +````` + +will be rendered as: + +```` +``` +hi +``` +```` + ## Tables Tables can be written using the standard [Github Flavoured Markdown syntax](https://github.github.com/gfm/#tables-extension-): @@ -746,33 +805,3 @@ This is because, in the current implementation, they may not be available to ref By default, a transition line (with a `footnotes` class) will be placed before any footnotes. This can be turned off by adding `myst_footnote_transition = False` to the config file. - - -## Code blocks - - -### Show backticks inside raw markdown blocks - -If you'd like to show backticks inside of your markdown, you can do so by nesting them -in backticks of a greater length. Markdown will treat the outer-most backticks as the -edges of the "raw" block and everything inside will show up. For example: - -``` `` `hi` `` ``` will be rendered as: `` `hi` `` - -and - -````` -```` -``` -hi -``` -```` -````` - -will be rendered as: - -```` -``` -hi -``` -```` diff --git a/myst_parser/__init__.py b/myst_parser/__init__.py index 7be42c57..d03229ed 100644 --- a/myst_parser/__init__.py +++ b/myst_parser/__init__.py @@ -33,9 +33,10 @@ def setup_sphinx(app: "Sphinx"): app.add_post_transform(MystReferenceResolver) - for name, default in MdParserConfig().as_dict().items(): - # TODO add types? - app.add_config_value(f"myst_{name}", default, "env") + for name, default, field in MdParserConfig().as_triple(): + if not field.metadata.get("docutils_only", False): + # TODO add types? + app.add_config_value(f"myst_{name}", default, "env") app.connect("builder-inited", create_myst_config) app.connect("builder-inited", override_mathjax) @@ -53,8 +54,8 @@ def create_myst_config(app): values = { name: app.config[f"myst_{name}"] - for name in MdParserConfig().as_dict().keys() - if name != "renderer" + for name, _, field in MdParserConfig().as_triple() + if not field.metadata.get("docutils_only", False) } try: diff --git a/myst_parser/docutils_.py b/myst_parser/docutils_.py index 49502cd7..bd3a0583 100644 --- a/myst_parser/docutils_.py +++ b/myst_parser/docutils_.py @@ -187,7 +187,6 @@ def parse(self, inputstring: str, document: nodes.document) -> None: :param inputstring: The source string to parse :param document: The root docutils node to add AST elements to """ - self.setup_parse(inputstring, document) # check for exorbitantly long lines diff --git a/myst_parser/docutils_renderer.py b/myst_parser/docutils_renderer.py index 33d1b97f..0b1f9d41 100644 --- a/myst_parser/docutils_renderer.py +++ b/myst_parser/docutils_renderer.py @@ -34,6 +34,7 @@ from docutils.statemachine import StringList from docutils.transforms.components import Filter from docutils.utils import Reporter, new_document +from docutils.utils.code_analyzer import Lexer, LexerError, NumberLines from markdown_it import MarkdownIt from markdown_it.common.utils import escapeHtml from markdown_it.renderer import RendererProtocol @@ -427,14 +428,84 @@ def render_code_inline(self, token: SyntaxTreeNode) -> None: self.add_line_and_source_path(node, token) self.current_node.append(node) + def create_highlighted_code_block( + self, + text: str, + lexer_name: str, + number_lines: bool = False, + lineno_start: int = 1, + source: Optional[str] = None, + line: Optional[int] = None, + ) -> nodes.literal_block: + """Create a literal block with syntax highlighting. + + This mimics the behaviour of the `code-block` directive. + + In docutils, this directive directly parses the text with the pygments lexer, + whereas in sphinx, the lexer name is only recorded as the `language` attribute, + and the text is lexed later by pygments within the `visit_literal_block` + method of the output format ``SphinxTranslator``. + + Note, this function does not add the literal block to the document. + """ + if self.sphinx_env is not None: + node = nodes.literal_block(text, text, language=lexer_name) + if number_lines: + node["linenos"] = True + if lineno_start != 1: + node["highlight_args"] = {"linenostart": lineno_start} + else: + node = nodes.literal_block( + text, classes=["code"] + ([lexer_name] if lexer_name else []) + ) + try: + lex_tokens = Lexer( + text, + lexer_name, + "short" + if self.config.get("myst_highlight_code_blocks", True) + else "none", + ) + except LexerError as err: + self.reporter.warning( + str(err), + **{ + name: value + for name, value in (("source", source), ("line", line)) + if value is not None + }, + ) + lex_tokens = Lexer(text, lexer_name, "none") + + if number_lines: + lex_tokens = NumberLines( + lex_tokens, lineno_start, lineno_start + len(text.splitlines()) + ) + + for classes, value in lex_tokens: + if classes: + node += nodes.inline(value, value, classes=classes) + else: + # insert as Text to decrease the verbosity of the output + node += nodes.Text(value) + + if source is not None: + node.source = source + if line is not None: + node.line = line + return node + def render_code_block(self, token: SyntaxTreeNode) -> None: # this should never have a language, since it is just indented text, however, # creating a literal_block with no language will raise a warning in sphinx - text = token.content - language = token.info.split()[0] if token.info else "none" - language = language or "none" - node = nodes.literal_block(text, text, language=language) - self.add_line_and_source_path(node, token) + lexer = token.info.split()[0] if token.info else None + lexer = lexer or "none" + node = self.create_highlighted_code_block( + token.content, + lexer, + source=self.document["source"], + line=token_line(token, 0) or None, + ) self.current_node.append(node) def render_fence(self, token: SyntaxTreeNode) -> None: @@ -465,16 +536,20 @@ def render_fence(self, token: SyntaxTreeNode) -> None: ): return self.render_directive(token) - if not language: - if self.sphinx_env is not None: - language = self.sphinx_env.temp_data.get( - "highlight_language", self.sphinx_env.config.highlight_language - ) + if not language and self.sphinx_env is not None: + # use the current highlight setting, via the ``highlight`` directive, + # or ``highlight_language`` configuration. + language = self.sphinx_env.temp_data.get( + "highlight_language", self.sphinx_env.config.highlight_language + ) - if not language: - language = self.config.get("highlight_language", "") - node = nodes.literal_block(text, text, language=language) - self.add_line_and_source_path(node, token) + node = self.create_highlighted_code_block( + text, + language, + number_lines=language in self.config.get("myst_number_code_blocks", ()), + source=self.document["source"], + line=token_line(token, 0) or None, + ) self.current_node.append(node) @property diff --git a/myst_parser/main.py b/myst_parser/main.py index 615eb354..30f98cc6 100644 --- a/myst_parser/main.py +++ b/myst_parser/main.py @@ -1,4 +1,4 @@ -from typing import Callable, Dict, Iterable, Optional, Tuple, Union, cast +from typing import Any, Callable, Dict, Iterable, Optional, Sequence, Tuple, Union, cast import attr from attr.validators import ( @@ -131,6 +131,21 @@ def check_extensions(self, attribute, value): metadata={"help": "Sphinx domain names to search in for references"}, ) + highlight_code_blocks: bool = attr.ib( + default=True, + validator=instance_of(bool), + metadata={ + "help": "Syntax highlight code blocks with pygments", + "docutils_only": True, + }, + ) + + number_code_blocks: Sequence[str] = attr.ib( + default=(), + validator=deep_iterable(instance_of(str)), + metadata={"help": "Add line numbers to code blocks with these languages"}, + ) + heading_anchors: Optional[int] = attr.ib( default=None, validator=optional(in_([1, 2, 3, 4, 5, 6, 7])), @@ -187,11 +202,19 @@ def check_sub_delimiters(self, attribute, value): @classmethod def get_fields(cls) -> Tuple[attr.Attribute, ...]: + """Return all attribute fields in this class.""" return attr.fields(cls) def as_dict(self, dict_factory=dict) -> dict: + """Return a dictionary of field name -> value.""" return attr.asdict(self, dict_factory=dict_factory) + def as_triple(self) -> Iterable[Tuple[str, Any, attr.Attribute]]: + """Yield triples of (name, value, field).""" + fields = attr.fields_dict(self.__class__) + for name, value in attr.asdict(self).items(): + yield name, value, fields[name] + def default_parser(config: MdParserConfig): raise NotImplementedError( @@ -282,6 +305,8 @@ def create_md_parser( "myst_substitutions": config.substitutions, "myst_html_meta": config.html_meta, "myst_footnote_transition": config.footnote_transition, + "myst_number_code_blocks": config.number_code_blocks, + "myst_highlight_code_blocks": config.highlight_code_blocks, } ) diff --git a/tests/test_renderers/fixtures/docutil_syntax_elements.md b/tests/test_renderers/fixtures/docutil_syntax_elements.md new file mode 100644 index 00000000..78fb45d3 --- /dev/null +++ b/tests/test_renderers/fixtures/docutil_syntax_elements.md @@ -0,0 +1,864 @@ +--------------------------- +Raw +. +foo +. + + + foo +. + +--------------------------- +Hard-break +. +foo\ +bar +. + + + foo + +
+ + \\ + bar +. + +--------------------------- +Strong: +. +**foo** +. + + + + foo +. + +--------------------------- +Emphasis +. +*foo* +. + + + + foo +. + +--------------------------- +Escaped Emphasis: +. +\*foo* +. + + + *foo* +. + +-------------------------- +Mixed Inline +. +a *b* **c** `abc` \\* +. + + + a + + b + + + c + + + abc + \* +. + +-------------------------- +Inline Code: +. +`foo` +. + + + + foo +. + +-------------------------- +Heading: +. +# foo +. + +
+ + foo +. + +-------------------------- +Heading Levels: +. +# a +## b +### c +# d +. +<document source="notset"> + <section ids="a" names="a"> + <title> + a + <section ids="b" names="b"> + <title> + b + <section ids="c" names="c"> + <title> + c + <section ids="d" names="d"> + <title> + d +. + + +-------------------------- +Block Code: +. + foo +. +<document source="notset"> + <literal_block classes="code none" xml:space="preserve"> + foo +. + +-------------------------- +Fenced Code: +. +```sh +foo +``` +. +<document source="notset"> + <literal_block classes="code sh" xml:space="preserve"> + foo +. + +-------------------------- +Fenced Code no language: +. +``` +foo +``` +. +<document source="notset"> + <literal_block classes="code" xml:space="preserve"> + foo +. + +-------------------------- +Fenced Code no language with trailing whitespace: +. +``` +foo +``` +. +<document source="notset"> + <literal_block classes="code" xml:space="preserve"> + foo +. + +-------------------------- +Image empty: +. +![]() +. +<document source="notset"> + <paragraph> + <image alt="" uri=""> +. + +-------------------------- +Image with alt and title: +. +![alt](src "title") +. +<document source="notset"> + <paragraph> + <image alt="alt" title="title" uri="src"> +. + +-------------------------- +Image with escapable html: +. +![alt](http://www.google<>.com) +. +<document source="notset"> + <paragraph> + <image alt="alt" uri="http://www.google%3C%3E.com"> +. + +-------------------------- +Block Quote: +. +> *foo* +. +<document source="notset"> + <block_quote> + <paragraph> + <emphasis> + foo +. + +-------------------------- +Bullet List: +. +- *foo* +* bar +. +<document source="notset"> + <bullet_list bullet="-"> + <list_item> + <paragraph> + <emphasis> + foo + <bullet_list bullet="*"> + <list_item> + <paragraph> + bar +. + +-------------------------- +Nested Bullets +. +- a + - b + - c + - d +. +<document source="notset"> + <bullet_list bullet="-"> + <list_item> + <paragraph> + a + <bullet_list bullet="-"> + <list_item> + <paragraph> + b + <bullet_list bullet="-"> + <list_item> + <paragraph> + c + <list_item> + <paragraph> + d +. + +-------------------------- +Enumerated List: +. +1. *foo* +. +<document source="notset"> + <enumerated_list enumtype="arabic" suffix="."> + <list_item> + <paragraph> + <emphasis> + foo +. + +-------------------------- +Nested Enumrated List: +. +1. a +2. b + 1. c +. +<document source="notset"> + <enumerated_list enumtype="arabic" suffix="."> + <list_item> + <paragraph> + a + <list_item> + <paragraph> + b + <enumerated_list enumtype="arabic" suffix="."> + <list_item> + <paragraph> + c +. + +-------------------------- +Inline Math: +. +$foo$ +. +<document source="notset"> + <paragraph> + <math> + foo +. + +-------------------------- +Inline Math, multi-line: +. +a $foo +bar$ b +. +<document source="notset"> + <paragraph> + a + <math> + foo + bar + b +. + +-------------------------- +Inline Math, multi-line with line break (invalid): +. +a $foo + +bar$ b +. +<document source="notset"> + <paragraph> + a $foo + <paragraph> + bar$ b +. + +-------------------------- +Math Block: +. +$$foo$$ +. +<document source="notset"> + <math_block nowrap="False" number="True" xml:space="preserve"> + foo +. + +-------------------------- +Sphinx Role containing backtick: +. +{code}``a=1{`}`` +. +<document source="notset"> + <paragraph> + <literal classes="code"> + a=1{`} +. + +-------------------------- +Target: +. +(target)= +. +<document source="notset"> + <target ids="target" names="target"> +. + +-------------------------- +Target with whitespace: +. +(target with space)= +. +<document source="notset"> + <target ids="target-with-space" names="target\ with\ space"> +. + +-------------------------- +Referencing: +. +(target)= + +Title +===== + +[alt1](target) + +[](target2) + +[alt2](https://www.google.com) + +[alt3](#target3) +. +<document source="notset"> + <target ids="target" names="target"> + <section ids="title" names="title"> + <title> + Title + <paragraph> + <reference refname="target"> + alt1 + <paragraph> + <reference refname="target2"> + <paragraph> + <reference refuri="https://www.google.com"> + alt2 + <paragraph> + <reference refname="#target3"> + alt3 +. + +-------------------------- +Comments: +. +line 1 +% a comment +line 2 +. +<document source="notset"> + <paragraph> + line 1 + <comment xml:space="preserve"> + a comment + <paragraph> + line 2 +. + +-------------------------- +Block Break: +. ++++ string +. +<document source="notset"> + <comment classes="block_break" xml:space="preserve"> + string +. + +-------------------------- +Link Reference: +. +[name][key] + +[key]: https://www.google.com "a title" +. +<document source="notset"> + <paragraph> + <reference refuri="https://www.google.com" title="a title"> + name +. + +-------------------------- +Link Reference short version: +. +[name] + +[name]: https://www.google.com "a title" +. +<document source="notset"> + <paragraph> + <reference refuri="https://www.google.com" title="a title"> + name +. + +-------------------------- +Block Quotes: +. +```{epigraph} +a b*c* + +-- a**b** +``` +. +<document source="notset"> + <block_quote classes="epigraph"> + <paragraph> + a b + <emphasis> + c + <attribution> + a + <strong> + b +. + +-------------------------- +Link Definition in directive: +. +```{note} +[a] +``` + +[a]: link +. +<document source="notset"> + <note> + <paragraph> + <reference refname="link"> + a +. + +-------------------------- +Link Definition in nested directives: +. +```{note} +[ref1]: link +``` + +```{note} +[ref1] +[ref2] +``` + +```{note} +[ref2]: link +``` +. +<document source="notset"> + <note> + <note> + <paragraph> + <reference refname="link"> + ref1 + + [ref2] + <note> +. + +-------------------------- +Footnotes: +. +[^a] + +[^a]: footnote*text* +. +<document source="notset"> + <paragraph> + <footnote_reference auto="1" ids="id1" refname="a"> + <transition classes="footnotes"> + <footnote auto="1" ids="a" names="a"> + <paragraph> + footnote + <emphasis> + text +. + +-------------------------- +Footnotes nested blocks: +. +[^a] + +[^a]: footnote*text* + + abc +xyz + + > a + + - b + + c + +finish +. +<document source="notset"> + <paragraph> + <footnote_reference auto="1" ids="id1" refname="a"> + <paragraph> + finish + <transition classes="footnotes"> + <footnote auto="1" ids="a" names="a"> + <paragraph> + footnote + <emphasis> + text + <paragraph> + abc + + xyz + <block_quote> + <paragraph> + a + <bullet_list bullet="-"> + <list_item> + <paragraph> + b + <paragraph> + c +. + +-------------------------- +Front Matter: +. +--- +a: 1 +b: foo +c: + d: 2 +--- +. +<document source="notset"> + <field_list> + <field> + <field_name> + a + <field_body> + <paragraph> + 1 + <field> + <field_name> + b + <field_body> + <paragraph> + foo + <field> + <field_name> + c + <field_body> + <paragraph> + {"d": 2} +. + +-------------------------- +Front Matter Biblio: +. +--- +author: Chris Sewell +authors: Chris Sewell, Chris Hodgraf +organization: EPFL +address: | + 1 Cedar Park Close + Thundersley + Essex +contact: <https://example.com> +version: 1.0 +revision: 1.1 +status: good +date: 2/12/1985 +copyright: MIT +dedication: | + To my *homies* +abstract: + Something something **dark** side +other: Something else +--- +. +<document source="notset"> + <field_list> + <field> + <field_name> + author + <field_body> + <paragraph> + Chris Sewell + <field> + <field_name> + authors + <field_body> + <paragraph> + Chris Sewell, Chris Hodgraf + <field> + <field_name> + organization + <field_body> + <paragraph> + EPFL + <field> + <field_name> + address + <field_body> + <paragraph> + 1 Cedar Park Close + + Thundersley + + Essex + + <field> + <field_name> + contact + <field_body> + <paragraph> + <reference refuri="https://example.com"> + https://example.com + <field> + <field_name> + version + <field_body> + <paragraph> + 1.0 + <field> + <field_name> + revision + <field_body> + <paragraph> + 1.1 + <field> + <field_name> + status + <field_body> + <paragraph> + good + <field> + <field_name> + date + <field_body> + <paragraph> + 2/12/1985 + <field> + <field_name> + copyright + <field_body> + <paragraph> + MIT + <field> + <field_name> + dedication + <field_body> + <paragraph> + To my + <emphasis> + homies + + <field> + <field_name> + abstract + <field_body> + <paragraph> + Something something + <strong> + dark + side + <field> + <field_name> + other + <field_body> + <paragraph> + Something else +. + +-------------------------- +Front Matter Bad Yaml: +. +--- +a: { +--- +. +<document source="notset"> + <system_message level="3" line="1" source="notset" type="ERROR"> + <paragraph> + Front matter block: + while parsing a flow node + expected the node content, but found '<stream end>' + in "<unicode string>", line 1, column 5: + a: { + ^ + <literal_block xml:space="preserve"> + a: { +. + +Front Matter HTML Meta +. +--- +html_meta: + keywords: Sphinx, documentation, builder + description lang=en: An amusing story + description lang=fr: Un histoire amusant + http-equiv=Content-Type: text/html; charset=ISO-8859-1 +--- +. +<document source="notset"> + <pending> + .. internal attributes: + .transform: docutils.transforms.components.Filter + .details: + component: 'writer' + format: 'html' + nodes: + <meta content="Sphinx, documentation, builder" name="keywords"> + <pending> + .. internal attributes: + .transform: docutils.transforms.components.Filter + .details: + component: 'writer' + format: 'html' + nodes: + <meta content="An amusing story" lang="en" name="description"> + <pending> + .. internal attributes: + .transform: docutils.transforms.components.Filter + .details: + component: 'writer' + format: 'html' + nodes: + <meta content="Un histoire amusant" lang="fr" name="description"> + <pending> + .. internal attributes: + .transform: docutils.transforms.components.Filter + .details: + component: 'writer' + format: 'html' + nodes: + <meta content="text/html; charset=ISO-8859-1" http-equiv="Content-Type"> +. + +-------------------------- +Full Test: +. +--- +a: 1 +--- + +(target)= +# header 1 +## sub header 1 + +a *b* **c** `abc` + +## sub header 2 + +x y [a](http://www.xyz.com) z + +--- + +# header 2 + +```::python {a=1} +a = 1 +``` + +[](target) +. +<document source="notset"> + <field_list> + <field> + <field_name> + a + <field_body> + <paragraph> + 1 + <target ids="target" names="target"> + <section ids="header-1" names="header\ 1"> + <title> + header 1 + <section ids="sub-header-1" names="sub\ header\ 1"> + <title> + sub header 1 + <paragraph> + a + <emphasis> + b + + <strong> + c + + <literal> + abc + <section ids="sub-header-2" names="sub\ header\ 2"> + <title> + sub header 2 + <paragraph> + x y + <reference refuri="http://www.xyz.com"> + a + z + <transition> + <section ids="header-2" names="header\ 2"> + <title> + header 2 + <literal_block classes="code ::python" xml:space="preserve"> + a = 1 + <paragraph> + <reference refname="target"> +. diff --git a/tests/test_renderers/fixtures/syntax_elements.md b/tests/test_renderers/fixtures/sphinx_syntax_elements.md similarity index 100% rename from tests/test_renderers/fixtures/syntax_elements.md rename to tests/test_renderers/fixtures/sphinx_syntax_elements.md diff --git a/tests/test_renderers/test_fixtures_docutils.py b/tests/test_renderers/test_fixtures_docutils.py index 3bfaf085..5b74e040 100644 --- a/tests/test_renderers/test_fixtures_docutils.py +++ b/tests/test_renderers/test_fixtures_docutils.py @@ -13,6 +13,29 @@ FIXTURE_PATH = Path(__file__).parent.joinpath("fixtures") +@pytest.mark.parametrize( + "line,title,input,expected", + read_fixture_file(FIXTURE_PATH.joinpath("docutil_syntax_elements.md")), + ids=[ + f"{i[0]}-{i[1]}" + for i in read_fixture_file(FIXTURE_PATH / "docutil_syntax_elements.md") + ], +) +def test_syntax_elements(line, title, input, expected): + parser = create_md_parser( + MdParserConfig(highlight_code_blocks=False), DocutilsRenderer + ) + parser.options["document"] = document = make_document() + parser.render(input) + try: + assert "\n".join( + [ll.rstrip() for ll in document.pformat().splitlines()] + ) == "\n".join([ll.rstrip() for ll in expected.splitlines()]) + except AssertionError: + print(document.pformat()) + raise + + @pytest.mark.parametrize( "line,title,input,expected", read_fixture_file(FIXTURE_PATH.joinpath("docutil_roles.md")), diff --git a/tests/test_renderers/test_fixtures_sphinx.py b/tests/test_renderers/test_fixtures_sphinx.py index c14d0b85..3ada5c58 100644 --- a/tests/test_renderers/test_fixtures_sphinx.py +++ b/tests/test_renderers/test_fixtures_sphinx.py @@ -22,9 +22,10 @@ def test_minimal_sphinx(): @pytest.mark.parametrize( "line,title,input,expected", - read_fixture_file(FIXTURE_PATH.joinpath("syntax_elements.md")), + read_fixture_file(FIXTURE_PATH.joinpath("sphinx_syntax_elements.md")), ids=[ - f"{i[0]}-{i[1]}" for i in read_fixture_file(FIXTURE_PATH / "syntax_elements.md") + f"{i[0]}-{i[1]}" + for i in read_fixture_file(FIXTURE_PATH / "sphinx_syntax_elements.md") ], ) def test_syntax_elements(line, title, input, expected): From 836f2a21e4569b1c46d2db1dfd48beabecf77b35 Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Wed, 29 Dec 2021 02:29:35 +0100 Subject: [PATCH 2/5] revert line deletion --- myst_parser/docutils_.py | 1 + 1 file changed, 1 insertion(+) diff --git a/myst_parser/docutils_.py b/myst_parser/docutils_.py index bd3a0583..49502cd7 100644 --- a/myst_parser/docutils_.py +++ b/myst_parser/docutils_.py @@ -187,6 +187,7 @@ def parse(self, inputstring: str, document: nodes.document) -> None: :param inputstring: The source string to parse :param document: The root docutils node to add AST elements to """ + self.setup_parse(inputstring, document) # check for exorbitantly long lines From cc1049cf65ada9653f88e7537a5112a523bb9ade Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Wed, 29 Dec 2021 02:43:34 +0100 Subject: [PATCH 3/5] Update tests.yml --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 089f8bb8..0dd67f00 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -67,6 +67,7 @@ jobs: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: docutils-version: ["0.16", "0.17", "0.18"] From 08e10a15688b2f6d3fc2fccf8c9ea4fbb37f87ad Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Wed, 29 Dec 2021 02:47:39 +0100 Subject: [PATCH 4/5] Update test_fixtures_docutils.py --- tests/test_renderers/test_fixtures_docutils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_renderers/test_fixtures_docutils.py b/tests/test_renderers/test_fixtures_docutils.py index 5b74e040..462900d3 100644 --- a/tests/test_renderers/test_fixtures_docutils.py +++ b/tests/test_renderers/test_fixtures_docutils.py @@ -27,10 +27,11 @@ def test_syntax_elements(line, title, input, expected): ) parser.options["document"] = document = make_document() parser.render(input) + outcome = "\n".join([ll.rstrip() for ll in document.pformat().splitlines()]) + # in docutils 0.18 footnote ids have changed + outcome = outcome.replace('"footnote-reference-1"', '"id1"') try: - assert "\n".join( - [ll.rstrip() for ll in document.pformat().splitlines()] - ) == "\n".join([ll.rstrip() for ll in expected.splitlines()]) + assert outcome == "\n".join([ll.rstrip() for ll in expected.splitlines()]) except AssertionError: print(document.pformat()) raise From 36205546b384a631484039eef444d79a8ca207c5 Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Wed, 29 Dec 2021 02:55:06 +0100 Subject: [PATCH 5/5] add to test --- .../sourcedirs/extended_syntaxes/conf.py | 1 + .../sourcedirs/extended_syntaxes/index.md | 6 ++++++ .../test_extended_syntaxes.sphinx3.html | 20 +++++++++++++++++++ .../test_extended_syntaxes.sphinx3.xml | 4 ++++ .../test_extended_syntaxes.sphinx4.html | 9 +++++++++ .../test_extended_syntaxes.sphinx4.xml | 4 ++++ 6 files changed, 44 insertions(+) diff --git a/tests/test_sphinx/sourcedirs/extended_syntaxes/conf.py b/tests/test_sphinx/sourcedirs/extended_syntaxes/conf.py index 84f5aa67..9845000a 100644 --- a/tests/test_sphinx/sourcedirs/extended_syntaxes/conf.py +++ b/tests/test_sphinx/sourcedirs/extended_syntaxes/conf.py @@ -13,6 +13,7 @@ "linkify", "tasklist", ] +myst_number_code_blocks = ["typescript"] myst_html_meta = { "description lang=en": "meta description", "property=og:locale": "en_US", diff --git a/tests/test_sphinx/sourcedirs/extended_syntaxes/index.md b/tests/test_sphinx/sourcedirs/extended_syntaxes/index.md index 08d7e33a..e9fcea93 100644 --- a/tests/test_sphinx/sourcedirs/extended_syntaxes/index.md +++ b/tests/test_sphinx/sourcedirs/extended_syntaxes/index.md @@ -59,3 +59,9 @@ linkify URL: www.example.com - [ ] hallo - [x] there + +Numbered code block: + +```typescript +type Result = "pass" | "fail" +``` diff --git a/tests/test_sphinx/test_sphinx_builds/test_extended_syntaxes.sphinx3.html b/tests/test_sphinx/test_sphinx_builds/test_extended_syntaxes.sphinx3.html index 78fd09cf..b1ff5caa 100644 --- a/tests/test_sphinx/test_sphinx_builds/test_extended_syntaxes.sphinx3.html +++ b/tests/test_sphinx/test_sphinx_builds/test_extended_syntaxes.sphinx3.html @@ -147,6 +147,26 @@ <h1> </p> </li> </ul> + <p> + Numbered code block: + </p> + <div class="highlight-typescript notranslate"> + <table class="highlighttable"> + <tr> + <td class="linenos"> + <div class="linenodiv"> + <pre><span class="normal">1</span></pre> + </div> + </td> + <td class="code"> + <div class="highlight"> + <pre><span></span><span class="kr">type</span> <span class="nx">Result</span> <span class="o">=</span> <span class="s2">"pass"</span> <span class="o">|</span> <span class="s2">"fail"</span> +</pre> + </div> + </td> + </tr> + </table> + </div> </div> </div> </div> diff --git a/tests/test_sphinx/test_sphinx_builds/test_extended_syntaxes.sphinx3.xml b/tests/test_sphinx/test_sphinx_builds/test_extended_syntaxes.sphinx3.xml index c9d9eb90..18802ecc 100644 --- a/tests/test_sphinx/test_sphinx_builds/test_extended_syntaxes.sphinx3.xml +++ b/tests/test_sphinx/test_sphinx_builds/test_extended_syntaxes.sphinx3.xml @@ -84,3 +84,7 @@ <raw format="html" xml:space="preserve"> <input class="task-list-item-checkbox" checked="checked" disabled="disabled" type="checkbox"> there + <paragraph> + Numbered code block: + <literal_block language="typescript" linenos="True" xml:space="preserve"> + type Result = "pass" | "fail" diff --git a/tests/test_sphinx/test_sphinx_builds/test_extended_syntaxes.sphinx4.html b/tests/test_sphinx/test_sphinx_builds/test_extended_syntaxes.sphinx4.html index 768888df..082eb20c 100644 --- a/tests/test_sphinx/test_sphinx_builds/test_extended_syntaxes.sphinx4.html +++ b/tests/test_sphinx/test_sphinx_builds/test_extended_syntaxes.sphinx4.html @@ -151,6 +151,15 @@ <h1> </p> </li> </ul> + <p> + Numbered code block: + </p> + <div class="highlight-typescript notranslate"> + <div class="highlight"> + <pre><span></span><span class="linenos">1</span><span class="kr">type</span> <span class="nx">Result</span> <span class="o">=</span> <span class="s2">"pass"</span> <span class="o">|</span> <span class="s2">"fail"</span> +</pre> + </div> + </div> </section> </div> </div> diff --git a/tests/test_sphinx/test_sphinx_builds/test_extended_syntaxes.sphinx4.xml b/tests/test_sphinx/test_sphinx_builds/test_extended_syntaxes.sphinx4.xml index ba3b7566..8cefbe3b 100644 --- a/tests/test_sphinx/test_sphinx_builds/test_extended_syntaxes.sphinx4.xml +++ b/tests/test_sphinx/test_sphinx_builds/test_extended_syntaxes.sphinx4.xml @@ -84,3 +84,7 @@ <raw format="html" xml:space="preserve"> <input class="task-list-item-checkbox" checked="checked" disabled="disabled" type="checkbox"> there + <paragraph> + Numbered code block: + <literal_block language="typescript" linenos="True" xml:space="preserve"> + type Result = "pass" | "fail"