Add MyST support [adamchainz#240]

CHANGELOG.rst

@@ -2,6 +2,10 @@
 Changelog
 =========
 
+* Add support for MyST - Markedly structured text.
+
+  Thanks to Zebedee Nicholls in `PR #239 <https://github.com/adamchainz/blacken-docs/pull/239>`__.
+
 1.16.0 (2023-08-16)
 -------------------
 

README.rst

@@ -243,3 +243,28 @@ In PythonTeX blocks:
     def hello():
         print("hello world")
     \end{pycode}
+
+MyST - Markedly Structured Text
+-------------------------------
+
+In “python” blocks:
+
+.. code-block:: markdown
+
+    ```python
+    def hello():
+        print("hello world")
+    ```
+
+In notebook blocks, including cell metadata and cell magic:
+
+.. code-block:: markdown
+
+    ```{code-cell}python
+    ---
+    tags: [some-tags]
+    ---
+    %matplotlib inline
+
+    plt.plot([0, 1, 2], [2, 4, 6])
+    ```

setup.cfg

@@ -32,6 +32,8 @@ project_urls =
 packages = find:
 install_requires =
     black>=22.1.0
+    tokenize-rt>=5.0.0  # required to call black format_cell
+    IPython  # required to call black format_cell
 python_requires = >=3.8
 include_package_data = True
 package_dir =

src/blacken_docs/__init__.py

@@ -27,6 +27,27 @@
     r"(?P<after>^(?P=indent)```.*$)",
     re.DOTALL | re.MULTILINE,
 )
+
+MYST_RE = re.compile(
+    r"(?P<before>"
+    r"^(?P<indent>\s*)```(\s*{code-cell})?\s*(?P<lang>\w+)?"
+    r"(?P<yaml>\n---\n.*?\n---)?"
+    r"\n)"
+    r"(?P<code>.*?)"
+    r"(?P<after>^(?P=indent)```\s*$)",
+    re.DOTALL | re.MULTILINE,
+)
+RST_PY_LANGS = frozenset(
+    (
+        "python",
+        "ipython",
+        "py",
+        "python3",
+        "ipython3",
+        "py3",
+    )
+)
+
 BLOCK_TYPES = "(code|code-block|sourcecode|ipython)"
 DOCTEST_TYPES = "(testsetup|testcleanup|testcode)"
 RST_RE = re.compile(
@@ -116,6 +137,19 @@ def _md_match(match: Match[str]) -> str:
         code = textwrap.indent(code, match["indent"])
         return f'{match["before"]}{code}{match["after"]}'
 
+    def _myst_match(match: Match[str]) -> str:
+        lang = match["lang"]
+        if lang is not None and lang not in RST_PY_LANGS:
+            return match[0]
+        trailing_ws_match = TRAILING_NL_RE.search(match["code"])
+        assert trailing_ws_match
+        trailing_ws = trailing_ws_match.group()
+        code = textwrap.dedent(match["code"])
+        with _collect_error(match):
+            code = black.format_cell(code, fast=False, mode=black_mode)
+        code = textwrap.indent(code, match["indent"])
+        return f'{match["before"]}{code.rstrip()}{trailing_ws}{match["after"]}'
+
     def _rst_match(match: Match[str]) -> str:
         lang = match["lang"]
         if lang is not None and lang not in PYGMENTS_PY_LANGS:
@@ -220,6 +254,7 @@ def _latex_pycon_match(match: Match[str]) -> str:
             _rst_literal_blocks_match,
             src,
         )
+    src = MYST_RE.sub(_myst_match, src)
     src = LATEX_RE.sub(_latex_match, src)
     src = LATEX_PYCON_RE.sub(_latex_pycon_match, src)
     src = PYTHONTEX_RE.sub(_latex_match, src)
@@ -249,8 +284,8 @@ def format_file(
         with open(filename, "w", encoding="UTF-8") as f:
             f.write(new_contents)
         return 1
-    else:
-        return 0
+
+    return 0
 
 
 def main(argv: Sequence[str] | None = None) -> int:

tests/test_blacken_docs.py

@@ -3,6 +3,7 @@
 from textwrap import dedent
 
 import black
+import pytest
 from black.const import DEFAULT_LINE_LENGTH
 
 import blacken_docs
@@ -944,3 +945,173 @@ def test_format_src_rst_pycon_comment_before_promopt():
         "    # Comment about next line\n"
         "    >>> pass\n"
     )
+
+
+def test_format_src_myst_simple():
+    before = dedent(
+        """\
+        ```{code-cell}
+        print('hello world')
+        ```
+        """
+    )
+    after, _ = blacken_docs.format_str(before, BLACK_MODE)
+    assert after == dedent(
+        """\
+        ```{code-cell}
+        print("hello world")
+        ```
+        """
+    )
+
+
+def test_format_src_myst_no_code_cell():
+    before = dedent(
+        """\
+        ```
+        ---
+        tags: [some-tags]
+        slideshow:
+          slide_type: ''
+        ---
+        print('hello world')
+        ```
+        """
+    )
+    after, _ = blacken_docs.format_str(before, BLACK_MODE)
+    assert after == dedent(
+        """\
+        ```
+        ---
+        tags: [some-tags]
+        slideshow:
+          slide_type: ''
+        ---
+        print("hello world")
+        ```
+        """
+    )
+
+
+def test_format_src_myst_tags():
+    before = dedent(
+        """\
+        ```{code-cell}
+        ---
+        tags: [some-tags]
+        ---
+        print('hello world')
+        ```
+        """
+    )
+    after, _ = blacken_docs.format_str(before, BLACK_MODE)
+    assert after == dedent(
+        """\
+        ```{code-cell}
+        ---
+        tags: [some-tags]
+        ---
+        print("hello world")
+        ```
+        """
+    )
+
+
+@pytest.mark.parametrize("language, supported", (
+    ("python", True),
+    ("pythona", False),
+    ("ipython", True),
+    ("py", True),
+    ("sage", False),
+    ("python3", True),
+    ("ipython3", True),
+    ("py3", True),
+    ("numpy", False),
+))
+def test_format_src_myst_tags_language(language, supported):
+    before = dedent(
+        """\
+        ```{code-cell}language
+        ---
+        tags: [some-tags]
+        ---
+        print('hello world')
+        ```
+        """
+    ).replace("language", language)
+    after, _ = blacken_docs.format_str(before, BLACK_MODE)
+
+    if supported:
+        assert after == dedent(
+            """\
+            ```{code-cell}language
+            ---
+            tags: [some-tags]
+            ---
+            print("hello world")
+            ```
+            """
+        ).replace("language", language)
+    else:
+        assert after == before
+
+
+def test_format_src_myst_function_def():
+    before = dedent(
+        """\
+        ```{code-cell}python
+        ---
+        tags: [some-tags]
+        ---
+        import copy
+
+        def deepcopy(inp:  Dict)  ->  Dict:
+            return copy.deepcopy(
+                                 inp)
+        ```
+        """
+    )
+    after, _ = blacken_docs.format_str(before, BLACK_MODE)
+    assert after == dedent(
+        """\
+        ```{code-cell}python
+        ---
+        tags: [some-tags]
+        ---
+        import copy
+
+
+        def deepcopy(inp: Dict) -> Dict:
+            return copy.deepcopy(inp)
+        ```
+        """
+    )
+
+
+def test_format_src_myst_magic_comand():
+    before = dedent(
+        """\
+        ```{code-cell}python
+        ---
+        tags: [some-tags]
+        ---
+        %matplotlib inline
+
+        plt.plot([0, 1,2], [2,4,6])
+        ```
+        """
+    )
+    after, errors = blacken_docs.format_str(before, BLACK_MODE)
+    assert not errors, str(errors[0].exc)
+    assert after == dedent(
+        """\
+        ```{code-cell}python
+        ---
+        tags: [some-tags]
+        ---
+        %matplotlib inline
+
+        plt.plot([0, 1, 2], [2, 4, 6])
+        ```
+        """
+    )