👌 IMPROVE: Support configuration through docutils.conf

This is part of executablebooksGH-420, an effort to make myst_parser integrate better with
Docutils.

* myst_parser/docutils_.py (Parser): Use new _myst_docutils_setting_tuples
function to generate a settings_spec.
(Parser.parse): Use new create_myst_config function to reconstruct a
MdParserConfig object from a Docutils configuration object.

myst_parser/docutils_.py

@@ -3,35 +3,128 @@
    .. include::
       :parser: myst_parser.docutils_
 """
-from typing import Tuple
+from typing import Iterable, Tuple, Union
 
-from docutils import nodes
+from attr import Attribute
+from docutils import nodes, frontend
 from docutils.parsers.rst import Parser as RstParser
 from markdown_it.token import Token
 from markdown_it.utils import AttrDict
 
 from myst_parser.main import MdParserConfig, default_parser
 
 
-class Parser(RstParser):
-    """Docutils parser for Markedly Structured Text (MyST)."""
+def validate_int(
+    setting, value, option_parser, config_parser=None, config_section=None
+):
+    return int(value)
+
+
+def validate_tuple(length: int):
+    def validate(
+        setting, value, option_parser, config_parser=None, config_section=None
+    ):
+        l = frontend.validate_comma_separated_list(
+            setting, value, option_parser, config_parser, config_section
+        )
+        if len(l) != length:
+            MSG = "Expecting {} items in {}, got {}: {}."
+            raise ValueError(MSG.format(length, setting, len(l)))
+        return tuple(l)
+
+    return validate
+
+
+DOCUTILS_UNSET = object()
+"""Sentinel for arguments not set through docutils.conf."""
+
+DOCUTILS_OPTPARSE_OVERRIDES = {
+    "url_schemes": {"validator": frontend.validate_comma_separated_list},
+    # url_schemes accepts an iterable or ``None``, but ``None`` is the same as ``()``.
+}
+"""Custom optparse configurations for docutils.conf entries."""
+
+DOCUTILS_EXCLUDED_ARGS = (
+    # docutils.conf can't represent callables
+    "heading_slug_func",
+    # docutils.conf can't represent dicts
+    "html_meta",
+    "substitutions",
+    # We don't want to set the renderer from docutils.conf
+    "renderer",
+)
+"""Names of settings that cannot be set in docutils.conf."""
 
-    supported: Tuple[str, ...] = ("md", "markdown", "myst")
-    """Aliases this parser supports."""
 
-    settings_spec = RstParser.settings_spec
-    """Runtime settings specification.
+def _docutils_optparse_options_of_attribute(a: Attribute):
+    override = DOCUTILS_OPTPARSE_OVERRIDES.get(a.name)
+    if override is not None:
+        return override
+    if a.type is int:
+        return {"validator": validate_int}
+    if a.type is bool:
+        return {"validator": frontend.validate_boolean}
+    if a.type is str:
+        return {}
+    if a.type == Iterable[str]:
+        return {"validator": frontend.validate_comma_separated_list}
+    if a.type == Tuple[str, str]:
+        return {"validator": validate_tuple(2)}
+    if a.type == Union[int, type(None)] and a.default is None:
+        return {"validator": validate_int, "default": None}
+    if a.type == Union[Iterable[str], type(None)] and a.default is None:
+        return {"validator": frontend.validate_comma_separated_list, "default": None}
+    raise AssertionError(
+        f"""Configuration option {a.name} not set up for use \
+in docutils.conf.  Either add {a.name} to docutils_.DOCUTILS_EXCLUDED_ARGS, or \
+add a new entry in _docutils_optparse_of_attribute."""
+    )
 
-    Defines runtime settings and associated command-line options, as used by
-    `docutils.frontend.OptionParser`.  This is a concatenation of tuples of:
 
-    - Option group title (string or `None` which implies no group, just a list
-      of single options).
+def _docutils_setting_tuple_of_attribute(attribute):
+    """Convert an ``MdParserConfig`` attribute into a Docutils setting tuple."""
+    name = "myst_" + attribute.name
+    flag = "--" + name.replace("_", "-")
+    options = {"dest": name, "default": DOCUTILS_UNSET}
+    options.update(_docutils_optparse_options_of_attribute(attribute))
+    return (None, [flag], options)
 
-    - Description (string or `None`).
 
-    - A sequence of option tuples
-    """
+def _myst_docutils_setting_tuples():
+    return tuple(
+        _docutils_setting_tuple_of_attribute(a)
+        for a in MdParserConfig.get_fields()
+        if not a.name in DOCUTILS_EXCLUDED_ARGS
+    )
+
+
+def create_myst_config(settings: frontend.Values):
+    values = {}
+    for attribute in MdParserConfig.get_fields():
+        if attribute.name in DOCUTILS_EXCLUDED_ARGS:
+            continue
+        setting = f"myst_{attribute.name}"
+        val = getattr(settings, setting)
+        delattr(settings, setting)
+        if val is not DOCUTILS_UNSET:
+            values[attribute.name] = val
+    values["renderer"] = "docutils"
+    return MdParserConfig(**values)
+
+
+class Parser(RstParser):
+    """Docutils parser for Markedly Structured Text (MyST)."""
+
+    supported: Tuple[str, ...] = ("md", "markdown", "myst")
+    """Aliases this parser supports."""
+
+    settings_spec = (
+        *RstParser.settings_spec,
+        "MyST options",
+        None,
+        _myst_docutils_setting_tuples(),
+    )
+    """Runtime settings specification."""
 
     config_section = "myst parser"
     config_section_dependencies = ("parsers",)
@@ -42,9 +135,12 @@ def parse(self, inputstring: str, document: nodes.document) -> None:
 
         :param inputstring: The source string to parse
         :param document: The root docutils node to add AST elements to
-
         """
-        config = MdParserConfig(renderer="docutils")
+        try:
+            config = create_myst_config(document.settings)
+        except (TypeError, ValueError) as error:
+            document.reporter.error(f"myst configuration invalid: {error.args[0]}")
+            config = MdParserConfig(renderer="docutils")
         parser = default_parser(config)
         parser.options["document"] = document
         env = AttrDict()