From 02ec9664c4d6d9f5d5e1af86960a07af865a4b29 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Fri, 9 Dec 2022 07:36:30 +0100 Subject: [PATCH 1/5] =?UTF-8?q?=E2=9C=A8=20NEW:=20Add=20inventory=20reader?= =?UTF-8?q?=20and=20CLI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/tests.yml | 2 +- myst_parser/inventory.py | 365 ++++++++++++++++++++ pyproject.toml | 1 + tests/static/objects_v1.inv | 5 + tests/static/objects_v2.inv | Bin 0 -> 236 bytes tests/test_cli.py | 22 +- tests/test_cli/test_read_inv_options0_.yaml | 24 ++ tests/test_cli/test_read_inv_options1_.yaml | 24 ++ tests/test_cli/test_read_inv_options2_.yaml | 8 + tests/test_cli/test_read_inv_options3_.yaml | 8 + tests/test_cli/test_read_inv_v1.yaml | 12 + 11 files changed, 468 insertions(+), 3 deletions(-) create mode 100644 myst_parser/inventory.py create mode 100644 tests/static/objects_v1.inv create mode 100644 tests/static/objects_v2.inv create mode 100644 tests/test_cli/test_read_inv_options0_.yaml create mode 100644 tests/test_cli/test_read_inv_options1_.yaml create mode 100644 tests/test_cli/test_read_inv_options2_.yaml create mode 100644 tests/test_cli/test_read_inv_options3_.yaml create mode 100644 tests/test_cli/test_read_inv_v1.yaml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c05356e6..2984d73f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -99,7 +99,7 @@ jobs: else: raise AssertionError()" - name: Run pytest for docutils-only tests - run: pytest tests/test_docutils.py tests/test_renderers/test_fixtures_docutils.py tests/test_renderers/test_include_directive.py + run: pytest tests/test_docutils.py tests/test_renderers/test_fixtures_docutils.py tests/test_renderers/test_include_directive.py tests/test_renderers/test_myst_config.py - name: Run docutils CLI run: echo "test" | myst-docutils-html diff --git a/myst_parser/inventory.py b/myst_parser/inventory.py new file mode 100644 index 00000000..ff3bcc58 --- /dev/null +++ b/myst_parser/inventory.py @@ -0,0 +1,365 @@ +"""Logic for dealing with sphinx style inventories (e.g. `objects.inv`). + +These contain mappings of reference names to ids, scoped by domain and object type. + +This is adapted from the Sphinx inventory.py module. +We replicate it here, so that it can be used without Sphinx. +""" +from __future__ import annotations + +import argparse +import json +import re +import zlib +from dataclasses import dataclass +from fnmatch import fnmatchcase +from typing import IO, TYPE_CHECKING, Iterator +from urllib.request import urlopen + +import yaml +from typing_extensions import TypedDict + +if TYPE_CHECKING: + from sphinx.util.typing import Inventory + + +class InventoryItemType(TypedDict): + """A single inventory item.""" + + loc: str + """The relative location of the item.""" + text: str | None + """Implicit text to show for the item.""" + + +class InventoryType(TypedDict): + """Inventory data.""" + + name: str + """The name of the project.""" + version: str + """The version of the project.""" + objects: dict[str, dict[str, dict[str, InventoryItemType]]] + """Mapping of domain -> object type -> name -> item.""" + + +def format_inventory(inv: Inventory) -> InventoryType: + """Convert a Sphinx inventory to one that is JSON compliant.""" + project = "" + version = "" + objs: dict[str, dict[str, dict[str, InventoryItemType]]] = {} + for domain_obj_name, data in inv.items(): + if ":" not in domain_obj_name: + continue + + domain_name, obj_type = domain_obj_name.split(":", 1) + objs.setdefault(domain_name, {}).setdefault(obj_type, {}) + for refname, refdata in data.items(): + project, version, uri, text = refdata + objs[domain_name][obj_type][refname] = { + "loc": uri, + "text": None if (not text or text == "-") else text, + } + + return { + "name": project, + "version": version, + "objects": objs, + } + + +def load(stream: IO) -> InventoryType: + """Load inventory data from a stream.""" + reader = InventoryFileReader(stream) + line = reader.readline().rstrip() + if line == "# Sphinx inventory version 1": + return _load_v1(reader) + elif line == "# Sphinx inventory version 2": + return _load_v2(reader) + else: + raise ValueError("invalid inventory header: %s" % line) + + +def _load_v1(stream: InventoryFileReader) -> InventoryType: + """Load inventory data (format v1) from a stream.""" + projname = stream.readline().rstrip()[11:] + version = stream.readline().rstrip()[11:] + invdata: InventoryType = { + "name": projname, + "version": version, + "objects": {}, + } + for line in stream.readlines(): + name, objtype, location = line.rstrip().split(None, 2) + # version 1 did not add anchors to the location + domain = "py" + if objtype == "mod": + objtype = "module" + location += "#module-" + name + else: + location += "#" + name + invdata["objects"].setdefault(domain, {}).setdefault(objtype, {}) + invdata["objects"][domain][objtype][name] = {"loc": location, "text": None} + + return invdata + + +def _load_v2(stream: InventoryFileReader) -> InventoryType: + """Load inventory data (format v2) from a stream.""" + projname = stream.readline().rstrip()[11:] + version = stream.readline().rstrip()[11:] + invdata: InventoryType = { + "name": projname, + "version": version, + "objects": {}, + } + line = stream.readline() + if "zlib" not in line: + raise ValueError("invalid inventory header (not compressed): %s" % line) + + for line in stream.read_compressed_lines(): + # be careful to handle names with embedded spaces correctly + m = re.match(r"(?x)(.+?)\s+(\S+)\s+(-?\d+)\s+?(\S*)\s+(.*)", line.rstrip()) + if not m: + continue + name: str + type: str + name, type, _, location, text = m.groups() + if ":" not in type: + # wrong type value. type should be in the form of "{domain}:{objtype}" + # + # Note: To avoid the regex DoS, this is implemented in python (refs: #8175) + continue + if ( + type == "py:module" + and type in invdata["objects"] + and name in invdata["objects"][type] + ): + # due to a bug in 1.1 and below, + # two inventory entries are created + # for Python modules, and the first + # one is correct + continue + if location.endswith("$"): + location = location[:-1] + name + domain, objtype = type.split(":", 1) + invdata["objects"].setdefault(domain, {}).setdefault(objtype, {}) + if not text or text == "-": + text = None + invdata["objects"][domain][objtype][name] = {"loc": location, "text": text} + return invdata + + +_BUFSIZE = 16 * 1024 + + +class InventoryFileReader: + """A file reader for an inventory file. + + This reader supports mixture of texts and compressed texts. + """ + + def __init__(self, stream: IO) -> None: + self.stream = stream + self.buffer = b"" + self.eof = False + + def read_buffer(self) -> None: + chunk = self.stream.read(_BUFSIZE) + if chunk == b"": + self.eof = True + self.buffer += chunk + + def readline(self) -> str: + pos = self.buffer.find(b"\n") + if pos != -1: + line = self.buffer[:pos].decode() + self.buffer = self.buffer[pos + 1 :] + elif self.eof: + line = self.buffer.decode() + self.buffer = b"" + else: + self.read_buffer() + line = self.readline() + + return line + + def readlines(self) -> Iterator[str]: + while not self.eof: + line = self.readline() + if line: + yield line + + def read_compressed_chunks(self) -> Iterator[bytes]: + decompressor = zlib.decompressobj() + while not self.eof: + self.read_buffer() + yield decompressor.decompress(self.buffer) + self.buffer = b"" + yield decompressor.flush() + + def read_compressed_lines(self) -> Iterator[str]: + buf = b"" + for chunk in self.read_compressed_chunks(): + buf += chunk + pos = buf.find(b"\n") + while pos != -1: + yield buf[:pos].decode() + buf = buf[pos + 1 :] + pos = buf.find(b"\n") + + +@dataclass +class InvMatch: + """A match from an inventory.""" + + inv: str + domain: str + otype: str + name: str + proj: str + version: str + uri: str + text: str + + +def resolve_inventory( + inventories: dict[str, Inventory], + ref_target: str, + *, + ref_inv: None | str = None, + ref_domain: None | str = None, + ref_otype: None | str = None, + match_end=False, +) -> list[InvMatch]: + """Resolve a cross-reference in the loaded sphinx inventories. + + :param inventories: Mapping of inventory name to inventory data + :param ref_target: The target to search for + :param ref_inv: The name of the sphinx inventory to search, if None then + all inventories will be searched + :param ref_domain: The name of the domain to search, if None then all domains + will be searched + :param ref_otype: The type of object to search for, if None then all types will be searched + :param match_end: Whether to match against targets that end with the ref_target + + :returns: matching results + """ + results: list[InvMatch] = [] + for inv_name, inv_data in inventories.items(): + + if ref_inv is not None and ref_inv != inv_name: + continue + + for domain_obj_name, data in inv_data.items(): + + if ":" not in domain_obj_name: + continue + + domain_name, obj_type = domain_obj_name.split(":", 1) + + if ref_domain is not None and ref_domain != domain_name: + continue + + if ref_otype is not None and ref_otype != obj_type: + continue + + if not match_end and ref_target in data: + results.append( + InvMatch( + inv_name, domain_name, obj_type, ref_target, *data[ref_target] + ) + ) + elif match_end: + for target in data: + if target.endswith(ref_target): + results.append( + InvMatch( + inv_name, domain_name, obj_type, target, *data[target] + ) + ) + + return results + + +def inventory_cli(inputs: None | list[str] = None): + """Command line interface for fetching and parsing an inventory.""" + parser = argparse.ArgumentParser(description="Parse an inventory file.") + parser.add_argument("uri", metavar="[URL|PATH]", help="URI of the inventory file") + parser.add_argument( + "-d", + "--domain", + metavar="DOMAIN", + help="Filter the inventory by domain pattern", + ) + parser.add_argument( + "-o", + "--object-type", + metavar="TYPE", + help="Filter the inventory by object type pattern", + ) + parser.add_argument( + "-n", + "--name", + metavar="NAME", + help="Filter the inventory by reference name pattern", + ) + parser.add_argument( + "-f", + "--format", + choices=["yaml", "json"], + default="yaml", + help="Output format (default: yaml)", + ) + args = parser.parse_args(inputs) + + if args.uri.startswith("http"): + try: + with urlopen(args.uri) as stream: + invdata = load(stream) + except Exception: + with urlopen(args.uri + "/objects.inv") as stream: + invdata = load(stream) + else: + with open(args.uri, "rb") as stream: + invdata = load(stream) + + # filter the inventory + if args.domain: + invdata["objects"] = { + d: invdata["objects"][d] + for d in invdata["objects"] + if fnmatchcase(d, args.domain) + } + if args.object_type: + for domain in list(invdata["objects"]): + invdata["objects"][domain] = { + t: invdata["objects"][domain][t] + for t in invdata["objects"][domain] + if fnmatchcase(t, args.object_type) + } + if args.name: + for domain in invdata["objects"]: + for otype in list(invdata["objects"][domain]): + invdata["objects"][domain][otype] = { + n: invdata["objects"][domain][otype][n] + for n in invdata["objects"][domain][otype] + if fnmatchcase(n, args.name) + } + + # clean up empty items + for domain in list(invdata["objects"]): + for otype in list(invdata["objects"][domain]): + if not invdata["objects"][domain][otype]: + del invdata["objects"][domain][otype] + if not invdata["objects"][domain]: + del invdata["objects"][domain] + + if args.format == "json": + print(json.dumps(invdata, indent=2, sort_keys=False)) + else: + print(yaml.dump(invdata, sort_keys=False)) + + +if __name__ == "__main__": + inventory_cli() diff --git a/pyproject.toml b/pyproject.toml index 5b4484fa..9bb68ab2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,7 @@ testing = [ [project.scripts] myst-anchors = "myst_parser.cli:print_anchors" +myst-inv = "myst_parser.inventory:inventory_cli" myst-docutils-html = "myst_parser.parsers.docutils_:cli_html" myst-docutils-html5 = "myst_parser.parsers.docutils_:cli_html5" myst-docutils-latex = "myst_parser.parsers.docutils_:cli_latex" diff --git a/tests/static/objects_v1.inv b/tests/static/objects_v1.inv new file mode 100644 index 00000000..66947723 --- /dev/null +++ b/tests/static/objects_v1.inv @@ -0,0 +1,5 @@ +# Sphinx inventory version 1 +# Project: foo +# Version: 1.0 +module mod foo.html +module.cls class foo.html diff --git a/tests/static/objects_v2.inv b/tests/static/objects_v2.inv new file mode 100644 index 0000000000000000000000000000000000000000..f620d7639f7b62d36171345b5a6eacdc79d694b4 GIT binary patch literal 236 zcmY#Z2rkIT%&Sny%qvUHE6FdaR47X=D$dN$Q!wIERtPA{&q_@$u~G=AEXl~v1B!$} zWUUl{?2wF9g`(8l#LT>u)FOraG=-9k%wmPK%$!sOAf23_TTql*T%4MsP+FXsm#$Ei zlbNK)RdLJP|Lo~A-kxg%H1s?-p7QkZIvaSwG{mF5>s9KMC(kr0nr6gsq-y>=so?6N z8 gF1lPzOf`LhR&$5r8Q!)N>?+Ha7cnx--x#+D0Popd3IG5A literal 0 HcmV?d00001 diff --git a/tests/test_cli.py b/tests/test_cli.py index 4725f930..5d77b625 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,11 +1,16 @@ +from io import StringIO +from pathlib import Path from unittest import mock +import pytest + from myst_parser.cli import print_anchors +from myst_parser.inventory import inventory_cli +STATIC = Path(__file__).parent.absolute() / "static" -def test_print_anchors(): - from io import StringIO +def test_print_anchors(): in_stream = StringIO("# a\n\n## b\n\ntext") out_stream = StringIO() with mock.patch("sys.stdin", in_stream): @@ -13,3 +18,16 @@ def test_print_anchors(): print_anchors(["-l", "1"]) out_stream.seek(0) assert out_stream.read().strip() == '

' + + +@pytest.mark.parametrize("options", [(), ("-d", "std"), ("-o", "doc"), ("-n", "ref")]) +def test_read_inv(options, capsys, file_regression): + inventory_cli([str(STATIC / "objects_v2.inv"), "-f", "yaml", *options]) + text = capsys.readouterr().out.strip() + "\n" + file_regression.check(text, extension=".yaml") + + +def test_read_inv_v1(capsys, file_regression): + inventory_cli([str(STATIC / "objects_v1.inv"), "-f", "yaml"]) + text = capsys.readouterr().out.strip() + "\n" + file_regression.check(text, extension=".yaml") diff --git a/tests/test_cli/test_read_inv_options0_.yaml b/tests/test_cli/test_read_inv_options0_.yaml new file mode 100644 index 00000000..d51b779f --- /dev/null +++ b/tests/test_cli/test_read_inv_options0_.yaml @@ -0,0 +1,24 @@ +name: Python +version: '' +objects: + std: + label: + genindex: + loc: genindex.html + text: Index + modindex: + loc: py-modindex.html + text: Module Index + py-modindex: + loc: py-modindex.html + text: Python Module Index + ref: + loc: index.html#ref + text: Title + search: + loc: search.html + text: Search Page + doc: + index: + loc: index.html + text: Title diff --git a/tests/test_cli/test_read_inv_options1_.yaml b/tests/test_cli/test_read_inv_options1_.yaml new file mode 100644 index 00000000..d51b779f --- /dev/null +++ b/tests/test_cli/test_read_inv_options1_.yaml @@ -0,0 +1,24 @@ +name: Python +version: '' +objects: + std: + label: + genindex: + loc: genindex.html + text: Index + modindex: + loc: py-modindex.html + text: Module Index + py-modindex: + loc: py-modindex.html + text: Python Module Index + ref: + loc: index.html#ref + text: Title + search: + loc: search.html + text: Search Page + doc: + index: + loc: index.html + text: Title diff --git a/tests/test_cli/test_read_inv_options2_.yaml b/tests/test_cli/test_read_inv_options2_.yaml new file mode 100644 index 00000000..9ea4200f --- /dev/null +++ b/tests/test_cli/test_read_inv_options2_.yaml @@ -0,0 +1,8 @@ +name: Python +version: '' +objects: + std: + doc: + index: + loc: index.html + text: Title diff --git a/tests/test_cli/test_read_inv_options3_.yaml b/tests/test_cli/test_read_inv_options3_.yaml new file mode 100644 index 00000000..e64e40ee --- /dev/null +++ b/tests/test_cli/test_read_inv_options3_.yaml @@ -0,0 +1,8 @@ +name: Python +version: '' +objects: + std: + label: + ref: + loc: index.html#ref + text: Title diff --git a/tests/test_cli/test_read_inv_v1.yaml b/tests/test_cli/test_read_inv_v1.yaml new file mode 100644 index 00000000..1eb78b85 --- /dev/null +++ b/tests/test_cli/test_read_inv_v1.yaml @@ -0,0 +1,12 @@ +name: foo +version: '1.0' +objects: + py: + module: + module: + loc: foo.html#module-module + text: null + class: + module.cls: + loc: foo.html#module.cls + text: null From 9d506169c0f9023aaad3139c52eb1fabe526fc71 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Tue, 13 Dec 2022 16:39:13 +0100 Subject: [PATCH 2/5] fix compat --- myst_parser/_compat.py | 10 ++++++++-- myst_parser/inventory.py | 3 ++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/myst_parser/_compat.py b/myst_parser/_compat.py index da6a8d76..8cd9128f 100644 --- a/myst_parser/_compat.py +++ b/myst_parser/_compat.py @@ -5,9 +5,15 @@ from docutils.nodes import Element if sys.version_info >= (3, 8): - from typing import Literal, Protocol, get_args, get_origin # noqa: F401 + from typing import Literal, Protocol, TypedDict, get_args, get_origin # noqa: F401 else: - from typing_extensions import Literal, Protocol, get_args, get_origin # noqa: F401 + from typing_extensions import ( # noqa: F401 + Literal, + Protocol, + TypedDict, + get_args, + get_origin, + ) def findall(node: Element) -> Callable[..., Iterable[Element]]: diff --git a/myst_parser/inventory.py b/myst_parser/inventory.py index ff3bcc58..813c3bb8 100644 --- a/myst_parser/inventory.py +++ b/myst_parser/inventory.py @@ -17,7 +17,8 @@ from urllib.request import urlopen import yaml -from typing_extensions import TypedDict + +from ._compat import TypedDict if TYPE_CHECKING: from sphinx.util.typing import Inventory From 5085a5b12916d246deebe9db1d2ecf1396c0d776 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Tue, 13 Dec 2022 16:54:46 +0100 Subject: [PATCH 3/5] Update .github/workflows/tests.yml --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2984d73f..c05356e6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -99,7 +99,7 @@ jobs: else: raise AssertionError()" - name: Run pytest for docutils-only tests - run: pytest tests/test_docutils.py tests/test_renderers/test_fixtures_docutils.py tests/test_renderers/test_include_directive.py tests/test_renderers/test_myst_config.py + run: pytest tests/test_docutils.py tests/test_renderers/test_fixtures_docutils.py tests/test_renderers/test_include_directive.py - name: Run docutils CLI run: echo "test" | myst-docutils-html From 132ad76c08373f337a154ed24235821019b8fa6a Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Tue, 13 Dec 2022 18:09:50 +0100 Subject: [PATCH 4/5] Add more tests --- myst_parser/inventory.py | 45 ++++++++++++------- tests/test_anchors.py | 14 ++++++ tests/test_cli.py | 33 -------------- tests/test_inventory.py | 38 ++++++++++++++++ .../test_inv_cli_v1.yaml} | 0 .../test_inv_cli_v2_options0_.yaml} | 0 .../test_inv_cli_v2_options1_.yaml} | 0 .../test_inv_cli_v2_options2_.yaml} | 0 .../test_inv_cli_v2_options3_.yaml} | 0 tests/test_inventory/test_inv_resolve.yml | 8 ++++ .../test_inv_resolve_fnmatch.yml | 32 +++++++++++++ 11 files changed, 122 insertions(+), 48 deletions(-) create mode 100644 tests/test_anchors.py delete mode 100644 tests/test_cli.py create mode 100644 tests/test_inventory.py rename tests/{test_cli/test_read_inv_v1.yaml => test_inventory/test_inv_cli_v1.yaml} (100%) rename tests/{test_cli/test_read_inv_options0_.yaml => test_inventory/test_inv_cli_v2_options0_.yaml} (100%) rename tests/{test_cli/test_read_inv_options1_.yaml => test_inventory/test_inv_cli_v2_options1_.yaml} (100%) rename tests/{test_cli/test_read_inv_options2_.yaml => test_inventory/test_inv_cli_v2_options2_.yaml} (100%) rename tests/{test_cli/test_read_inv_options3_.yaml => test_inventory/test_inv_cli_v2_options3_.yaml} (100%) create mode 100644 tests/test_inventory/test_inv_resolve.yml create mode 100644 tests/test_inventory/test_inv_resolve_fnmatch.yml diff --git a/myst_parser/inventory.py b/myst_parser/inventory.py index 813c3bb8..80082199 100644 --- a/myst_parser/inventory.py +++ b/myst_parser/inventory.py @@ -11,7 +11,7 @@ import json import re import zlib -from dataclasses import dataclass +from dataclasses import asdict, dataclass from fnmatch import fnmatchcase from typing import IO, TYPE_CHECKING, Iterator from urllib.request import urlopen @@ -44,7 +44,7 @@ class InventoryType(TypedDict): """Mapping of domain -> object type -> name -> item.""" -def format_inventory(inv: Inventory) -> InventoryType: +def from_sphinx(inv: Inventory) -> InventoryType: """Convert a Sphinx inventory to one that is JSON compliant.""" project = "" version = "" @@ -69,6 +69,21 @@ def format_inventory(inv: Inventory) -> InventoryType: } +def to_sphinx(inv: InventoryType) -> Inventory: + """Convert a JSON compliant inventory to one that is Sphinx compliant.""" + objs: Inventory = {} + for domain_name, obj_types in inv["objects"].items(): + for obj_type, refs in obj_types.items(): + for refname, refdata in refs.items(): + objs.setdefault(f"{domain_name}:{obj_type}", {})[refname] = ( + inv["name"], + inv["version"], + refdata["loc"], + refdata["text"] or "-", + ) + return objs + + def load(stream: IO) -> InventoryType: """Load inventory data from a stream.""" reader = InventoryFileReader(stream) @@ -223,16 +238,19 @@ class InvMatch: uri: str text: str + def asdict(self) -> dict[str, str]: + return asdict(self) -def resolve_inventory( + +def filter_inventories( inventories: dict[str, Inventory], ref_target: str, *, ref_inv: None | str = None, ref_domain: None | str = None, ref_otype: None | str = None, - match_end=False, -) -> list[InvMatch]: + fnmatch_target=False, +) -> Iterator[InvMatch]: """Resolve a cross-reference in the loaded sphinx inventories. :param inventories: Mapping of inventory name to inventory data @@ -242,11 +260,10 @@ def resolve_inventory( :param ref_domain: The name of the domain to search, if None then all domains will be searched :param ref_otype: The type of object to search for, if None then all types will be searched - :param match_end: Whether to match against targets that end with the ref_target + :param fnmatch_target: Whether to match ref_target using fnmatchcase - :returns: matching results + :yields: matching results """ - results: list[InvMatch] = [] for inv_name, inv_data in inventories.items(): if ref_inv is not None and ref_inv != inv_name: @@ -265,23 +282,21 @@ def resolve_inventory( if ref_otype is not None and ref_otype != obj_type: continue - if not match_end and ref_target in data: - results.append( + if not fnmatch_target and ref_target in data: + yield ( InvMatch( inv_name, domain_name, obj_type, ref_target, *data[ref_target] ) ) - elif match_end: + elif fnmatch_target: for target in data: - if target.endswith(ref_target): - results.append( + if fnmatchcase(target, ref_target): + yield ( InvMatch( inv_name, domain_name, obj_type, target, *data[target] ) ) - return results - def inventory_cli(inputs: None | list[str] = None): """Command line interface for fetching and parsing an inventory.""" diff --git a/tests/test_anchors.py b/tests/test_anchors.py new file mode 100644 index 00000000..8092f183 --- /dev/null +++ b/tests/test_anchors.py @@ -0,0 +1,14 @@ +from io import StringIO +from unittest import mock + +from myst_parser.cli import print_anchors + + +def test_print_anchors(): + in_stream = StringIO("# a\n\n## b\n\ntext") + out_stream = StringIO() + with mock.patch("sys.stdin", in_stream): + with mock.patch("sys.stdout", out_stream): + print_anchors(["-l", "1"]) + out_stream.seek(0) + assert out_stream.read().strip() == '

' diff --git a/tests/test_cli.py b/tests/test_cli.py deleted file mode 100644 index 5d77b625..00000000 --- a/tests/test_cli.py +++ /dev/null @@ -1,33 +0,0 @@ -from io import StringIO -from pathlib import Path -from unittest import mock - -import pytest - -from myst_parser.cli import print_anchors -from myst_parser.inventory import inventory_cli - -STATIC = Path(__file__).parent.absolute() / "static" - - -def test_print_anchors(): - in_stream = StringIO("# a\n\n## b\n\ntext") - out_stream = StringIO() - with mock.patch("sys.stdin", in_stream): - with mock.patch("sys.stdout", out_stream): - print_anchors(["-l", "1"]) - out_stream.seek(0) - assert out_stream.read().strip() == '

' - - -@pytest.mark.parametrize("options", [(), ("-d", "std"), ("-o", "doc"), ("-n", "ref")]) -def test_read_inv(options, capsys, file_regression): - inventory_cli([str(STATIC / "objects_v2.inv"), "-f", "yaml", *options]) - text = capsys.readouterr().out.strip() + "\n" - file_regression.check(text, extension=".yaml") - - -def test_read_inv_v1(capsys, file_regression): - inventory_cli([str(STATIC / "objects_v1.inv"), "-f", "yaml"]) - text = capsys.readouterr().out.strip() + "\n" - file_regression.check(text, extension=".yaml") diff --git a/tests/test_inventory.py b/tests/test_inventory.py new file mode 100644 index 00000000..229872a2 --- /dev/null +++ b/tests/test_inventory.py @@ -0,0 +1,38 @@ +"""Test reading of inventory files.""" +from pathlib import Path + +import pytest + +from myst_parser.inventory import filter_inventories, inventory_cli, load, to_sphinx + +STATIC = Path(__file__).parent.absolute() / "static" + + +def test_inv_resolve(data_regression): + with (STATIC / "objects_v2.inv").open("rb") as f: + inv = to_sphinx(load(f)) + output = [m.asdict() for m in filter_inventories({"inv": inv}, "index")] + data_regression.check(output) + + +def test_inv_resolve_fnmatch(data_regression): + with (STATIC / "objects_v2.inv").open("rb") as f: + inv = to_sphinx(load(f)) + output = [ + m.asdict() + for m in filter_inventories({"inv": inv}, "*index", fnmatch_target=True) + ] + data_regression.check(output) + + +@pytest.mark.parametrize("options", [(), ("-d", "std"), ("-o", "doc"), ("-n", "ref")]) +def test_inv_cli_v2(options, capsys, file_regression): + inventory_cli([str(STATIC / "objects_v2.inv"), "-f", "yaml", *options]) + text = capsys.readouterr().out.strip() + "\n" + file_regression.check(text, extension=".yaml") + + +def test_inv_cli_v1(capsys, file_regression): + inventory_cli([str(STATIC / "objects_v1.inv"), "-f", "yaml"]) + text = capsys.readouterr().out.strip() + "\n" + file_regression.check(text, extension=".yaml") diff --git a/tests/test_cli/test_read_inv_v1.yaml b/tests/test_inventory/test_inv_cli_v1.yaml similarity index 100% rename from tests/test_cli/test_read_inv_v1.yaml rename to tests/test_inventory/test_inv_cli_v1.yaml diff --git a/tests/test_cli/test_read_inv_options0_.yaml b/tests/test_inventory/test_inv_cli_v2_options0_.yaml similarity index 100% rename from tests/test_cli/test_read_inv_options0_.yaml rename to tests/test_inventory/test_inv_cli_v2_options0_.yaml diff --git a/tests/test_cli/test_read_inv_options1_.yaml b/tests/test_inventory/test_inv_cli_v2_options1_.yaml similarity index 100% rename from tests/test_cli/test_read_inv_options1_.yaml rename to tests/test_inventory/test_inv_cli_v2_options1_.yaml diff --git a/tests/test_cli/test_read_inv_options2_.yaml b/tests/test_inventory/test_inv_cli_v2_options2_.yaml similarity index 100% rename from tests/test_cli/test_read_inv_options2_.yaml rename to tests/test_inventory/test_inv_cli_v2_options2_.yaml diff --git a/tests/test_cli/test_read_inv_options3_.yaml b/tests/test_inventory/test_inv_cli_v2_options3_.yaml similarity index 100% rename from tests/test_cli/test_read_inv_options3_.yaml rename to tests/test_inventory/test_inv_cli_v2_options3_.yaml diff --git a/tests/test_inventory/test_inv_resolve.yml b/tests/test_inventory/test_inv_resolve.yml new file mode 100644 index 00000000..8ac5d5f0 --- /dev/null +++ b/tests/test_inventory/test_inv_resolve.yml @@ -0,0 +1,8 @@ +- domain: std + inv: inv + name: index + otype: doc + proj: Python + text: Title + uri: index.html + version: '' diff --git a/tests/test_inventory/test_inv_resolve_fnmatch.yml b/tests/test_inventory/test_inv_resolve_fnmatch.yml new file mode 100644 index 00000000..aa24f602 --- /dev/null +++ b/tests/test_inventory/test_inv_resolve_fnmatch.yml @@ -0,0 +1,32 @@ +- domain: std + inv: inv + name: genindex + otype: label + proj: Python + text: Index + uri: genindex.html + version: '' +- domain: std + inv: inv + name: modindex + otype: label + proj: Python + text: Module Index + uri: py-modindex.html + version: '' +- domain: std + inv: inv + name: py-modindex + otype: label + proj: Python + text: Python Module Index + uri: py-modindex.html + version: '' +- domain: std + inv: inv + name: index + otype: doc + proj: Python + text: Title + uri: index.html + version: '' From 4765b09c55fb5fbd1a777d4facab9c7c0756235e Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Tue, 13 Dec 2022 18:18:28 +0100 Subject: [PATCH 5/5] more tests --- tests/test_inventory.py | 18 +++++++++++++++--- ...est_inv_resolve.yml => test_inv_filter.yml} | 0 ...fnmatch.yml => test_inv_filter_fnmatch.yml} | 0 3 files changed, 15 insertions(+), 3 deletions(-) rename tests/test_inventory/{test_inv_resolve.yml => test_inv_filter.yml} (100%) rename tests/test_inventory/{test_inv_resolve_fnmatch.yml => test_inv_filter_fnmatch.yml} (100%) diff --git a/tests/test_inventory.py b/tests/test_inventory.py index 229872a2..f6d1b3d2 100644 --- a/tests/test_inventory.py +++ b/tests/test_inventory.py @@ -3,19 +3,31 @@ import pytest -from myst_parser.inventory import filter_inventories, inventory_cli, load, to_sphinx +from myst_parser.inventory import ( + filter_inventories, + from_sphinx, + inventory_cli, + load, + to_sphinx, +) STATIC = Path(__file__).parent.absolute() / "static" -def test_inv_resolve(data_regression): +def test_convert_roundtrip(): + with (STATIC / "objects_v2.inv").open("rb") as f: + inv = load(f) + assert inv == from_sphinx(to_sphinx(inv)) + + +def test_inv_filter(data_regression): with (STATIC / "objects_v2.inv").open("rb") as f: inv = to_sphinx(load(f)) output = [m.asdict() for m in filter_inventories({"inv": inv}, "index")] data_regression.check(output) -def test_inv_resolve_fnmatch(data_regression): +def test_inv_filter_fnmatch(data_regression): with (STATIC / "objects_v2.inv").open("rb") as f: inv = to_sphinx(load(f)) output = [ diff --git a/tests/test_inventory/test_inv_resolve.yml b/tests/test_inventory/test_inv_filter.yml similarity index 100% rename from tests/test_inventory/test_inv_resolve.yml rename to tests/test_inventory/test_inv_filter.yml diff --git a/tests/test_inventory/test_inv_resolve_fnmatch.yml b/tests/test_inventory/test_inv_filter_fnmatch.yml similarity index 100% rename from tests/test_inventory/test_inv_resolve_fnmatch.yml rename to tests/test_inventory/test_inv_filter_fnmatch.yml