Use blib2to3 parser from black package to support match statement (#80

)

* Use blib2to3 parser to support match statement

The built-in lib2to3 does not support pattern matching (Python 3.10+):

https://docs.python.org/3.11/library/2to3.html#module-lib2to3

The [black][] project managed to get some level of parsing support for
`match` out of their modified version `blib2to3`, see:

1.  psf/black#2242
2.  psf/black#2586

[black]: https://github.com/psf/black

This change adds `black` as a dependency and switches to using
`blib2to3` to parse. Tests pass, but that's all that's been attempted
thus far.

* Add a _unreleased changelog for blib2to3 integration

* fix mypy in docspec/src/docspec/__init__.py

* fix mypy

* update changelog format

* update GitHub workflow

* fix workflow

* insert PR url

* use `--no-venv-check` also for `slap run` in docs job

---------

Co-authored-by: Niklas Rosenstein <rosensteinniklas@gmail.com>

.github/workflows/python.yml

@@ -17,9 +17,9 @@ jobs:
         python-version: ["3.7", "3.8", "3.9", "3.10", "3.x"]
         project: ["docspec", "docspec-python"]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - uses: NiklasRosenstein/slap@gha/install/v1
-    - uses: actions/setup-python@v2
+    - uses: actions/setup-python@v3
       with: { python-version: "${{ matrix.python-version }}" }
     - run: slap install --only ${{ matrix.project }} --no-venv-check -v
     - run: DOCSPEC_TEST_NO_DEVELOP=true slap test ${{ matrix.project }}
@@ -28,16 +28,16 @@ jobs:
     runs-on: ubuntu-latest
     if: github.event_name == 'pull_request'
     steps:
-      - uses: actions/checkout@v2
-      - uses: NiklasRosenstein/slap@gha/changelog/update/v1
+      - uses: actions/checkout@v3
+      - uses: NiklasRosenstein/slap@gha/changelog/update/v2
 
   docs:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - uses: NiklasRosenstein/slap@gha/install/v1
-    - run: slap install --only-extras docs --no-venv-check
-    - run: slap run docs:build
+    - run: slap install --no-venv-check --only-extras docs
+    - run: slap run --no-venv-check docs:build
     - uses: JamesIves/github-pages-deploy-action@4.1.4
       if: github.ref == 'refs/heads/develop'
       with: { branch: gh-pages, folder: docs/_site, ssh-key: "${{ secrets.DEPLOY_KEY }}" }

docspec-python/.changelog/_unreleased.toml

@@ -0,0 +1,6 @@
+[[entries]]
+id = "8628524b-3376-45db-a676-240b00c20d08"
+type = "fix"
+description = "Swap in `blib2to3` parser (bundled with the `black` package) for the stdlib `lib2to3` module in order to support `match` statements (PEP 634 - Structural Pattern Matching)."
+author = "@nrser"
+pr = "https://github.com/NiklasRosenstein/docspec/pull/80"

docspec-python/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "docspec-python"
-version = "2.0.2"
+version = "2.0.2+blib2to3"
 description = "A parser based on lib2to3 producing docspec data from Python source code."
 authors = ["Niklas Rosenstein <rosensteinniklas@gmail.com>"]
 license = "MIT"
@@ -12,6 +12,7 @@ packages = [{ include = "docspec_python", from="src" }]
 python = "^3.7"
 docspec = "^2.0.2"
 "nr.util" = ">=0.7.0"
+black = "^23.1.0"
 
 [tool.poetry.dev-dependencies]
 mypy = "*"
@@ -28,6 +29,10 @@ typed = true
 pytest = "pytest test/ -vv"
 mypy = "mypy src/ test/ --check-untyped-defs"
 
+[[tool.mypy.overrides]]
+module = "blib2to3.*"
+ignore_missing_imports = true
+
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"

docspec-python/src/docspec_python/__init__.py

@@ -46,10 +46,10 @@
 
 
 def load_python_modules(
-  modules: t.Sequence[str] = None,
-  packages: t.Sequence[str] = None,
-  search_path: t.Sequence[t.Union[str, Path]] = None,
-  options: ParserOptions = None,
+  modules: t.Optional[t.Sequence[str]] = None,
+  packages: t.Optional[t.Sequence[str]] = None,
+  search_path: t.Optional[t.Sequence[t.Union[str, Path]]] = None,
+  options: t.Optional[ParserOptions] = None,
   raise_: bool = True,
   encoding: t.Optional[str] = None,
 ) -> t.Iterable[Module]:
@@ -133,7 +133,7 @@ def parse_python_module(  # type: ignore
   return parser.parse(ast, filename, module_name)
 
 
-def find_module(module_name: str, search_path: t.Sequence[t.Union[str, Path]] = None) -> str:
+def find_module(module_name: str, search_path: t.Optional[t.Sequence[t.Union[str, Path]]] = None) -> str:
   """ Finds the filename of a module that can be parsed with #parse_python_module(). If *search_path* is not set,
   the default #sys.path is used to search for the module. If *module_name* is a Python package, it will return the
   path to the package's `__init__.py` file. If the module does not exist, an #ImportError is raised. This is also
@@ -165,7 +165,7 @@ def find_module(module_name: str, search_path: t.Sequence[t.Union[str, Path]] =
 
 def iter_package_files(
   package_name: str,
-  search_path: t.Sequence[t.Union[str, Path]] = None,
+  search_path: t.Optional[t.Sequence[t.Union[str, Path]]] = None,
 ) -> t.Iterable[t.Tuple[str, str]]:
   """ Returns an iterator for the Python source files in the specified package. The items returned
   by the iterator are tuples of the module name and filename. Supports a PEP 420 namespace package