-
-
Notifications
You must be signed in to change notification settings - Fork 2.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
New
get-deps
command: infer PyPI depedencies from mkdocs.yml
The user story is that the following command should let you "just build" any MkDocs site: pip install $(mkdocs get-deps) && mkdocs build This cross-references 2 files: * mkdocs.yml - `theme`, `plugins`, `markdown_extensions` * projects.yaml - a registry of all popular MkDocs-related projects and which entry points they provide - downloaded on the fly -and prints the names of Python packages from PyPI that need to be installed to build the current MkDocs project
- Loading branch information
Showing
4 changed files
with
223 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,129 @@ | ||
from __future__ import annotations | ||
|
||
import dataclasses | ||
import datetime | ||
import functools | ||
import logging | ||
from typing import Iterator, Mapping, Optional | ||
|
||
import yaml | ||
|
||
from mkdocs import utils | ||
from mkdocs.config.base import _open_config_file | ||
from mkdocs.plugins import EntryPoint, entry_points | ||
from mkdocs.utils.cache import download_and_cache_url | ||
|
||
log = logging.getLogger(__name__) | ||
|
||
|
||
def _extract_names(cfg, key: str) -> Iterator[str]: | ||
"""Get names of plugins/extensions from the config - in either a list of dicts or a dict.""" | ||
try: | ||
items = iter(cfg.get(key, ())) | ||
except TypeError: | ||
log.error(f"Invalid config entry '{key}'") | ||
for item in items: | ||
try: | ||
if not isinstance(item, str): | ||
[item] = item | ||
yield item | ||
except (ValueError, TypeError): | ||
log.error(f"Invalid config entry '{key}': {item}") | ||
|
||
|
||
@functools.lru_cache() | ||
def _entry_points(group: str) -> Mapping[str, EntryPoint]: | ||
eps = {ep.name: ep for ep in entry_points(group=group)} | ||
log.debug(f"Available '{group}' entry points: {sorted(eps)}") | ||
return eps | ||
|
||
|
||
@dataclasses.dataclass(frozen=True) | ||
class PluginKind: | ||
projects_key: str | ||
entry_points_key: str | ||
|
||
def __str__(self) -> str: | ||
return self.projects_key.rpartition('_')[-1] | ||
|
||
|
||
def get_deps(projects_file_url: str, config_file_path: Optional[str] = None) -> None: | ||
""" | ||
Print PyPI package dependencies inferred from a mkdocs.yml file based on a reverse mapping of known projects. | ||
Parameters: | ||
projects_file_url: URL or local path of the registry file that declares all known MkDocs-related projects. | ||
The file is in YAML format and contains `projects: [{mkdocs_theme:, mkdocs_plugin:, markdown_extension:}] | ||
config_file_path: Non-default path to mkdocs.yml. | ||
""" | ||
with _open_config_file(config_file_path) as f: | ||
cfg = utils.yaml_load(f) | ||
|
||
if all(c not in cfg for c in ('site_name', 'theme', 'plugins', 'markdown_extensions')): | ||
log.warning("The passed config file doesn't seem to be a mkdocs.yml config file") | ||
|
||
try: | ||
theme = cfg['theme']['name'] | ||
except (KeyError, TypeError): | ||
theme = cfg.get('theme') | ||
themes = {theme} if theme else set() | ||
|
||
plugins = set(_extract_names(cfg, 'plugins')) | ||
extensions = set(_extract_names(cfg, 'markdown_extensions')) | ||
|
||
wanted_plugins = ( | ||
(PluginKind('mkdocs_theme', 'mkdocs.themes'), themes - {'mkdocs', 'readthedocs'}), | ||
(PluginKind('mkdocs_plugin', 'mkdocs.plugins'), plugins - {'search'}), | ||
(PluginKind('markdown_extension', 'markdown.extensions'), extensions), | ||
) | ||
for kind, wanted in wanted_plugins: | ||
log.debug(f'Wanted {kind}s: {sorted(wanted)}') | ||
|
||
content = download_and_cache_url(projects_file_url, datetime.timedelta(days=7)) | ||
projects = yaml.safe_load(content)['projects'] | ||
|
||
packages_to_install = set() | ||
for project in projects: | ||
for kind, wanted in wanted_plugins: | ||
available = project.get(kind.projects_key, ()) | ||
if isinstance(available, str): | ||
available = (available,) | ||
for entry_name in available: | ||
if entry_name in wanted or ( | ||
# Also check theme-namespaced plugin names against the current theme. | ||
'/' in entry_name | ||
and theme is not None | ||
and kind.projects_key == 'mkdocs_plugin' | ||
and entry_name.startswith(f'{theme}/') | ||
and entry_name[len(theme) + 1 :] in wanted | ||
): | ||
if 'pypi_id' in project: | ||
install_name = project['pypi_id'] | ||
elif 'github_id' in project: | ||
install_name = 'git+https://github.com/{github_id}'.format_map(project) | ||
else: | ||
log.error( | ||
f"Can't find how to install {kind} '{entry_name}' although it was identified as {project}" | ||
) | ||
continue | ||
packages_to_install.add(install_name) | ||
wanted.remove(entry_name) | ||
|
||
for kind, wanted in wanted_plugins: | ||
for entry_name in sorted(wanted): | ||
dist_name = None | ||
ep = _entry_points(kind.entry_points_key).get(entry_name) | ||
if ep is not None and ep.dist is not None: | ||
dist_name = ep.dist.name | ||
if dist_name not in ('mkdocs', 'Markdown'): | ||
warning = f"{str(kind).capitalize()} '{entry_name}' is not provided by any registered project" | ||
if ep is not None: | ||
warning += " but is installed locally" | ||
if dist_name: | ||
warning += f" from '{dist_name}'" | ||
log.info(warning) | ||
else: | ||
log.warning(warning) | ||
|
||
for pkg in sorted(packages_to_install): | ||
print(pkg) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,64 @@ | ||
import datetime | ||
import hashlib | ||
import logging | ||
import os | ||
import urllib.parse | ||
import urllib.request | ||
|
||
import click | ||
import platformdirs | ||
|
||
log = logging.getLogger(__name__) | ||
|
||
|
||
def download_and_cache_url( | ||
url: str, | ||
cache_duration: datetime.timedelta, | ||
comment: bytes = b'# ', | ||
) -> bytes: | ||
""" | ||
Downloads a file from the URL, stores it under ~/.cache/, and returns its content. | ||
If the URL is a local path, it is simply read and returned instead. | ||
For tracking the age of the content, a prefix is inserted into the stored file, rather than relying on mtime. | ||
Parameters: | ||
url: URL or local path of the file to use. | ||
cache_duration: how long to consider the URL content cached. | ||
comment: The appropriate comment prefix for this file format. | ||
""" | ||
|
||
if urllib.parse.urlsplit(url).scheme not in ('http', 'https'): | ||
with open(url, 'rb') as f: | ||
return f.read() | ||
|
||
directory = os.path.join(platformdirs.user_cache_dir('mkdocs'), 'mkdocs_url_cache') | ||
name_hash = hashlib.sha256(url.encode()).hexdigest()[:32] | ||
path = os.path.join(directory, name_hash + os.path.splitext(url)[1]) | ||
|
||
now = int(datetime.datetime.now(datetime.timezone.utc).timestamp()) | ||
prefix = b'%s%s downloaded at timestamp ' % (comment, url.encode()) | ||
# Check for cached file and try to return it | ||
if os.path.isfile(path): | ||
try: | ||
with open(path, 'rb') as f: | ||
line = f.readline() | ||
if line.startswith(prefix): | ||
line = line[len(prefix) :] | ||
timestamp = int(line) | ||
if datetime.timedelta(seconds=(now - timestamp)) <= cache_duration: | ||
log.debug(f"Using cached '{path}' for '{url}'") | ||
return f.read() | ||
except (IOError, ValueError) as e: | ||
log.debug(f'{type(e).__name__}: {e}') | ||
|
||
# Download and cache the file | ||
log.debug(f"Downloading '{url}' to '{path}'") | ||
with urllib.request.urlopen(url) as resp: | ||
content = resp.read() | ||
os.makedirs(directory, exist_ok=True) | ||
with click.open_file(path, 'wb', atomic=True) as f: | ||
f.write(b'%s%d\n' % (prefix, now)) | ||
f.write(content) | ||
return content |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters