# Copyright Kevin Deldycke <[email protected]> and contributors.
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
"""Regenerate Sphinx API docs and dynamic documentation content.
Backs the `update-docs` command, which runs the phases of {class}`DocsPhase`
in order: `sphinx-apidoc`, the RST-to-MyST conversion, the toctree wiring, the
project's `docs/docs_update.py` script, the reformat of a `pyproject.toml` that
script changed, and the self-updating directive-block refresh.
{func}`update_docs` returns one {class}`PhaseResult` per phase it ran, and the
command reports them. Configuration is read from `[tool.repomatic.docs]`.
"""
from __future__ import annotations
import logging
import re
import subprocess
from dataclasses import dataclass
from enum import Enum, auto
from pathlib import Path
from click_extra import convert_rst_files_in_directory
from .deps.uv import uv_cmd
from .file_inventory import FileInventory
from .metadata.core import Metadata
from .tooling.registry import ToolInstallError
from .tooling.runner import run_tool
TYPE_CHECKING = False
if TYPE_CHECKING:
from collections.abc import Callable, Mapping
from .config import Config
DIRECTIVE_BLOCK_MARKERS: tuple[str, ...] = (
"{matrix}",
"<!-- matrix",
"<!-- envelope",
":mirror:",
"<!-- mirror",
)
"""Markers of a self-updating block `click-extra refresh-directives` rewrites.
Every form the refresh recognizes: the ``{matrix}`` MyST fence (live-rendered by
Sphinx), the `<!-- matrix -->` comment region (whose embedded table renders on
GitHub too), the `<!-- envelope -->` region listing the obsolete bounds of the
support envelope, and the `python:render` `:mirror:` region (`<!-- mirror -->`,
whose generator Python the refresh executes).
"""
TOCTREE_ENTRY_RE = re.compile(r"^[ \t]*(\w+(?:\.\w+)+)\s*$", re.MULTILINE)
"""A `toctree` entry of a generated index page: a dotted module name alone on its line.
The reST directive that `click_extra` wraps in an ``{eval-rst}`` block indents
its entries. A native MyST ``{toctree}`` fence holds them at the margin.
"""
[docs]
class DocsPhase(Enum):
"""One phase of `update-docs`, in the order the phases run.
The help of the command lists the phases from this registry, so the two
cannot drift apart.
"""
APIDOC = (
"sphinx-apidoc",
"Run sphinx-apidoc to generate the RST stub of every module.",
)
CONVERSION = (
"RST conversion",
"Convert the RST stubs to MyST Markdown, if the project uses MyST-Parser.",
)
TOCTREE = (
"Toctree wiring",
"List every generated page in the toctree of its parent index page.",
)
SCRIPT = (
"Docs update script",
"Run docs/docs_update.py (docs.update-script), if it exists.",
)
PYPROJECT = (
"pyproject.toml reformat",
"Reformat with pyproject-fmt each pyproject.toml an earlier phase changed.",
)
DIRECTIVES = (
"Directive-block refresh",
(
"Refresh the {matrix}, <!-- envelope --> and :mirror: blocks of docs/"
" and readme.md."
),
)
def __init__(self, label: str, description: str) -> None:
self.label = label
"""Short name of the phase, for a report line."""
self.description = description
"""What the phase does, in the plain text of the command's help."""
@property
def checks(self) -> bool:
"""Whether the phase has a check mode, so `update-docs --check` runs it.
The other phases write files and have no dry-run mode.
"""
return self in {DocsPhase.SCRIPT, DocsPhase.DIRECTIVES}
[docs]
class PhaseStatus(Enum):
"""How one phase of `update-docs` ended."""
DONE = auto()
"""The phase ran, and wrote or found everything up to date."""
SKIPPED = auto()
"""The phase has nothing to do in this project."""
DRIFT = auto()
"""In check mode, the phase found out-of-date content."""
FAILED = auto()
"""The phase failed, which stops the phases after it."""
[docs]
@dataclass(frozen=True)
class PhaseResult:
"""What one phase of `update-docs` did."""
phase: DocsPhase
"""The phase that ran."""
status: PhaseStatus
"""How it ended."""
message: str = ""
"""What to tell the user: the work done, the reason for a skip, the
out-of-date content, or the failure."""
[docs]
def validate_docs_script_path(script: str, repo_root: Path) -> Path | None:
"""Validate and resolve a docs update script path.
:param script: Configured `docs.update-script` path, relative to the repo.
:param repo_root: Repository root the script path resolves against.
:return: The resolved path, or `None` when the configured value is empty.
:raises ValueError: If the path escapes the repository root or is not a
`.py` file under `docs/`.
"""
if not script:
return None
script_path = (repo_root / script).resolve()
if not script_path.is_relative_to(repo_root):
msg = f"docs.update-script escapes repository root: {script}"
raise ValueError(msg)
if not script_path.is_relative_to((repo_root / "docs").resolve()):
msg = f"docs.update-script must be under docs/: {script}"
raise ValueError(msg)
if script_path.suffix != ".py":
msg = f"docs.update-script must be a .py file: {script}"
raise ValueError(msg)
return script_path
[docs]
def has_directive_block(path: Path) -> bool:
"""Whether *path* carries a self-updating block worth refreshing.
:param path: Markdown file to scan.
:return: `True` when any {data}`DIRECTIVE_BLOCK_MARKERS` entry appears.
"""
text = path.read_text(encoding="UTF-8")
return any(marker in text for marker in DIRECTIVE_BLOCK_MARKERS)
def _run_docs_tool(*args: str) -> int:
"""Run a tool from the `docs` dependency group through uv.
The command runs with the frozen lockfile and the `docs` group installed.
:param args: The command and its arguments, passed after `uv run --`.
:return: The command's exit code.
"""
cmd = [*uv_cmd("run", frozen=True), "--group", "docs", "--", *args]
logging.info(f"Running: {' '.join(cmd)}")
return subprocess.run(cmd, check=False).returncode
def _tool_result(
phase: DocsPhase, code: int, *, check: bool = False, label: str = ""
) -> PhaseResult:
"""Read the exit code of a phase's tool.
:param phase: The phase the tool ran for.
:param code: The tool's exit code.
:param check: Whether the tool ran in its check mode, where a non-zero
code means out-of-date content rather than a failure.
:param label: The name of the phase in the messages. Defaults to its
{attr}`DocsPhase.label`.
"""
label = label or phase.label
if not code:
message = "" if check else f"{label} completed."
return PhaseResult(phase, PhaseStatus.DONE, message)
if check:
return PhaseResult(phase, PhaseStatus.DRIFT, label)
return PhaseResult(
phase, PhaseStatus.FAILED, f"{label} failed with exit code {code}."
)
def _toctree_entries(page: Path) -> set[str]:
"""Return the dotted names listed in every `toctree` of an index page."""
return set(TOCTREE_ENTRY_RE.findall(page.read_text(encoding="UTF-8")))
def _insert_into_toctree(page: Path, section: str, entry: str) -> bool:
"""Add `entry` to the `toctree` under `section` of `page`, sorted.
:param page: The index page to rewrite.
:param section: Heading whose `toctree` receives the entry, like
`Submodules`.
:param entry: Dotted module name to insert.
:return: `True` when the page was rewritten, `False` when it carries no
such section.
"""
lines = page.read_text(encoding="UTF-8").splitlines(keepends=True)
try:
start = lines.index(f"## {section}\n")
except ValueError:
return False
# The entries are the dotted names of the first toctree below the heading.
# Collect that one run, then splice the new name into it.
run: list[int] = []
for offset, line in enumerate(lines[start:], start=start):
if TOCTREE_ENTRY_RE.fullmatch(line):
run.append(offset)
elif run:
break
if not run:
return False
first = lines[run[0]]
indent = first[: len(first) - len(first.lstrip())]
block = sorted([*lines[run[0] : run[-1] + 1], f"{indent}{entry}\n"])
lines[run[0] : run[-1] + 1] = block
page.write_text("".join(lines), encoding="UTF-8")
return True
def _is_package_page(page: Path) -> bool:
"""Whether a generated page documents a package rather than a module.
`sphinx-apidoc` titles a page `<name> package` or `<name> module`, and the
title is the one place that says which. Without `--separate` the modules of
a package sit inline in its page, so a package page has no page below it.
"""
title = page.read_text(encoding="UTF-8").partition("\n")[0]
return title.startswith("# ") and title.rstrip().endswith(" package")
[docs]
def wire_api_pages_into_toctrees(docs_dir: Path) -> list[str]:
"""List every generated API page in the `toctree` of its parent index.
`sphinx-apidoc --no-toc` writes one page per module and no package index,
so a module added after an index was last written stays orphaned: the page
exists, nothing points at it, and Sphinx reports
`document isn't included in any toctree` as a warning the build survives.
Wiring it here is what keeps the roster whole without a hand edit per
module, and it is the half {func}`click_extra.convert_rst_files_in_directory`
cannot do, since that one only declines to overwrite an existing page.
A page goes under `Subpackages` when it documents a package: it is itself
an index, or its title says so. It goes under `Submodules` otherwise. An
index missing the section it needs is left alone and named in the return
value, since inventing a heading would guess at a page layout this function
does not own.
:param docs_dir: The `docs/` directory holding the generated pages.
:return: One `<page> -> <index>` line per entry inserted, plus a
`<page>: <reason>` line per page that could not be wired.
"""
stems = {path.stem for path in docs_dir.glob("*.md")}
# An index is any page some other page's dotted name sits below.
indexes = {
stem
for stem in stems
if any(o.startswith(f"{stem}.") for o in stems if o != stem)
}
report: list[str] = []
for stem in sorted(stems):
if "." not in stem:
continue
parts = stem.split(".")
parent = next(
(
".".join(parts[:cut])
for cut in range(len(parts) - 1, 0, -1)
if ".".join(parts[:cut]) in indexes
),
None,
)
if parent is None:
report.append(f"{stem}: no parent index page")
continue
index = docs_dir / f"{parent}.md"
if stem in _toctree_entries(index):
continue
is_package = stem in indexes or _is_package_page(docs_dir / f"{stem}.md")
section = "Subpackages" if is_package else "Submodules"
if _insert_into_toctree(index, section, stem):
report.append(f"{stem} -> {parent}.md ({section})")
else:
report.append(f"{stem}: {parent}.md carries no {section} toctree")
return report
def _apidoc(config: Config, meta: Metadata, docs_dir: Path) -> PhaseResult:
"""Run `sphinx-apidoc` over the project, when autodoc is active."""
phase = DocsPhase.APIDOC
if not meta.active_autodoc:
return PhaseResult(phase, PhaseStatus.SKIPPED, "No active autodoc extension.")
code = _run_docs_tool(
"sphinx-apidoc",
"--no-toc",
"--module-first",
"--output-dir",
str(docs_dir),
*config.docs.apidoc_extra_args,
".",
*config.docs.apidoc_exclude,
)
return _tool_result(phase, code)
def _convert(meta: Metadata, docs_dir: Path) -> PhaseResult:
"""Convert the RST stubs to MyST Markdown, when the project uses MyST."""
phase = DocsPhase.CONVERSION
if not meta.uses_myst:
return PhaseResult(phase, PhaseStatus.SKIPPED, "MyST-Parser not detected.")
if not docs_dir.is_dir():
return PhaseResult(phase, PhaseStatus.SKIPPED, f"No {docs_dir} directory.")
converted = convert_rst_files_in_directory(docs_dir)
if not converted:
return PhaseResult(phase, PhaseStatus.SKIPPED, "No RST file to convert.")
return PhaseResult(
phase,
PhaseStatus.DONE,
f"Converted {len(converted)} RST file(s) to MyST Markdown.",
)
def _wire(meta: Metadata, docs_dir: Path) -> PhaseResult:
"""List the generated pages in the toctree of their parent index."""
phase = DocsPhase.TOCTREE
if not meta.active_autodoc or not docs_dir.is_dir():
return PhaseResult(phase, PhaseStatus.SKIPPED, "No generated API page.")
wired = wire_api_pages_into_toctrees(docs_dir)
if not wired:
return PhaseResult(phase, PhaseStatus.DONE)
lines = "".join(f"\n {line}" for line in wired)
return PhaseResult(
phase, PhaseStatus.DONE, f"Wired API pages into their toctree:{lines}"
)
def _script(config: Config, repo_root: Path, check: bool) -> PhaseResult:
"""Run the project's docs update script, if it exists."""
phase = DocsPhase.SCRIPT
script_path = validate_docs_script_path(config.docs.update_script, repo_root)
if not script_path:
return PhaseResult(
phase, PhaseStatus.SKIPPED, "Docs update script disabled (empty path)."
)
if not script_path.is_file():
return PhaseResult(
phase, PhaseStatus.SKIPPED, f"Docs update script not found: {script_path}"
)
code = _run_docs_tool("python", str(script_path), *(["--check"] if check else []))
return _tool_result(
phase, code, check=check, label=f"{phase.label} ({script_path.name})"
)
def _reformat_pyproject(before: Mapping[Path, bytes]) -> PhaseResult:
"""Reformat each `pyproject.toml` an earlier phase of this run changed.
The docs update script may regenerate sections of `pyproject.toml`, like
label rules or keywords. Reformatting what it wrote keeps the output of
`update-docs` a `pyproject-fmt` fixpoint, so its pull request never fights
the one of the `format-pyproject` job. A file this run left alone keeps
any formatting drift it had: that drift belongs to `format-pyproject`.
:param before: The content of each `pyproject.toml` before the first phase.
"""
phase = DocsPhase.PYPROJECT
changed = [
str(path)
for path, content in before.items()
if path.is_file() and path.read_bytes() != content
]
if not changed:
return PhaseResult(phase, PhaseStatus.SKIPPED, "No pyproject.toml changed.")
try:
code = run_tool("pyproject-fmt", extra_args=changed)
except ToolInstallError as error:
return PhaseResult(phase, PhaseStatus.FAILED, str(error))
return _tool_result(phase, code)
def _refresh(repo_root: Path, docs_dir: Path, check: bool) -> PhaseResult:
"""Refresh the self-updating blocks of the docs and the readme.
Only files already carrying a block are passed, so repositories without
any stay clear of the sphinx extra that `refresh-directives` requires.
"""
phase = DocsPhase.DIRECTIVES
candidates = sorted(docs_dir.rglob("*.md")) if docs_dir.is_dir() else []
readme_path = repo_root / "readme.md"
if readme_path.is_file():
candidates.append(readme_path)
directive_files = [path for path in candidates if has_directive_block(path)]
if not directive_files:
return PhaseResult(
phase, PhaseStatus.SKIPPED, "No self-updating directive block found."
)
code = _run_docs_tool(
"click-extra",
"refresh-directives",
*(["--check"] if check else []),
*(str(path) for path in directive_files),
)
return _tool_result(phase, code, check=check)
[docs]
def update_docs(config: Config, *, check: bool = False) -> list[PhaseResult]:
"""Run the phases of {class}`DocsPhase` in order, and say how each ended.
A failed phase stops the phases after it. A project with no Sphinx
configuration runs no phase.
:param config: The resolved `[tool.repomatic]` configuration.
:param check: Report out-of-date content without writing, for CI drift
detection. Only the phases whose {attr}`DocsPhase.checks` is true
run, in their own check modes (`docs_update.py --check` and
`refresh-directives --check`). The update script must accept a
``--check`` flag to participate: a script that ignores it will still
write.
:return: One result per phase that ran.
:raises ValueError: When `docs.update-script` names a path outside
`docs/`, or a file that is not Python.
"""
repo_root = Path.cwd()
docs_dir = repo_root / "docs"
# Detect Sphinx capabilities from conf.py.
meta = Metadata()
if not meta.is_sphinx:
logging.info("No Sphinx configuration found. Nothing to do.")
return []
# What the phases start from, for the reformat to tell what they changed.
pyproject_before = {
path: path.read_bytes() for path in FileInventory().pyproject_files
}
runners: dict[DocsPhase, Callable[[], PhaseResult]] = {
DocsPhase.APIDOC: lambda: _apidoc(config, meta, docs_dir),
DocsPhase.CONVERSION: lambda: _convert(meta, docs_dir),
DocsPhase.TOCTREE: lambda: _wire(meta, docs_dir),
DocsPhase.SCRIPT: lambda: _script(config, repo_root, check),
DocsPhase.PYPROJECT: lambda: _reformat_pyproject(pyproject_before),
DocsPhase.DIRECTIVES: lambda: _refresh(repo_root, docs_dir, check),
}
results: list[PhaseResult] = []
for phase in DocsPhase:
if check and not phase.checks:
continue
result = runners[phase]()
results.append(result)
if result.status is PhaseStatus.FAILED:
break
return results