feat: add CI documentation rules checker and docs-verify job

Enforce link resolution, glossary anchors, and readability rules on every
push and PR; docs-only pushes to main still run docs-verify while skipping
the simulator gate.

scripts/check_docs.py

@@ -0,0 +1,219 @@
+#!/usr/bin/env python3
+"""Validate markdown links, glossary anchors, and readability rules."""
+
+from __future__ import annotations
+
+import argparse
+import re
+import sys
+from pathlib import Path
+
+try:
+    import yaml
+except ImportError:  # pragma: no cover - stdlib fallback for minimal envs
+    yaml = None  # type: ignore[assignment]
+
+ROOT = Path(__file__).resolve().parents[1]
+RULES_PATH = Path(__file__).with_name("docs_rules.yaml")
+
+LINK_RE = re.compile(r"!?\[[^\]]*\]\(([^)]+)\)")
+BARE_SEE_RE = re.compile(r"(?<![\[(/])see\s+(\d{2})(?![\w-])", re.IGNORECASE)
+SKIP_SCHEMES = ("http://", "https://", "mailto:", "tel:", "#")
+
+
+def load_rules(path: Path = RULES_PATH) -> dict:
+    text = path.read_text(encoding="utf-8")
+    if yaml is not None:
+        return yaml.safe_load(text)
+    raise RuntimeError("PyYAML is required to load docs_rules.yaml")
+
+
+def slug_heading(text: str) -> str:
+    """Anchor slug used in this repository's glossary links."""
+    text = text.strip()
+    text = re.sub(r"\s&\s", " — ", text)
+    if " — " in text:
+        left, right = text.split(" — ", 1)
+        return f"{_slug_part(left)}--{_slug_part(right)}"
+    return _slug_part(text)
+
+
+def _slug_part(text: str) -> str:
+    slug = text.strip().lower()
+    slug = re.sub(r"[^\w\s-]", "", slug)
+    slug = re.sub(r"\s+", "-", slug)
+    return slug
+
+
+def collect_anchors(md_path: Path) -> set[str]:
+    anchors: set[str] = set()
+    for line in md_path.read_text(encoding="utf-8").splitlines():
+        if line.startswith("##"):
+            anchors.add(slug_heading(line.lstrip("#").strip()))
+    return anchors
+
+
+def expand_markdown_paths(patterns: list[str]) -> list[Path]:
+    files: set[Path] = set()
+    for pattern in patterns:
+        for path in ROOT.glob(pattern):
+            if path.is_file() and path.suffix == ".md":
+                files.add(path)
+    return sorted(files)
+
+
+def split_link_target(raw: str) -> tuple[str, str | None]:
+    target = raw.strip()
+    if target.startswith("<") and target.endswith(">"):
+        target = target[1:-1]
+    if "#" in target:
+        path_part, fragment = target.split("#", 1)
+        return path_part, fragment or None
+    return target, None
+
+
+def resolve_path(source: Path, link_path: str) -> Path | None:
+    if not link_path or link_path.startswith(SKIP_SCHEMES):
+        return None
+    candidate = (source.parent / link_path).resolve()
+    try:
+        candidate.relative_to(ROOT.resolve())
+    except ValueError:
+        return None
+    return candidate
+
+
+def is_glossary_link(link_path: str, rules: dict) -> bool:
+    glossary = rules["glossary"]["file"].replace("\\", "/")
+    normalized = link_path.replace("\\", "/")
+    return (
+        normalized == glossary
+        or normalized.endswith("/" + glossary)
+        or normalized.endswith(glossary.split("/")[-1])
+    )
+
+
+def check_file(
+    md_path: Path,
+    rules: dict,
+    anchor_cache: dict[Path, set[str]],
+) -> tuple[list[str], list[str]]:
+    errors: list[str] = []
+    warnings: list[str] = []
+    rel = md_path.relative_to(ROOT)
+    text = md_path.read_text(encoding="utf-8")
+
+    if rules.get("readability", {}).get("forbid_bare_see_refs"):
+        prose = re.sub(r"`[^`]*`", "", text)
+        for match in BARE_SEE_RE.finditer(prose):
+            errors.append(
+                f"{rel}:{_line_no(text, match.start())}: bare cross-ref "
+                f"'see {match.group(1)}' — use a markdown link"
+            )
+
+    for match in LINK_RE.finditer(text):
+        raw = match.group(1)
+        if any(raw.startswith(s) for s in SKIP_SCHEMES):
+            continue
+
+        link_path, fragment = split_link_target(raw)
+        if not link_path.endswith(".md"):
+            continue
+
+        for banned in rules.get("banned_link_targets", []):
+            if banned in link_path.replace("\\", "/"):
+                errors.append(
+                    f"{rel}:{_line_no(text, match.start())}: banned link target "
+                    f"'{link_path}'"
+                )
+
+        target = resolve_path(md_path, link_path)
+        if target is None:
+            errors.append(
+                f"{rel}:{_line_no(text, match.start())}: link escapes repo or "
+                f"is invalid: {raw}"
+            )
+            continue
+        if not target.is_file():
+            errors.append(
+                f"{rel}:{_line_no(text, match.start())}: broken link "
+                f"{link_path} -> {target.relative_to(ROOT)}"
+            )
+            continue
+
+        if fragment:
+            if target not in anchor_cache:
+                anchor_cache[target] = collect_anchors(target)
+            if fragment not in anchor_cache[target]:
+                errors.append(
+                    f"{rel}:{_line_no(text, match.start())}: unknown anchor "
+                    f"#{fragment} in {target.relative_to(ROOT)}"
+                )
+
+        if fragment and is_glossary_link(link_path, rules):
+            section_anchors = set(rules.get("glossary_section_anchors", []))
+            if fragment in section_anchors:
+                warnings.append(
+                    f"{rel}:{_line_no(text, match.start())}: glossary section "
+                    f"link #{fragment} — prefer a ### term anchor where one exists"
+                )
+
+    return errors, warnings
+
+
+def _line_no(text: str, index: int) -> int:
+    return text.count("\n", 0, index) + 1
+
+
+def run_checks(warnings_as_errors: bool = False) -> int:
+    rules = load_rules(ROOT / "scripts/docs_rules.yaml")
+    anchor_cache: dict[Path, set[str]] = {}
+    all_errors: list[str] = []
+    all_warnings: list[str] = []
+
+    glossary_path = ROOT / rules["glossary"]["file"]
+    anchor_cache[glossary_path] = collect_anchors(glossary_path)
+
+    for md_path in expand_markdown_paths(rules["markdown_paths"]):
+        errors, warnings = check_file(md_path, rules, anchor_cache)
+        all_errors.extend(errors)
+        all_warnings.extend(warnings)
+
+    if all_warnings:
+        print("Documentation warnings:")
+        for msg in sorted(all_warnings):
+            print(f"  warning: {msg}")
+        print()
+
+    if all_errors:
+        print("Documentation errors:")
+        for msg in sorted(all_errors):
+            print(f"  error: {msg}")
+        print(f"\n{len(all_errors)} error(s), {len(all_warnings)} warning(s)")
+        return 1
+
+    if warnings_as_errors and all_warnings:
+        print(f"\n{len(all_warnings)} warning(s) treated as errors")
+        return 1
+
+    print(
+        f"Documentation checks passed "
+        f"({len(expand_markdown_paths(rules['markdown_paths']))} files, "
+        f"{len(all_warnings)} warning(s))"
+    )
+    return 0
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--warnings-as-errors",
+        action="store_true",
+        help="Exit non-zero when section-link warnings are present",
+    )
+    args = parser.parse_args()
+    sys.exit(run_checks(warnings_as_errors=args.warnings_as_errors))
+
+
+if __name__ == "__main__":
+    main()

scripts/docs_rules.yaml

@@ -0,0 +1,47 @@
+# Documentation rules enforced by CI (scripts/check_docs.py).
+# See docs/improvement/ci-rules.md for rationale and PR workflow.
+
+markdown_paths:
+  - docs/**/*.md
+  - README.md
+  - CONTRIBUTING.md
+  - AGENTS.md
+  - simulator/README.md
+  - prototype/README.md
+  - infra/**/README.md
+
+glossary:
+  file: docs/00-glossary.md
+
+# Relative link targets that must never reappear (known regressions).
+banned_link_targets:
+  - 03-storage-design.md
+
+# Glossary links to these section headers are allowed but reported as warnings
+# when a term-specific ### anchor exists (see preferred_term_anchors).
+glossary_section_anchors:
+  - data--storage
+  - swarm--robotics
+  - infrastructure--delivery
+
+# First-use vocabulary should link to these ### anchors, not section headers.
+preferred_term_anchors:
+  - t0--hot
+  - t1--warm
+  - t2--shared
+  - t3--warehouse
+  - t4--dev
+  - source-of-truth
+  - architecture-layer-1--ingestion
+  - architecture-layer-2--storage-and-transform
+  - architecture-layer-3--serving-and-sync
+  - pipeline-stage
+  - bulk-sync
+  - offload
+  - rclone
+  - adr
+  - asr
+
+readability:
+  # Fail when prose uses bare "see 05" instead of a markdown link.
+  forbid_bare_see_refs: true