eSlider
4fdc0ef4a8
- tools/kblib.py: ladybug schema, embeddings, FTS+vector, hybrid RRF
- bin/kb/{index,search,get,stats,eval}: corpus indexing + deduction search
- bin/facts/{extract,audit}: 2-source evidence acquisition + gates
- serve/: async Go HTTP server (goroutines, bounded worker pool), TDD
- docker/ flattened to root: compose.yaml + Dockerfile (multi-stage Go)
- docker scripts -> bin/ shebang pattern (kb-watch, docker-entrypoint)
- bin/ci/semver + tools/semver.py: conventional-commit semver release
- ci.yml: go tests + shell checks; drop release-please (PR toggle blocked)
84 lines
2.6 KiB
Python
84 lines
2.6 KiB
Python
import json
|
|
import re
|
|
from pathlib import Path
|
|
|
|
import mistune
|
|
|
|
|
|
def extract_frontmatter(text: str) -> tuple[dict, str]:
|
|
"""Return (frontmatter dict, body). Accepts leading --- yaml ---."""
|
|
if not text.startswith("---"):
|
|
return {}, text
|
|
end = text.find("\n---", 3)
|
|
if end == -1:
|
|
return {}, text
|
|
fm = text[3:end].strip()
|
|
body = text[end + 4 :]
|
|
meta: dict = {}
|
|
for line in fm.splitlines():
|
|
if ":" in line:
|
|
key, _, value = line.partition(":")
|
|
meta[key.strip()] = value.strip().strip("\"'")
|
|
return meta, body
|
|
|
|
|
|
def split_leafs(meta: dict, body: str) -> list[dict]:
|
|
"""Split a markdown body into leaf chunks on H2 (##) boundaries.
|
|
|
|
Each leaf keeps the document-level frontmatter (type, related) and gets
|
|
its own heading + text. H1 is treated as document title, prepended to the
|
|
first chunk.
|
|
"""
|
|
title = ""
|
|
lines = body.splitlines()
|
|
headers: list[tuple[str, int]] = []
|
|
for i, line in enumerate(lines):
|
|
if re.match(r"^# \S", line):
|
|
title = line.lstrip("#").strip()
|
|
elif re.match(r"^## \S", line):
|
|
headers.append((line.lstrip("##").strip(), i))
|
|
if not headers:
|
|
text = "\n".join(l for l in lines if l.strip())
|
|
return [{"heading": title, "text": text.strip()}]
|
|
|
|
leafs: list[dict] = []
|
|
for idx, (heading, start) in enumerate(headers):
|
|
end = headers[idx + 1][1] if idx + 1 < len(headers) else len(lines)
|
|
chunk = "\n".join(l for l in lines[start:end] if l.strip())
|
|
text = chunk
|
|
if idx == 0 and title:
|
|
text = f"{title}\n\n{chunk}"
|
|
leafs.append({"heading": heading, "text": text.strip()})
|
|
return leafs
|
|
|
|
|
|
def to_all(text: str, path: str | Path, repo: str = "") -> list[dict]:
|
|
meta, body = extract_frontmatter(text)
|
|
meta.setdefault("type", "reference")
|
|
meta.setdefault("status", "current")
|
|
path = str(path)
|
|
leafs = split_leafs(meta, body)
|
|
out = []
|
|
for lf in leafs:
|
|
out.append({
|
|
"source": path,
|
|
"repo": repo,
|
|
"heading": lf["heading"],
|
|
"text": lf["text"],
|
|
"type": meta.get("type", "reference"),
|
|
"status": meta.get("status", "current"),
|
|
"related": meta.get("related", ""),
|
|
})
|
|
return out
|
|
|
|
|
|
def read_markdown(path: Path) -> str:
|
|
return path.read_text(encoding="utf-8", errors="replace")
|
|
|
|
|
|
def walk_markdown(root: Path) -> list[Path]:
|
|
return sorted(p for p in root.rglob("*") if p.suffix.lower() in (".md", ".markdown"))
|
|
|
|
|
|
def leaves_to_json(leaves: list[dict]) -> str:
|
|
return json.dumps(leaves, ensure_ascii=False, indent=2) |