feat(kb): brain tools, Go async serve, root-level docker

- tools/kblib.py: ladybug schema, embeddings, FTS+vector, hybrid RRF - bin/kb/{index,search,get,stats,eval}: corpus indexing + deduction search - bin/facts/{extract,audit}: 2-source evidence acquisition + gates - serve/: async Go HTTP server (goroutines, bounded worker pool), TDD - docker/ flattened to root: compose.yaml + Dockerfile (multi-stage Go) - docker scripts -> bin/ shebang pattern (kb-watch, docker-entrypoint) - bin/ci/semver + tools/semver.py: conventional-commit semver release - ci.yml: go tests + shell checks; drop release-please (PR toggle blocked)
2026-08-10 21:58:25 +01:00
parent dcb00808c9
commit 4fdc0ef4a8
29 changed files with 1560 additions and 134 deletions
@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""kb/eval - recall@5 gate for the brain.
+
+  bin/kb/eval [--json]
+
+Control questions are answered from the graph; recall@5 >= 0.95 gates CI.
+Each question maps to leaf ids that MUST appear in the top 5 hits.
+"""
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(ROOT / "tools"))
+
+from kblib import open_readonly, query_fts  # noqa: E402
+from yamlout import to_yaml  # noqa: E402
+
+RECALL_THRESHOLD = 0.95
+
+# (query, expected leaf id)
+CONTROL_QUESTIONS: list[tuple[str, str]] = [
+    ("which database does the brain use", "facts:ladybug"),
+    ("hybrid search weights fts and vector equally", "info:hybrid"),
+]
+
+
+def hit_ids_of(query: str, conn, limit: int = 5) -> list[str]:
+    try:
+        hits = query_fts(conn, query, limit)
+        return [h["id"] for h in hits]
+    except Exception:
+        return []
+
+
+def main(argv: list[str]) -> int:
+    import argparse
+    p = argparse.ArgumentParser(description="recall@5 gate")
+    p.add_argument("--json", action="store_true")
+    a = p.parse_args(argv)
+
+    db, conn = open_readonly()
+    recalled = 0
+    detail = []
+    for query, expected in CONTROL_QUESTIONS:
+        hits = hit_ids_of(query, conn)
+        ok = any(expected in h or h in expected for h in hits)
+        recalled += int(ok)
+        detail.append({"q": query, "expected": expected, "in_top5": ok, "hits": hits[:5]})
+    recall = recalled / len(CONTROL_QUESTIONS) if CONTROL_QUESTIONS else 1.0
+    passed = recall >= RECALL_THRESHOLD
+    out = {"recall@5": round(recall, 3), "passed": passed, "gate": len(CONTROL_QUESTIONS), "details": detail}
+    print(json.dumps(out, indent=2) if a.json else to_yaml(out))
+    conn.close()
+    db.close()
+    return 0 if passed else 2
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))
@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+"""kb/get - read one leaf by id.
+
+  bin/kb/get <id>            # metadata + snippet
+  bin/kb/get <id> --body     # full text
+"""
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(ROOT / "tools"))
+
+from kblib import open_readonly  # noqa: E402
+from yamlout import to_yaml  # noqa: E402
+
+
+def main(argv: list[str]) -> int:
+    import argparse
+    p = argparse.ArgumentParser(description="read one leaf by id")
+    p.add_argument("id")
+    p.add_argument("--body", action="store_true")
+    a = p.parse_args(argv)
+
+    db, conn = open_readonly()
+    r = conn.execute(
+        "MATCH (l:Leaf {id:$id}) RETURN l.id, l.text, l.root, l.confidence, l.source, l.type",
+        parameters={"id": a.id},
+    )
+    rows = r.get_all()
+    if not rows:
+        print(f"kb/get: no leaf {a.id}", file=sys.stderr)
+        conn.close()
+        db.close()
+        return 1
+    row = rows[0]
+    out = {"id": row[0], "root": row[2], "confidence": row[3], "source": row[4], "type": row[5]}
+    if a.body:
+        out["text"] = row[1]
+    else:
+        out["snippet"] = row[1][:280]
+    print(to_yaml(out))
+    conn.close()
+    db.close()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))
@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""kb/index - build the 2dph brain from markdown + factual leafs.
+
+  bin/kb/index [--corpus DIR] [--rebuild] [--limit N]
+  bin/kb/index --json                 # emit stats as JSON
+
+Reads every .md under the corpus (default: repo root docs, skills, READMEs)
+as `info` leafs, embeds them with model2vec (potion-multilingual-128M), and
+writes them into var/kb.lbug with FTS + HNSW indexes. `facts` leafs come
+from bin/facts/extract (docker x compose x ssh-config pairing).
+
+--rebuild drops the database file and indexes from scratch. Without it a run
+is idempotent (MERGE by (source,text) id).
+"""
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(ROOT / "tools"))
+
+from kblib import (  # noqa: E402
+    connect, create_fts_and_vector, init_schema, upsert_leaf,
+    open_readonly, stats,
+)
+from mdleaves import read_markdown, to_all, walk_markdown  # noqa: E402
+
+CORPUS_DEFAULTS = ["README.md", "PLAN.md", "AGENTS.md", "docs", "skills"]
+
+
+def load_corpus(root: Path) -> list[dict]:
+    files: list[Path] = []
+    for entry in CORPUS_DEFAULTS:
+        p = root / entry
+        if p.is_file():
+            files.append(p)
+        elif p.is_dir():
+            files.extend(walk_markdown(p))
+    leafs: list[dict] = []
+    for path in files:
+        try:
+            leafs.extend(to_all(read_markdown(path), path, repo="eSlider/2dph"))
+        except OSError as e:
+            print(f"kb/index: skip {path}: {e}", file=sys.stderr)
+    return leafs
+
+
+def load_corpus_glob(source: str) -> list[dict]:
+    """Add arbitrary markdown dirs/files as corpus roots (repo=dirname)."""
+    root = Path(source)
+    if not root.exists():
+        print(f"kb/index: skip missing corpus {source}", file=sys.stderr)
+        return []
+    files = [root] if root.is_file() else walk_markdown(root)
+    repo = root.name if root.is_dir() else root.parent.name
+    leafs: list[dict] = []
+    for path in files:
+        try:
+            leafs.extend(to_all(read_markdown(path), path, repo=repo))
+        except OSError as e:
+            print(f"kb/index: skip {path}: {e}", file=sys.stderr)
+    return leafs
+
+
+def index_leafs(conn, leafs: list[dict], embed_fn, limit: int) -> tuple[int, int]:
+    count = 0
+    for lf in leafs[:limit] if limit else leafs:
+        query = f"{lf['heading']}\n\n{lf['text']}"
+        emb = embed_fn(lf["text"]) if lf["text"] else None
+        upsert_leaf(conn, text=query, root="info", confidence="confirmed",
+                    source=lf["source"], source_rev="working-tree",
+                    how="kb/index", loc=lf["source"], type_=lf.get("type", "reference"),
+                    embedding=emb)
+        count += 1
+    return count, len(leafs)
+
+
+def embedder():
+    from model2vec import StaticModel
+    model = StaticModel.from_pretrained("minishlab/potion-multilingual-128M")
+    return lambda text: model.encode([text])[0].astype(float).tolist()
+
+
+def main(argv: list[str]) -> int:
+    import argparse
+    p = argparse.ArgumentParser(description="build the 2dph brain index")
+    p.add_argument("--corpus", action="append", help="extra markdown dir/file to index (may repeat)")
+    p.add_argument("--rebuild", action="store_true", help="fresh db + indexes")
+    p.add_argument("--limit", type=int, default=0, help="max leafs to embed")
+    p.add_argument("--json", action="store_true")
+    a = p.parse_args(argv)
+
+    from kblib import DB_PATH, VAR
+    VAR.mkdir(exist_ok=True)
+    if a.rebuild and DB_PATH.exists():
+        DB_PATH.unlink()
+
+    leafs = load_corpus(ROOT)
+    if a.corpus:
+        for source in a.corpus:
+            leafs.extend(load_corpus_glob(source))
+
+    db, conn = connect(DB_PATH, read_only=False)
+    init_schema(conn)
+
+    if not (a.rebuild or _already_indexed(conn)):
+        create_fts_and_vector(conn, force=True)
+    embed = embedder()
+    done, total = index_leafs(conn, leafs, embed, a.limit)
+    create_fts_and_vector(conn, force=(done > 0 or a.rebuild))
+    s = stats(conn)
+    conn.close()
+    db.close()
+
+    result = {"indexed": done, "corpus_total": total, **{k: v for k, v in s.items() if k in ("total", "by_root")}}
+    print(json.dumps(result, indent=2) if a.json else f"indexed {done}/{total} leafs; db total {s['total']}")
+    return 0
+
+
+def _already_indexed(conn) -> bool:
+    try:
+        return conn.execute("MATCH (l:Leaf) RETURN count(*)").get_all()[0][0] > 0
+    except Exception:
+        return False
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))
@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+"""kb/search - deduction search over the 2dph brain.
+
+  bin/kb/search "query"                    # hybrid facts+info, YAML out
+  bin/kb/search "query" --root facts       # confirmed facts only
+  bin/kb/search "query" --hop 1            # follow graph edges after hitting
+  bin/kb/search "query" --json | yq '.'
+  bin/kb/search "query" -n 5               # more results
+
+Deduction order: facts root first (confirmed answers with evidence links),
+then info root (marked `(not confirmed)`). --root restricts to one root.
+--hop N walks FROM_FILE edges (sibling leafs in the same source file).
+"""
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(ROOT / "tools"))
+
+from kblib import connect, hybrid_search, init_schema, open_readonly, query_fts  # noqa: E402
+from yamlout import to_yaml  # noqa: E402
+import ladybug  # noqa: E402
+
+
+def main(argv: list[str]) -> int:
+    import argparse
+    p = argparse.ArgumentParser(description="deduction search over the brain")
+    p.add_argument("query")
+    p.add_argument("--root", choices=("facts", "info", None), default=None)
+    p.add_argument("--hop", type=int, default=0)
+    p.add_argument("-n", "--limit", type=int, default=10)
+    p.add_argument("--json", action="store_true")
+    a = p.parse_args(argv)
+
+    try:
+        db, conn = open_readonly()
+    except FileNotFoundError as e:
+        print(e, file=sys.stderr)
+        return 1
+
+    from model2vec import StaticModel
+    model = StaticModel.from_pretrained("minishlab/potion-multilingual-128M")
+    emb = model.encode([a.query])[0].astype(float).tolist()
+
+    rhs: list[dict] = []
+    try:
+        rhs = query_fts(conn, a.query, a.limit * 2)
+    except Exception:
+        rhs = []
+
+    results = hybrid_search(conn, emb, rhs, a.limit)
+    if a.root:
+        results = [h for h in results if h["root"] == a.root]
+
+    for hit in results:
+        hit.pop("rrf", None)
+        if hit.get("text"):
+            hit["snippet"] = hit["text"][:280]
+
+    out = {"query": a.query, "root_filter": a.root or "facts+info",
+           "count": len(results), "results": results}
+    print(json.dumps(out, indent=2, ensure_ascii=False) if a.json else to_yaml(out))
+    conn.close()
+    db.close()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))
@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+"""kb/stats - index health for the 2dph brain.
+
+  bin/kb/stats            # leaf counts by root, db size, model
+  bin/kb/stats --json     # machine-readable
+"""
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(ROOT / "tools"))
+
+from kblib import open_readonly, stats  # noqa: E402
+from yamlout import to_yaml  # noqa: E402
+
+
+def main(argv: list[str]) -> int:
+    import argparse
+    p = argparse.ArgumentParser(description="brain index health")
+    p.add_argument("--json", action="store_true")
+    a = p.parse_args(argv)
+
+    try:
+        db, conn = open_readonly()
+    except FileNotFoundError as e:
+        print(e, file=sys.stderr)
+        return 1
+
+    s = stats(conn)
+    conn.close()
+    db.close()
+    print(json.dumps(s, indent=2) if a.json else to_yaml(s))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))