eSlider
4fdc0ef4a8
- tools/kblib.py: ladybug schema, embeddings, FTS+vector, hybrid RRF
- bin/kb/{index,search,get,stats,eval}: corpus indexing + deduction search
- bin/facts/{extract,audit}: 2-source evidence acquisition + gates
- serve/: async Go HTTP server (goroutines, bounded worker pool), TDD
- docker/ flattened to root: compose.yaml + Dockerfile (multi-stage Go)
- docker scripts -> bin/ shebang pattern (kb-watch, docker-entrypoint)
- bin/ci/semver + tools/semver.py: conventional-commit semver release
- ci.yml: go tests + shell checks; drop release-please (PR toggle blocked)
62 lines
1.8 KiB
Python
Executable File
62 lines
1.8 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""kb/eval - recall@5 gate for the brain.
|
|
|
|
bin/kb/eval [--json]
|
|
|
|
Control questions are answered from the graph; recall@5 >= 0.95 gates CI.
|
|
Each question maps to leaf ids that MUST appear in the top 5 hits.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / "tools"))
|
|
|
|
from kblib import open_readonly, query_fts # noqa: E402
|
|
from yamlout import to_yaml # noqa: E402
|
|
|
|
RECALL_THRESHOLD = 0.95
|
|
|
|
# (query, expected leaf id)
|
|
CONTROL_QUESTIONS: list[tuple[str, str]] = [
|
|
("which database does the brain use", "facts:ladybug"),
|
|
("hybrid search weights fts and vector equally", "info:hybrid"),
|
|
]
|
|
|
|
|
|
def hit_ids_of(query: str, conn, limit: int = 5) -> list[str]:
|
|
try:
|
|
hits = query_fts(conn, query, limit)
|
|
return [h["id"] for h in hits]
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
def main(argv: list[str]) -> int:
|
|
import argparse
|
|
p = argparse.ArgumentParser(description="recall@5 gate")
|
|
p.add_argument("--json", action="store_true")
|
|
a = p.parse_args(argv)
|
|
|
|
db, conn = open_readonly()
|
|
recalled = 0
|
|
detail = []
|
|
for query, expected in CONTROL_QUESTIONS:
|
|
hits = hit_ids_of(query, conn)
|
|
ok = any(expected in h or h in expected for h in hits)
|
|
recalled += int(ok)
|
|
detail.append({"q": query, "expected": expected, "in_top5": ok, "hits": hits[:5]})
|
|
recall = recalled / len(CONTROL_QUESTIONS) if CONTROL_QUESTIONS else 1.0
|
|
passed = recall >= RECALL_THRESHOLD
|
|
out = {"recall@5": round(recall, 3), "passed": passed, "gate": len(CONTROL_QUESTIONS), "details": detail}
|
|
print(json.dumps(out, indent=2) if a.json else to_yaml(out))
|
|
conn.close()
|
|
db.close()
|
|
return 0 if passed else 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv[1:])) |