refactor(tools): bin/{subject}/{method} layout; Go serve+watch modules
Move serve/ (module) -> bin/server, tools/ -> bin/tools, replace bin/kb-watch bash with bin/watch Go package; self-executing Go shebangs bin/serve.go and bin/kb/watch.go; Docker + CI + git/import + docs repointed. Multi-stage image builds static serve+watch binaries (no Go runtime in container).
This commit is contained in:
@@ -0,0 +1,195 @@
|
||||
"""kblib - the 2dph brain core over LadybugDB.
|
||||
|
||||
Single embedded graph `var/kb.lbug`. Two roots: facts (assertions backed by
|
||||
>=2 independent sources) and info (narrative leafs). Hybrid retrieval: BM25
|
||||
(FTS extension) + HNSW cosine (VECTOR extension) + Cypher graph hops.
|
||||
|
||||
All access is read-only unless `--rebuild` is passed to kb/index.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
import ladybug
|
||||
|
||||
MODEL = "minishlab/potion-multilingual-128M"
|
||||
EMBED_DIM = 256
|
||||
ROOT_FACTS = "facts"
|
||||
ROOT_INFO = "info"
|
||||
CONF_CONFIRMED = "confirmed"
|
||||
|
||||
def _repo_root() -> Path:
|
||||
p = Path(__file__).resolve().parent
|
||||
while True:
|
||||
if (p / "var").is_dir() or (p / ".git").is_dir() or (p / "pyproject.toml").is_file():
|
||||
return p
|
||||
if p.parent == p:
|
||||
return Path(__file__).resolve().parents[2]
|
||||
p = p.parent
|
||||
|
||||
|
||||
VAR = _repo_root() / "var"
|
||||
DB_PATH = VAR / "kb.lbug"
|
||||
|
||||
|
||||
def sha256_b64(text: str) -> str:
|
||||
return hashlib.sha256(text.encode()).hexdigest()
|
||||
|
||||
|
||||
def _load_extension(conn: ladybug.Connection, name: str) -> None:
|
||||
"""Install (download once) and load a ladybug extension."""
|
||||
try:
|
||||
conn.execute(f"INSTALL {name}")
|
||||
except Exception:
|
||||
pass # already installed / offline-ok when present
|
||||
conn.execute(f"LOAD EXTENSION {name}")
|
||||
|
||||
|
||||
def connect(path: Path | str | None = None, read_only: bool = True) -> tuple[ladybug.Database, ladybug.Connection]:
|
||||
db = ladybug.Database(str(path or DB_PATH), read_only=read_only)
|
||||
conn = ladybug.Connection(db)
|
||||
_load_extension(conn, "FTS")
|
||||
_load_extension(conn, "VECTOR")
|
||||
return db, conn
|
||||
|
||||
|
||||
def init_schema(conn: ladybug.Connection) -> None:
|
||||
conn.execute(
|
||||
"CREATE NODE TABLE IF NOT EXISTS Leaf ("
|
||||
" id STRING, text STRING, root STRING, confidence STRING, "
|
||||
" sha256 STRING, source STRING, source_rev STRING, observed_at STRING, "
|
||||
" how STRING, loc STRING, type STRING, embedding FLOAT[256], "
|
||||
" PRIMARY KEY(id))"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE NODE TABLE IF NOT EXISTS File ("
|
||||
" id STRING, path STRING, repo STRING, mtime STRING, PRIMARY KEY(id))"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE REL TABLE IF NOT EXISTS FROM_FILE (FROM Leaf TO File)"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE NODE TABLE IF NOT EXISTS Host (id STRING, hostname STRING, user STRING, PRIMARY KEY(id))"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE REL TABLE IF NOT EXISTS RUNS_ON (FROM Leaf TO Host)"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE NODE TABLE IF NOT EXISTS Commit (id STRING, repo STRING, subject STRING, "
|
||||
"author STRING, email STRING, date STRING, PRIMARY KEY(id))"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE NODE TABLE IF NOT EXISTS Person (id STRING, name STRING, email STRING, PRIMARY KEY(id))"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE REL TABLE IF NOT EXISTS HAS_VERSION (FROM File TO Commit)"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE REL TABLE IF NOT EXISTS AUTHORED (FROM Commit TO Person)"
|
||||
)
|
||||
|
||||
|
||||
def leaf_id(text: str, source: str) -> str:
|
||||
return sha256_b64(f"{source}\0{text}")[:24]
|
||||
|
||||
|
||||
def upsert_leaf(conn: ladybug.Connection, *, text: str, root: str, confidence: str,
|
||||
source: str, source_rev: str, how: str, loc: str, type_: str,
|
||||
embedding: list[float] | None) -> str:
|
||||
lid = leaf_id(text, source)
|
||||
obs = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
conn.execute(
|
||||
"MERGE (l:Leaf {id:$id}) "
|
||||
"SET l.text=$text, l.root=$root, l.confidence=$confidence, "
|
||||
" l.sha256=$sha, l.source=$source, l.source_rev=$rev, l.observed_at=$obs, "
|
||||
" l.how=$how, l.loc=$location, l.type=$type"
|
||||
+ (", l.embedding=$emb" if embedding else ""),
|
||||
parameters={
|
||||
"id": lid, "text": text, "root": root, "confidence": confidence,
|
||||
"sha": sha256_b64(text), "source": source, "rev": source_rev,
|
||||
"obs": obs, "how": how, "location": loc, "type": type_,
|
||||
"emb": (embedding if embedding else None),
|
||||
},
|
||||
)
|
||||
return lid
|
||||
|
||||
|
||||
def create_fts_and_vector(conn: ladybug.Connection, force: bool = False) -> None:
|
||||
if force:
|
||||
conn.execute("DROP INDEX IF EXISTS Leaf.Leaf_fts")
|
||||
conn.execute("DROP INDEX IF EXISTS Leaf.Leaf_vec")
|
||||
try:
|
||||
conn.execute("CALL CREATE_FTS_INDEX('Leaf', 'id', ['text'])")
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
conn.execute("CALL CREATE_VECTOR_INDEX('Leaf', 'Leaf_vec', 'embedding', metric := 'cosine')")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def drop_indexes(conn: ladybug.Connection) -> None:
|
||||
"""Drop FTS + vector indexes so bulk MERGEs don't corrupt them.
|
||||
|
||||
Ladybug's FTS index goes inconsistent when rows are inserted while the
|
||||
index exists ("document for node offset N is missing during delete").
|
||||
Importers that add many leafs must drop indexes first, write, then
|
||||
recreate via create_fts_and_vector().
|
||||
"""
|
||||
conn.execute("DROP INDEX IF EXISTS Leaf.Leaf_fts")
|
||||
conn.execute("DROP INDEX IF EXISTS Leaf.Leaf_vec")
|
||||
|
||||
|
||||
def query_fts(conn: ladybug.Connection, text: str, limit: int = 10) -> list[dict]:
|
||||
r = conn.execute(
|
||||
"CALL QUERY_FTS_INDEX('Leaf', 'id', $q) "
|
||||
"RETURN node.id, node.text, node.root, score ORDER BY score DESC LIMIT $n",
|
||||
parameters={"q": text, "n": limit},
|
||||
)
|
||||
return [{"id": row[0], "text": row[1], "root": row[2], "score": row[3]} for row in r.get_all()]
|
||||
|
||||
|
||||
def query_vector(conn: ladybug.Connection, embedding: list[float], limit: int = 10) -> list[dict]:
|
||||
r = conn.execute(
|
||||
"CALL QUERY_VECTOR_INDEX('Leaf', 'Leaf_vec', $q, $n) "
|
||||
"RETURN node.id, node.text, node.root, distance ORDER BY distance LIMIT $n",
|
||||
parameters={"q": embedding, "n": limit},
|
||||
)
|
||||
out = []
|
||||
for row in r.get_all():
|
||||
# distance -> similarity reasonable for cosine
|
||||
score = 1.0 - row[3] if row[3] is not None else 0.0
|
||||
out.append({"id": row[0], "text": row[1], "root": row[2], "score": score})
|
||||
return out
|
||||
|
||||
|
||||
def hybrid_search(conn: ladybug.Connection, embedding: list[float], fts_hits: list[dict],
|
||||
limit: int = 10) -> list[dict]:
|
||||
"""Merge FTS + vector by reciprocal rank fusion."""
|
||||
fused: dict[str, dict] = {}
|
||||
for rank, hit in enumerate(fts_hits):
|
||||
fused.setdefault(hit["id"], {**hit, "rrf": 0.0})["rrf"] = 1.0 / (60 + rank + 1)
|
||||
for rank, hit in enumerate(query_vector(conn, embedding, limit * 3)):
|
||||
entry = fused.setdefault(hit["id"], {**hit, "rrf": 0.0})
|
||||
entry["rrf"] += 1.0 / (60 + rank + 1)
|
||||
entry.setdefault("score", hit.get("score", 0.0))
|
||||
ranked = sorted(fused.values(), key=lambda h: h.get("rrf", 0.0), reverse=True)
|
||||
return ranked[:limit]
|
||||
|
||||
|
||||
def stats(conn: ladybug.Connection) -> dict:
|
||||
r = conn.execute("MATCH (l:Leaf) RETURN l.root, count(*)")
|
||||
rows = {row[0]: row[1] for row in r.get_all()}
|
||||
total = conn.execute("MATCH (l:Leaf) RETURN count(*)").get_all()[0][0]
|
||||
return {"total": total, "by_root": rows, "db": str(DB_PATH), "model": MODEL}
|
||||
|
||||
|
||||
def open_readonly() -> tuple[ladybug.Database, ladybug.Connection]:
|
||||
if not DB_PATH.exists():
|
||||
raise FileNotFoundError(f"{DB_PATH} missing - run bin/kb/index first")
|
||||
db, conn = connect(read_only=True)
|
||||
return db, conn
|
||||
Reference in New Issue
Block a user