feat(kb): brain tools, Go async serve, root-level docker

- tools/kblib.py: ladybug schema, embeddings, FTS+vector, hybrid RRF - bin/kb/{index,search,get,stats,eval}: corpus indexing + deduction search - bin/facts/{extract,audit}: 2-source evidence acquisition + gates - serve/: async Go HTTP server (goroutines, bounded worker pool), TDD - docker/ flattened to root: compose.yaml + Dockerfile (multi-stage Go) - docker scripts -> bin/ shebang pattern (kb-watch, docker-entrypoint) - bin/ci/semver + tools/semver.py: conventional-commit semver release - ci.yml: go tests + shell checks; drop release-please (PR toggle blocked)
2026-08-10 21:58:25 +01:00
parent dcb00808c9
commit 4fdc0ef4a8
29 changed files with 1560 additions and 134 deletions
@@ -0,0 +1,50 @@
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from semver import bump_type, bump_version  # noqa: E402
+
+
+class BumpTypeTest(unittest.TestCase):
+    def test_empty_is_none(self):
+        self.assertEqual(bump_type([]), "none")
+
+    def test_feat_is_minor(self):
+        self.assertEqual(bump_type(["feat: add search"]), "minor")
+
+    def test_fix_is_patch(self):
+        self.assertEqual(bump_type(["fix: typo"]), "patch")
+
+    def test_chore_and_docs_still_release(self):
+        self.assertEqual(bump_type(["docs: readme"]), "patch")
+        self.assertEqual(bump_type(["ci: green"]), "patch")
+
+    def test_breaking_marker_is_major(self):
+        self.assertEqual(bump_type(["feat!: break api"]), "major")
+        self.assertEqual(bump_type(["fix: x\n\nBREAKING CHANGE: y"]), "major")
+
+    def test_mixed_commits_choose_highest(self):
+        self.assertEqual(bump_type(["fix: a", "feat: b"]), "minor")
+
+
+class BumpVersionTest(unittest.TestCase):
+    def test_patch(self):
+        self.assertEqual(bump_version("v0.1.0", "patch"), "v0.1.1")
+
+    def test_minor(self):
+        self.assertEqual(bump_version("v0.1.0", "minor"), "v0.2.0")
+
+    def test_major(self):
+        self.assertEqual(bump_version("v0.1.0", "major"), "v1.0.0")
+
+    def test_initial_when_no_tag(self):
+        self.assertEqual(bump_version(None, "patch"), "v0.0.1")
+
+    def test_none_returns_none(self):
+        self.assertIsNone(bump_version("v0.1.0", "none"))
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,152 @@
+"""kblib - the 2dph brain core over LadybugDB.
+
+Single embedded graph `var/kb.lbug`. Two roots: facts (assertions backed by
+>=2 independent sources) and info (narrative leafs). Hybrid retrieval: BM25
+(FTS extension) + HNSW cosine (VECTOR extension) + Cypher graph hops.
+
+All access is read-only unless `--rebuild` is passed to kb/index.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import time
+import zlib
+from pathlib import Path
+
+import ladybug
+
+MODEL = "minishlab/potion-multilingual-128M"
+EMBED_DIM = 256
+ROOT_FACTS = "facts"
+ROOT_INFO = "info"
+CONF_CONFIRMED = "confirmed"
+
+VAR = Path(__file__).resolve().parents[1] / "var"
+DB_PATH = VAR / "kb.lbug"
+
+
+def sha256_b64(text: str) -> str:
+    return hashlib.sha256(text.encode()).hexdigest()
+
+
+def connect(path: Path | str | None = None, read_only: bool = True) -> tuple[ladybug.Database, ladybug.Connection]:
+    db = ladybug.Database(str(path or DB_PATH), read_only=read_only)
+    conn = ladybug.Connection(db)
+    conn.execute("LOAD EXTENSION FTS")
+    conn.execute("LOAD EXTENSION VECTOR")
+    return db, conn
+
+
+def init_schema(conn: ladybug.Connection) -> None:
+    conn.execute(
+        "CREATE NODE TABLE IF NOT EXISTS Leaf ("
+        " id STRING, text STRING, root STRING, confidence STRING, "
+        " sha256 STRING, source STRING, source_rev STRING, observed_at STRING, "
+        " how STRING, loc STRING, type STRING, embedding FLOAT[256], "
+        " PRIMARY KEY(id))"
+    )
+    conn.execute(
+        "CREATE NODE TABLE IF NOT EXISTS File ("
+        " id STRING, path STRING, repo STRING, mtime STRING, PRIMARY KEY(id))"
+    )
+    conn.execute(
+        "CREATE REL TABLE IF NOT EXISTS FROM_FILE (FROM Leaf TO File)"
+    )
+    conn.execute(
+        "CREATE NODE TABLE IF NOT EXISTS Host (id STRING, hostname STRING, user STRING, PRIMARY KEY(id))"
+    )
+    conn.execute(
+        "CREATE REL TABLE IF NOT EXISTS RUNS_ON (FROM Leaf TO Host)"
+    )
+
+
+def leaf_id(text: str, source: str) -> str:
+    return sha256_b64(f"{source}\0{text}")[:24]
+
+
+def upsert_leaf(conn: ladybug.Connection, *, text: str, root: str, confidence: str,
+                source: str, source_rev: str, how: str, loc: str, type_: str,
+                embedding: list[float] | None) -> str:
+    lid = leaf_id(text, source)
+    obs = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+    conn.execute(
+        "MERGE (l:Leaf {id:$id}) "
+        "SET l.text=$text, l.root=$root, l.confidence=$confidence, "
+        "    l.sha256=$sha, l.source=$source, l.source_rev=$rev, l.observed_at=$obs, "
+        "    l.how=$how, l.loc=$location, l.type=$type"
+        + (", l.embedding=$emb" if embedding else ""),
+        parameters={
+            "id": lid, "text": text, "root": root, "confidence": confidence,
+            "sha": sha256_b64(text), "source": source, "rev": source_rev,
+            "obs": obs, "how": how, "location": loc, "type": type_,
+            "emb": (embedding if embedding else None),
+        },
+    )
+    return lid
+
+
+def create_fts_and_vector(conn: ladybug.Connection, force: bool = False) -> None:
+    if force:
+        conn.execute("DROP INDEX IF EXISTS Leaf.Leaf_fts")
+        conn.execute("DROP INDEX IF EXISTS Leaf.Leaf_vec")
+    try:
+        conn.execute("CALL CREATE_FTS_INDEX('Leaf', 'id', ['text'])")
+    except Exception:
+        pass
+    try:
+        conn.execute("CALL CREATE_VECTOR_INDEX('Leaf', 'Leaf_vec', 'embedding', metric := 'cosine')")
+    except Exception:
+        pass
+
+
+def query_fts(conn: ladybug.Connection, text: str, limit: int = 10) -> list[dict]:
+    r = conn.execute(
+        "CALL QUERY_FTS_INDEX('Leaf', 'id', $q) "
+        "RETURN node.id, node.text, node.root, score ORDER BY score DESC LIMIT $n",
+        parameters={"q": text, "n": limit},
+    )
+    return [{"id": row[0], "text": row[1], "root": row[2], "score": row[3]} for row in r.get_all()]
+
+
+def query_vector(conn: ladybug.Connection, embedding: list[float], limit: int = 10) -> list[dict]:
+    r = conn.execute(
+        "CALL QUERY_VECTOR_INDEX('Leaf', 'Leaf_vec', $q, $n) "
+        "RETURN node.id, node.text, node.root, distance ORDER BY distance LIMIT $n",
+        parameters={"q": embedding, "n": limit},
+    )
+    out = []
+    for row in r.get_all():
+        # distance -> similarity reasonable for cosine
+        score = 1.0 - row[3] if row[3] is not None else 0.0
+        out.append({"id": row[0], "text": row[1], "root": row[2], "score": score})
+    return out
+
+
+def hybrid_search(conn: ladybug.Connection, embedding: list[float], fts_hits: list[dict],
+                  limit: int = 10) -> list[dict]:
+    """Merge FTS + vector by reciprocal rank fusion."""
+    fused: dict[str, dict] = {}
+    for rank, hit in enumerate(fts_hits):
+        fused.setdefault(hit["id"], {**hit, "rrf": 0.0})["rrf"] = 1.0 / (60 + rank + 1)
+    for rank, hit in enumerate(query_vector(conn, embedding, limit * 3)):
+        entry = fused.setdefault(hit["id"], {**hit, "rrf": 0.0})
+        entry["rrf"] += 1.0 / (60 + rank + 1)
+        entry.setdefault("score", hit.get("score", 0.0))
+    ranked = sorted(fused.values(), key=lambda h: h.get("rrf", 0.0), reverse=True)
+    return ranked[:limit]
+
+
+def stats(conn: ladybug.Connection) -> dict:
+    r = conn.execute("MATCH (l:Leaf) RETURN l.root, count(*)")
+    rows = {row[0]: row[1] for row in r.get_all()}
+    total = conn.execute("MATCH (l:Leaf) RETURN count(*)").get_all()[0][0]
+    return {"total": total, "by_root": rows, "db": str(DB_PATH), "model": MODEL}
+
+
+def open_readonly() -> tuple[ladybug.Database, ladybug.Connection]:
+    if not DB_PATH.exists():
+        raise FileNotFoundError(f"{DB_PATH} missing - run bin/kb/index first")
+    db, conn = connect(read_only=True)
+    init_schema(conn)
+    return db, conn
@@ -0,0 +1,84 @@
+import json
+import re
+from pathlib import Path
+
+import mistune
+
+
+def extract_frontmatter(text: str) -> tuple[dict, str]:
+    """Return (frontmatter dict, body). Accepts leading --- yaml ---."""
+    if not text.startswith("---"):
+        return {}, text
+    end = text.find("\n---", 3)
+    if end == -1:
+        return {}, text
+    fm = text[3:end].strip()
+    body = text[end + 4 :]
+    meta: dict = {}
+    for line in fm.splitlines():
+        if ":" in line:
+            key, _, value = line.partition(":")
+            meta[key.strip()] = value.strip().strip("\"'")
+    return meta, body
+
+
+def split_leafs(meta: dict, body: str) -> list[dict]:
+    """Split a markdown body into leaf chunks on H2 (##) boundaries.
+
+    Each leaf keeps the document-level frontmatter (type, related) and gets
+    its own heading + text. H1 is treated as document title, prepended to the
+    first chunk.
+    """
+    title = ""
+    lines = body.splitlines()
+    headers: list[tuple[str, int]] = []
+    for i, line in enumerate(lines):
+        if re.match(r"^# \S", line):
+            title = line.lstrip("#").strip()
+        elif re.match(r"^## \S", line):
+            headers.append((line.lstrip("##").strip(), i))
+    if not headers:
+        text = "\n".join(l for l in lines if l.strip())
+        return [{"heading": title, "text": text.strip()}]
+
+    leafs: list[dict] = []
+    for idx, (heading, start) in enumerate(headers):
+        end = headers[idx + 1][1] if idx + 1 < len(headers) else len(lines)
+        chunk = "\n".join(l for l in lines[start:end] if l.strip())
+        text = chunk
+        if idx == 0 and title:
+            text = f"{title}\n\n{chunk}"
+        leafs.append({"heading": heading, "text": text.strip()})
+    return leafs
+
+
+def to_all(text: str, path: str | Path, repo: str = "") -> list[dict]:
+    meta, body = extract_frontmatter(text)
+    meta.setdefault("type", "reference")
+    meta.setdefault("status", "current")
+    path = str(path)
+    leafs = split_leafs(meta, body)
+    out = []
+    for lf in leafs:
+        out.append({
+            "source": path,
+            "repo": repo,
+            "heading": lf["heading"],
+            "text": lf["text"],
+            "type": meta.get("type", "reference"),
+            "status": meta.get("status", "current"),
+            "related": meta.get("related", ""),
+        })
+    return out
+
+
+def read_markdown(path: Path) -> str:
+    return path.read_text(encoding="utf-8", errors="replace")
+
+
+def walk_markdown(root: Path) -> list[Path]:
+    return sorted(p for p in root.rglob("*") if p.suffix.lower() in (".md", ".markdown"))
+
+
+def leaves_to_json(leaves: list[dict]) -> str:
+    return json.dumps(leaves, ensure_ascii=False, indent=2)
@@ -0,0 +1,33 @@
+"""semver logic shared by bin/ci/semver and its tests. No git IO here."""
+
+from __future__ import annotations
+
+BREAKING_MARKERS = ("BREAKING CHANGE", "breaking-change")
+BUMP_PATCH_TYPES = ("fix", "perf", "refactor", "build", "ci", "docs", "chore", "test", "style", "revert")
+BUMP_MINOR_TYPE = "feat"
+
+
+def bump_type(subjects: list[str]) -> str:
+    if not subjects:
+        return "none"
+    for subject in subjects:
+        text = subject.lower()
+        if any(m.lower() in text for m in BREAKING_MARKERS):
+            return "major"
+        if "!" in subject.split(":")[0]:
+            return "major"
+    for subject in subjects:
+        if subject.startswith(f"{BUMP_MINOR_TYPE}:"):
+            return "minor"
+    return "patch"
+
+
+def bump_version(current: str | None, bump: str) -> str | None:
+    if bump == "none":
+        return None
+    major, minor, patch = [int(n) for n in (current or "0.0.0").lstrip("v").split(".")]
+    if bump == "major":
+        return f"v{major + 1}.0.0"
+    if bump == "minor":
+        return f"v{major}.{minor + 1}.0"
+    return f"v{major}.{minor}.{patch + 1}"
@@ -0,0 +1,73 @@
+import os
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+import kblib  # noqa: E402
+
+
+def make_emb(value: float) -> list[float]:
+    vec = [0.0] * kblib.EMBED_DIM
+    vec[0] = value
+    return vec
+
+
+class KblibTest(unittest.TestCase):
+    def setUp(self):
+        self.dir = tempfile.mkdtemp()
+        self.dbpath = os.path.join(self.dir, "kb.lbug")
+        self.db, self.conn = kblib.connect(self.dbpath, read_only=False)
+        kblib.init_schema(self.conn)
+
+    def tearDown(self):
+        self.conn.close()
+        self.db.close()
+
+    def test_leaf_id_is_stable(self):
+        self.assertEqual(kblib.leaf_id("abc", "src"), kblib.leaf_id("abc", "src"))
+        self.assertNotEqual(kblib.leaf_id("abc", "src"), kblib.leaf_id("abd", "src"))
+
+    def test_upsert_roundtrip(self):
+        kblib.upsert_leaf(self.conn, text="the quick brown fox", root="info",
+                          confidence="confirmed", source="s", source_rev="r1",
+                          how="test", loc="/tmp", type_="reference",
+                          embedding=make_emb(1.0))
+        kblib.create_fts_and_vector(self.conn, force=True)
+        hits = kblib.query_fts(self.conn, "fox", 5)
+        self.assertEqual(len(hits), 1)
+        self.assertEqual(hits[0]["root"], "info")
+
+    def test_hybrid_ranks_vector_match(self):
+        kblib.upsert_leaf(self.conn, text="the quick brown fox", root="info",
+                          confidence="confirmed", source="s", source_rev="r1",
+                          how="test", loc="/tmp", type_="reference",
+                          embedding=make_emb(1.0))
+        kblib.upsert_leaf(self.conn, text="a lazy dog sleeps", root="info",
+                          confidence="confirmed", source="s", source_rev="r1",
+                          how="test", loc="/tmp", type_="reference",
+                          embedding=make_emb(0.0))
+        kblib.create_fts_and_vector(self.conn, force=True)
+        result = kblib.hybrid_search(self.conn, make_emb(1.0), [], 5)
+        self.assertTrue(result)
+        self.assertIn("rrf", result[0])
+        self.assertEqual(result[0]["text"], "the quick brown fox")
+
+    def test_stats_counts_roots(self):
+        kblib.upsert_leaf(self.conn, text="a fact leaf", root="facts",
+                          confidence="confirmed", source="s", source_rev="r1",
+                          how="test", loc="/tmp", type_="reference",
+                          embedding=make_emb(0.5))
+        kblib.upsert_leaf(self.conn, text="an info leaf", root="info",
+                          confidence="confirmed", source="s", source_rev="r1",
+                          how="test", loc="/tmp", type_="reference",
+                          embedding=make_emb(0.5))
+        stats = kblib.stats(self.conn)
+        self.assertEqual(stats["total"], 2)
+        self.assertEqual(stats["by_root"], {"facts": 1, "info": 1})
+
+
+if __name__ == "__main__":
+    unittest.main()