eSlider/2dph
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(kb): stop DROP INDEX killing HNSW via Ladybug ghost catalog

Browse Source 
Ladybug 0.19 DROP INDEX leaves `_0_Leaf_vec_UPPER` / `0_id_docs` in catalog so
CREATE fails while SHOW_INDEXES omits the index; create_fts_and_vector used to
swallow that. Never drop FTS/VECTOR; ensure_indexes after upserts; rebuild =
delete kb.lbug. Add compose.edelweiss.yml + regression tests.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
eSlider
2026-08-12 16:05:14 +01:00
co-authored by Cursor
parent 80e3b7a1cf
commit a5a1f91d95
11 changed files with 356 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/docker-entrypoint
Regular → Executable
+5 -1
View File
@@ -6,6 +6,8 @@
# brain index bin/kb/index
# brain watch <dir> watchdog re-indexer (bin/kb/watch)
# brain serve async Go HTTP server (bin/serve)
# brain extract bin/facts/extract (docker×compose pairing)
# brain audit bin/facts/audit
#
# Usage comment starts at line 2 (self-describing convention).
set -euo pipefail
@@ -19,5 +21,7 @@ case "$CMD" in
index) exec "$KB_PY" /app/bin/kb/index "$@" ;;
watch) exec /app/bin/watch "$@" ;;
serve) exec /app/bin/serve "$@" ;;
extract) exec "$KB_PY" /app/bin/facts/extract "$@" ;;
audit) exec "$KB_PY" /app/bin/facts/audit "$@" ;;
*) echo "unknown command: $CMD" >&2; exit 2 ;;
esac
esac
bin/facts/extract
+3 -2
View File
@@ -176,8 +176,7 @@ def dedupe(facts: list[dict]) -> list[dict]:
def write_facts(facts: list[dict]) -> None:
from kblib import connect, init_schema, upsert_leaf
from kblib import VAR
from kblib import VAR, connect, ensure_indexes, init_schema, upsert_leaf
VAR.mkdir(exist_ok=True)
db, conn = connect(VAR / "kb.lbug", read_only=False)
init_schema(conn)
@@ -188,6 +187,8 @@ def write_facts(facts: list[dict]) -> None:
upsert_leaf(conn, text=f["text"], root="facts", confidence="confirmed",
source=f["source"], source_rev=REPO, how=f["how"],
loc=f["loc"], type_="fact", embedding=emb)
# Upsert-with-index is safe; never DROP+recreate (ghost catalog kills HNSW).
ensure_indexes(conn)
conn.close()
db.close()
bin/git/import
+4 -3
View File
@@ -24,7 +24,7 @@ ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "bin" / "tools"))
from kblib import ( # noqa: E402
connect, create_fts_and_vector, drop_indexes, init_schema, upsert_leaf,
connect, ensure_indexes, init_schema, upsert_leaf,
)
from gitimport import commits_to_leafs, ensure_git_schema, index_commits, parse_log # noqa: E402
@@ -125,9 +125,10 @@ def main(argv: list[str]) -> int:
print(f"{r['repo']:<24} {r['commits']:>5} commits -> {r['leafs']} leafs {r['path']}")
return 0
# Never DROP FTS/VECTOR (ghost catalog). Upsert while indexes exist is OK;
# ensure_indexes only CREATEs when missing.
db, conn = connect(ROOT / "var" / "kb.lbug", read_only=False)
init_schema(conn)
drop_indexes(conn)
embed = embedder()
rows: list[dict] = []
for repo in repos:
@@ -136,7 +137,7 @@ def main(argv: list[str]) -> int:
reached, written = import_repo(conn, repo, embed, a.limit, a.since)
total_commits += reached
rows.append({"repo": repo_name(repo), "commits": reached, "written": written})
create_fts_and_vector(conn, force=True)
ensure_indexes(conn)
conn.close()
db.close()
bin/kb/index
+4 -15
View File
@@ -22,7 +22,7 @@ ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "bin" / "tools"))
from kblib import ( # noqa: E402
connect, create_fts_and_vector, init_schema, upsert_leaf,
connect, ensure_indexes, init_schema, upsert_leaf,
open_readonly, stats,
)
from mdleaves import read_markdown, to_all, walk_markdown # noqa: E402
@@ -116,13 +116,11 @@ def main(argv: list[str]) -> int:
db, conn = connect(DB_PATH, read_only=False)
init_schema(conn)
# Keep the FTS/VECTOR indexes in place across incremental runs: ladybug's
# DROP INDEX leaves the backing tables registered on migrated DBs, so a
# drop+recreate silently kills the vector index. Only create when missing.
# Never DROP FTS/VECTOR (ghost catalog). Write leafs, then ensure indexes.
# --rebuild already deleted kb.lbug above, so CREATE runs on a clean DB.
embed = embedder()
done, total = index_leafs(conn, leafs, embed, a.limit)
if a.rebuild or not _has_indexes(conn):
create_fts_and_vector(conn, force=True)
ensure_indexes(conn)
s = stats(conn)
conn.close()
db.close()
@@ -132,14 +130,5 @@ def main(argv: list[str]) -> int:
return 0
def _has_indexes(conn) -> bool:
try:
rows = conn.execute("CALL SHOW_INDEXES() RETURN *").get_all()
names = {row[1] for row in rows if row[0] == "Leaf"}
return "id" in names and "Leaf_vec" in names
except Exception:
return False
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
bin/mail/index_mail
+2 -2
View File
@@ -22,7 +22,7 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "bin" / "tools"))
from kblib import DB_PATH, VAR, connect, create_fts_and_vector, init_schema, stats, upsert_leaf # noqa: E402
from kblib import DB_PATH, VAR, connect, ensure_indexes, init_schema, stats, upsert_leaf # noqa: E402
from mdleaves import read_markdown, to_all, walk_markdown # noqa: E402
@@ -81,7 +81,7 @@ def main(argv: list[str]) -> int:
init_schema(conn)
embed = _embedder()
done, total = _index_leafs(conn, leafs, embed)
create_fts_and_vector(conn, force=True)
ensure_indexes(conn)
s = stats(conn)
conn.close()
db.close()
bin/tools/kblib.py
+59 -18
View File
@@ -118,30 +118,71 @@ def upsert_leaf(conn: ladybug.Connection, *, text: str, root: str, confidence: s
return lid
def leaf_index_names(conn: ladybug.Connection) -> set[str]:
"""Return index names on the Leaf table (e.g. {'id', 'Leaf_vec', '_PK'})."""
rows = conn.execute("CALL SHOW_INDEXES() RETURN *").get_all()
return {row[1] for row in rows if row[0] == "Leaf"}
def create_fts_and_vector(conn: ladybug.Connection, force: bool = False) -> None:
if force:
conn.execute("DROP INDEX IF EXISTS Leaf.Leaf_fts")
conn.execute("DROP INDEX IF EXISTS Leaf.Leaf_vec")
try:
conn.execute("CALL CREATE_FTS_INDEX('Leaf', 'id', ['text'])")
except Exception:
pass
try:
conn.execute("CALL CREATE_VECTOR_INDEX('Leaf', 'Leaf_vec', 'embedding', metric := 'cosine')")
except Exception:
pass
"""Create FTS (BM25) + HNSW vector indexes if missing.
Never DROP INDEX for FTS/VECTOR. Ladybug 0.19 leaves ghost catalog
entries after DROP (`_0_Leaf_vec_UPPER`, `0_id_docs`), so a later
CREATE fails with "already exists in catalog" while SHOW_INDEXES
still omits the index. Swallowing that error made HNSW look "OK"
until the first QUERY_VECTOR_INDEX.
`force=True` is accepted for API compatibility but does **not** drop.
Fresh indexes require deleting `var/kb.lbug` and rebuilding
(`bin/kb/index --rebuild`).
"""
del force # API compat; DROP is unsafe — see docstring
names = leaf_index_names(conn)
if "id" not in names:
try:
conn.execute("CALL CREATE_FTS_INDEX('Leaf', 'id', ['text'])")
except Exception as e:
raise RuntimeError(
"CREATE_FTS_INDEX failed (often ghost catalog after DROP INDEX). "
"Delete var/kb.lbug and run bin/kb/index --rebuild. "
f"Cause: {e}"
) from e
if "Leaf_vec" not in names:
try:
conn.execute(
"CALL CREATE_VECTOR_INDEX('Leaf', 'Leaf_vec', 'embedding', "
"metric := 'cosine')"
)
except Exception as e:
raise RuntimeError(
"CREATE_VECTOR_INDEX failed (often ghost catalog after DROP INDEX "
"Leaf.Leaf_vec → `_0_Leaf_vec_UPPER already exists in catalog`). "
"Delete var/kb.lbug and run bin/kb/index --rebuild. "
f"Cause: {e}"
) from e
names = leaf_index_names(conn)
missing = {"id", "Leaf_vec"} - names
if missing:
raise RuntimeError(
f"Leaf indexes incomplete after create: missing {sorted(missing)}; "
f"have {sorted(names)}. Delete var/kb.lbug and --rebuild."
)
def ensure_indexes(conn: ladybug.Connection) -> None:
"""Idempotent: create FTS + HNSW only when missing. Safe after upserts."""
create_fts_and_vector(conn, force=False)
def drop_indexes(conn: ladybug.Connection) -> None:
"""Drop FTS + vector indexes so bulk MERGEs don't corrupt them.
"""No-op. Kept for callers; DROP INDEX is fatal on Ladybug 0.19.
Ladybug's FTS index goes inconsistent when rows are inserted while the
index exists ("document for node offset N is missing during delete").
Importers that add many leafs must drop indexes first, write, then
recreate via create_fts_and_vector().
Historical note claimed "drop before bulk MERGE". Measured: upsert while
indexes exist keeps HNSW queryable; DROP leaves ghost catalog tables that
block recreate. Bulk rebuilders must delete `var/kb.lbug` instead.
"""
conn.execute("DROP INDEX IF EXISTS Leaf.Leaf_fts")
conn.execute("DROP INDEX IF EXISTS Leaf.Leaf_vec")
return
def query_fts(conn: ladybug.Connection, text: str, limit: int = 10) -> list[dict]:
bin/tools/test_kblib.py
+33 -3
View File
@@ -35,7 +35,7 @@ class KblibTest(unittest.TestCase):
confidence="confirmed", source="s", source_rev="r1",
how="test", loc="/tmp", type_="reference",
embedding=make_emb(1.0))
kblib.create_fts_and_vector(self.conn, force=True)
kblib.ensure_indexes(self.conn)
hits = kblib.query_fts(self.conn, "fox", 5)
self.assertEqual(len(hits), 1)
self.assertEqual(hits[0]["root"], "info")
@@ -49,12 +49,42 @@ class KblibTest(unittest.TestCase):
confidence="confirmed", source="s", source_rev="r1",
how="test", loc="/tmp", type_="reference",
embedding=make_emb(0.0))
kblib.create_fts_and_vector(self.conn, force=True)
kblib.ensure_indexes(self.conn)
result = kblib.hybrid_search(self.conn, make_emb(1.0), [], 5)
self.assertTrue(result)
self.assertIn("rrf", result[0])
self.assertEqual(result[0]["text"], "the quick brown fox")
def test_upsert_keeps_hnsw_queryable(self):
"""Upsert while HNSW exists must not kill vector search."""
kblib.upsert_leaf(self.conn, text="seed leaf", root="info",
confidence="confirmed", source="s", source_rev="r1",
how="test", loc="/tmp", type_="reference",
embedding=make_emb(0.2))
kblib.ensure_indexes(self.conn)
self.assertIn("Leaf_vec", kblib.leaf_index_names(self.conn))
kblib.upsert_leaf(self.conn, text="added after index", root="facts",
confidence="confirmed", source="a.md x b.md",
source_rev="r1", how="test", loc="/tmp", type_="fact",
embedding=make_emb(0.9))
hits = kblib.query_vector(self.conn, make_emb(0.9), 5)
self.assertTrue(hits)
self.assertIn("Leaf_vec", kblib.leaf_index_names(self.conn))
def test_drop_vector_then_create_raises_clear_error(self):
"""DROP INDEX leaves ghost catalog; create_fts_and_vector must raise."""
kblib.upsert_leaf(self.conn, text="seed", root="info",
confidence="confirmed", source="s", source_rev="r1",
how="test", loc="/tmp", type_="reference",
embedding=make_emb(0.1))
kblib.ensure_indexes(self.conn)
self.conn.execute("DROP INDEX IF EXISTS Leaf.Leaf_vec")
with self.assertRaises(RuntimeError) as ctx:
kblib.create_fts_and_vector(self.conn, force=True)
msg = str(ctx.exception)
self.assertIn("CREATE_VECTOR_INDEX failed", msg)
self.assertIn("--rebuild", msg)
def test_stats_counts_roots(self):
kblib.upsert_leaf(self.conn, text="a fact leaf", root="facts",
confidence="confirmed", source="s", source_rev="r1",
@@ -70,4 +100,4 @@ class KblibTest(unittest.TestCase):
if __name__ == "__main__":
unittest.main()
unittest.main()