eSlider/2dph
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(kb): stop DROP INDEX killing HNSW via Ladybug ghost catalog

Browse Source 
Ladybug 0.19 DROP INDEX leaves `_0_Leaf_vec_UPPER` / `0_id_docs` in catalog so
CREATE fails while SHOW_INDEXES omits the index; create_fts_and_vector used to
swallow that. Never drop FTS/VECTOR; ensure_indexes after upserts; rebuild =
delete kb.lbug. Add compose.edelweiss.yml + regression tests.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
eSlider
2026-08-12 16:05:14 +01:00
co-authored by Cursor
parent 80e3b7a1cf
commit a5a1f91d95
11 changed files with 356 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/tools/kblib.py
+59 -18
View File
@@ -118,30 +118,71 @@ def upsert_leaf(conn: ladybug.Connection, *, text: str, root: str, confidence: s
return lid
def leaf_index_names(conn: ladybug.Connection) -> set[str]:
"""Return index names on the Leaf table (e.g. {'id', 'Leaf_vec', '_PK'})."""
rows = conn.execute("CALL SHOW_INDEXES() RETURN *").get_all()
return {row[1] for row in rows if row[0] == "Leaf"}
def create_fts_and_vector(conn: ladybug.Connection, force: bool = False) -> None:
if force:
conn.execute("DROP INDEX IF EXISTS Leaf.Leaf_fts")
conn.execute("DROP INDEX IF EXISTS Leaf.Leaf_vec")
try:
conn.execute("CALL CREATE_FTS_INDEX('Leaf', 'id', ['text'])")
except Exception:
pass
try:
conn.execute("CALL CREATE_VECTOR_INDEX('Leaf', 'Leaf_vec', 'embedding', metric := 'cosine')")
except Exception:
pass
"""Create FTS (BM25) + HNSW vector indexes if missing.
Never DROP INDEX for FTS/VECTOR. Ladybug 0.19 leaves ghost catalog
entries after DROP (`_0_Leaf_vec_UPPER`, `0_id_docs`), so a later
CREATE fails with "already exists in catalog" while SHOW_INDEXES
still omits the index. Swallowing that error made HNSW look "OK"
until the first QUERY_VECTOR_INDEX.
`force=True` is accepted for API compatibility but does **not** drop.
Fresh indexes require deleting `var/kb.lbug` and rebuilding
(`bin/kb/index --rebuild`).
"""
del force # API compat; DROP is unsafe — see docstring
names = leaf_index_names(conn)
if "id" not in names:
try:
conn.execute("CALL CREATE_FTS_INDEX('Leaf', 'id', ['text'])")
except Exception as e:
raise RuntimeError(
"CREATE_FTS_INDEX failed (often ghost catalog after DROP INDEX). "
"Delete var/kb.lbug and run bin/kb/index --rebuild. "
f"Cause: {e}"
) from e
if "Leaf_vec" not in names:
try:
conn.execute(
"CALL CREATE_VECTOR_INDEX('Leaf', 'Leaf_vec', 'embedding', "
"metric := 'cosine')"
)
except Exception as e:
raise RuntimeError(
"CREATE_VECTOR_INDEX failed (often ghost catalog after DROP INDEX "
"Leaf.Leaf_vec → `_0_Leaf_vec_UPPER already exists in catalog`). "
"Delete var/kb.lbug and run bin/kb/index --rebuild. "
f"Cause: {e}"
) from e
names = leaf_index_names(conn)
missing = {"id", "Leaf_vec"} - names
if missing:
raise RuntimeError(
f"Leaf indexes incomplete after create: missing {sorted(missing)}; "
f"have {sorted(names)}. Delete var/kb.lbug and --rebuild."
)
def ensure_indexes(conn: ladybug.Connection) -> None:
"""Idempotent: create FTS + HNSW only when missing. Safe after upserts."""
create_fts_and_vector(conn, force=False)
def drop_indexes(conn: ladybug.Connection) -> None:
"""Drop FTS + vector indexes so bulk MERGEs don't corrupt them.
"""No-op. Kept for callers; DROP INDEX is fatal on Ladybug 0.19.
Ladybug's FTS index goes inconsistent when rows are inserted while the
index exists ("document for node offset N is missing during delete").
Importers that add many leafs must drop indexes first, write, then
recreate via create_fts_and_vector().
Historical note claimed "drop before bulk MERGE". Measured: upsert while
indexes exist keeps HNSW queryable; DROP leaves ghost catalog tables that
block recreate. Bulk rebuilders must delete `var/kb.lbug` instead.
"""
conn.execute("DROP INDEX IF EXISTS Leaf.Leaf_fts")
conn.execute("DROP INDEX IF EXISTS Leaf.Leaf_vec")
return
def query_fts(conn: ladybug.Connection, text: str, limit: int = 10) -> list[dict]:
bin/tools/test_kblib.py
+33 -3
View File
@@ -35,7 +35,7 @@ class KblibTest(unittest.TestCase):
confidence="confirmed", source="s", source_rev="r1",
how="test", loc="/tmp", type_="reference",
embedding=make_emb(1.0))
kblib.create_fts_and_vector(self.conn, force=True)
kblib.ensure_indexes(self.conn)
hits = kblib.query_fts(self.conn, "fox", 5)
self.assertEqual(len(hits), 1)
self.assertEqual(hits[0]["root"], "info")
@@ -49,12 +49,42 @@ class KblibTest(unittest.TestCase):
confidence="confirmed", source="s", source_rev="r1",
how="test", loc="/tmp", type_="reference",
embedding=make_emb(0.0))
kblib.create_fts_and_vector(self.conn, force=True)
kblib.ensure_indexes(self.conn)
result = kblib.hybrid_search(self.conn, make_emb(1.0), [], 5)
self.assertTrue(result)
self.assertIn("rrf", result[0])
self.assertEqual(result[0]["text"], "the quick brown fox")
def test_upsert_keeps_hnsw_queryable(self):
"""Upsert while HNSW exists must not kill vector search."""
kblib.upsert_leaf(self.conn, text="seed leaf", root="info",
confidence="confirmed", source="s", source_rev="r1",
how="test", loc="/tmp", type_="reference",
embedding=make_emb(0.2))
kblib.ensure_indexes(self.conn)
self.assertIn("Leaf_vec", kblib.leaf_index_names(self.conn))
kblib.upsert_leaf(self.conn, text="added after index", root="facts",
confidence="confirmed", source="a.md x b.md",
source_rev="r1", how="test", loc="/tmp", type_="fact",
embedding=make_emb(0.9))
hits = kblib.query_vector(self.conn, make_emb(0.9), 5)
self.assertTrue(hits)
self.assertIn("Leaf_vec", kblib.leaf_index_names(self.conn))
def test_drop_vector_then_create_raises_clear_error(self):
"""DROP INDEX leaves ghost catalog; create_fts_and_vector must raise."""
kblib.upsert_leaf(self.conn, text="seed", root="info",
confidence="confirmed", source="s", source_rev="r1",
how="test", loc="/tmp", type_="reference",
embedding=make_emb(0.1))
kblib.ensure_indexes(self.conn)
self.conn.execute("DROP INDEX IF EXISTS Leaf.Leaf_vec")
with self.assertRaises(RuntimeError) as ctx:
kblib.create_fts_and_vector(self.conn, force=True)
msg = str(ctx.exception)
self.assertIn("CREATE_VECTOR_INDEX failed", msg)
self.assertIn("--rebuild", msg)
def test_stats_counts_roots(self):
kblib.upsert_leaf(self.conn, text="a fact leaf", root="facts",
confidence="confirmed", source="s", source_rev="r1",
@@ -70,4 +100,4 @@ class KblibTest(unittest.TestCase):
if __name__ == "__main__":
unittest.main()
unittest.main()