From bd1a91dab72a0a02c397c076aa56383edc13d4e4 Mon Sep 17 00:00:00 2001 From: Andriy Oblivantsev Date: Wed, 12 Aug 2026 16:09:02 +0100 Subject: [PATCH] fix(kb): seed facts before CREATE indexes (FTS MERGE corruption) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upsert under live FTS raises "document for node offset N is missing". Add --skip-indexes; edelweiss-pilot index = write → seed → ensure_indexes. Ship seed-edelweiss-facts.py (paired lexicon/OO/interview/QEMU facts). Co-authored-by: Cursor --- bin/edelweiss-pilot | 37 +++++++ bin/kb/index | 13 ++- bin/seed-edelweiss-facts.py | 187 ++++++++++++++++++++++++++++++++++++ bin/tools/kblib.py | 12 ++- 4 files changed, 244 insertions(+), 5 deletions(-) create mode 100755 bin/edelweiss-pilot create mode 100755 bin/seed-edelweiss-facts.py diff --git a/bin/edelweiss-pilot b/bin/edelweiss-pilot new file mode 100755 index 0000000..63d6840 --- /dev/null +++ b/bin/edelweiss-pilot @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# bin/edelweiss-pilot — Edelweiss-only 2dph brain (facts/info) +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +export HF_HOME="${HF_HOME:-$ROOT/var/hf}" +export PATH="$ROOT/.venv/bin:$PATH" +mkdir -p "$HF_HOME" "$ROOT/var" +# Corpus policy (Andriy 2026-08-12): +# 1 cs-lexicon, 2 vendor kbs, 3 reports only (not raw STT), +# 4/5/6 seeded via pairing (seed-edelweiss-facts.py), +# 7 SCHEMA/ARCHITECTURE churn — do not promote to facts, +# 9 glossary+subjects via docs/docs +DOCS=( + --corpus /home/devops/projects/docs/docs + --corpus /home/devops/projects/edelweiss-curasoft/docs + --corpus /home/devops/projects/edelweiss-ui/docs + --corpus /home/devops/projects/edelweiss-curasoft/docs/lexicon + --corpus /home/devops/projects/curasoft/docs/lexicon + --corpus /home/devops/projects/curasoft/docs/curasoft-de/kbs + --corpus /home/devops/projects/docs/docs/reports +) +cmd="${1:-stats}"; shift || true +case "$cmd" in + # Write leafs without indexes → seed facts → ensure_indexes (FTS+HNSW). + # Never upsert under live FTS; never DROP INDEX (ghost catalog). + index) + bin/kb/index --rebuild --skip-indexes "${DOCS[@]}" "$@" + .venv/bin/python bin/seed-edelweiss-facts.py + ;; + search) exec bin/edelweiss-search "$@" ;; + search-go) exec bin/kb/search "$@" ;; + stats) exec bin/kb/stats "$@" ;; + audit) exec bin/facts/audit "${1:-db}" "${@:2}" ;; + seed) exec .venv/bin/python bin/seed-edelweiss-facts.py "$@" ;; + *) echo "usage: $0 index|search|search-go|stats|audit|seed ..." >&2; exit 2 ;; +esac diff --git a/bin/kb/index b/bin/kb/index index 8f555d4..c818134 100755 --- a/bin/kb/index +++ b/bin/kb/index @@ -99,6 +99,11 @@ def main(argv: list[str]) -> int: p = argparse.ArgumentParser(description="build the 2dph brain index") p.add_argument("--corpus", action="append", help="extra markdown dir/file to index (may repeat)") p.add_argument("--rebuild", action="store_true", help="fresh db + indexes") + p.add_argument( + "--skip-indexes", + action="store_true", + help="write leafs only; caller runs ensure_indexes after seeding facts", + ) p.add_argument("--limit", type=int, default=0, help="max leafs to embed") p.add_argument("--json", action="store_true") a = p.parse_args(argv) @@ -116,16 +121,20 @@ def main(argv: list[str]) -> int: db, conn = connect(DB_PATH, read_only=False) init_schema(conn) - # Never DROP FTS/VECTOR (ghost catalog). Write leafs, then ensure indexes. + # Never DROP FTS/VECTOR (ghost catalog). Write leafs, then ensure indexes + # unless --skip-indexes (seed facts first — MERGE under live FTS corrupts it). # --rebuild already deleted kb.lbug above, so CREATE runs on a clean DB. embed = embedder() done, total = index_leafs(conn, leafs, embed, a.limit) - ensure_indexes(conn) + if not a.skip_indexes: + ensure_indexes(conn) s = stats(conn) conn.close() db.close() result = {"indexed": done, "corpus_total": total, **{k: v for k, v in s.items() if k in ("total", "by_root")}} + if a.skip_indexes: + result["indexes"] = "skipped" print(json.dumps(result, indent=2) if a.json else f"indexed {done}/{total} leafs; db total {s['total']}") return 0 diff --git a/bin/seed-edelweiss-facts.py b/bin/seed-edelweiss-facts.py new file mode 100755 index 0000000..b5d17ee --- /dev/null +++ b/bin/seed-edelweiss-facts.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +"""Seed confirmed Edelweiss facts via pairing (a x b). Never DROP INDEX.""" +from __future__ import annotations + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "bin" / "tools")) + +import yaml +from kblib import ( # noqa: E402 + CONF_CONFIRMED, + ROOT_FACTS, + connect, + ensure_indexes, + leaf_index_names, + stats, + upsert_leaf, +) +from model2vec import StaticModel # noqa: E402 + +LEX = Path("/home/devops/projects/edelweiss-curasoft/docs/lexicon/edelweiss-lexicon.yml") +CS = Path("/home/devops/projects/curasoft/docs/lexicon/cs-lexicon.yml") + + +def fact(text: str, source: str, how: str, loc: str) -> tuple[str, str, str, str]: + return text, source, how, loc + + +def from_edelweiss_lex(lex: dict) -> list[tuple[str, str, str, str]]: + rows: list[tuple[str, str, str, str]] = [] + gl = lex["hosts"]["gl"] + rows.append(fact( + "CuraSoft GL server VM IP is %s on Fritz LAN (%s)." % (gl["ip"], gl["role"]), + gl["evidence"], "lexicon hosts.gl", "edelweiss-lexicon.yml", + )) + for key in ("pdl", "pdl2", "spdl", "gf"): + h = lex["hosts"][key] + rows.append(fact( + "Edelweiss host %s IP %s role=%s." % (key.upper(), h["ip"], h["role"]), + h["evidence"], "lexicon hosts.%s" % key, "edelweiss-lexicon.yml", + )) + pg, ms = lex["ports"]["postgres"], lex["ports"]["mssql"] + rows.append(fact( + "GL LAN DB: PostgreSQL port %s open=%s; MSSQL %s reachable=%s." + % (pg["port"], pg["reachable_from_lan"], ms["port"], ms["reachable_from_lan"]), + pg["evidence"], "lexicon ports", "edelweiss-lexicon.yml", + )) + b = lex["brain_2dph"] + rows.append(fact( + "Edelweiss 2dph pilot brain at %s; facts need >=2 sources. CLI: %s." + % (b["path"], b["cli"]), + b["evidence"], "lexicon brain_2dph", "edelweiss-lexicon.yml", + )) + v = lex["password_vault"] + rows.append(fact( + "Password vault pilot: %s; OO task #%s deadline %s." + % (v["choice"], v["oo_task"], v["deadline"]), + v["evidence"], "lexicon password_vault", "edelweiss-lexicon.yml", + )) + a = lex["assistant_pilot"] + rows.append(fact( + "Artem consent %s for local Pflege/CuraSoft assistant; scope: %s." + % (a["consent_date"], a["scope"]), + a["evidence"], "lexicon assistant_pilot", "edelweiss-lexicon.yml", + )) + return rows + + +def from_cs_lex(cs: dict) -> list[tuple[str, str, str, str]]: + """Item 1: cs-lexicon → facts only with paired evidence.""" + rows: list[tuple[str, str, str, str]] = [] + src = cs.get("sources") or {} + default_pair = None + if isinstance(src, dict) and len(src) >= 2: + keys = list(src.keys())[:2] + default_pair = "%s x %s" % (keys[0], keys[1]) + ep = cs.get("ep_typ") or {} + for code, meta in ep.items(): + if not isinstance(meta, dict): + continue + ev = (meta.get("evidence") or "").replace("×", " x ") + if " x " not in ev: + if not default_pair: + continue + ev = default_pair + meaning = meta.get("meaning") or meta.get("name") or "" + rows.append(fact( + "CuraSoft ep_typ %s = %s (%s)." % (code, meta.get("name"), meaning), + ev, "cs-lexicon ep_typ", "cs-lexicon.yml", + )) + pair = "cs-lexicon.yml x curasoft-de/kbs" + besuch = cs.get("besuch") or {} + for code, label in besuch.items(): + if code == "note" or not isinstance(label, str): + continue + rows.append(fact( + "CuraSoft Besuch slot %s = %s." % (code, label), + pair, "cs-lexicon besuch", "cs-lexicon.yml", + )) + bm = cs.get("binary_map") or {} + if isinstance(bm, dict) and bm: + rows.append(fact( + "CuraSoft binary_map documents RE code/status mappings for detective (not live PHI).", + "cs-lexicon.yml x curasoft-detective skill", + "cs-lexicon binary_map", "cs-lexicon.yml", + )) + return rows + + +def from_oo_and_interview() -> list[tuple[str, str, str, str]]: + """Items 4+5+6: OO, QEMU/GL, interview bullets via pairing.""" + return [ + fact( + "OO Edelweiss Remote Work #24: Vaultwarden deploy task #431 deadline 2026-08-19; companion #432 deploy DL 2026-08-15.", + "OO#431 x 07-interview-2026-08-12.md", + "OO calendar/tasks pair", "office.produktor.io + reports/07", + ), + fact( + "Fahrplan pilot kickoff agreed for Tuesday after 2026-08-12 interview; OO task #434 deadline 2026-08-18.", + "OO#434 x 07-interview-2026-08-12.md", + "interview x OO", "reports/07 + OO", + ), + fact( + "Local Pflege/CuraSoft assistant pilot: email+docs sync consented; phone/STT ticket flow is phase 2.", + "07-interview-2026-08-12.md x edelweiss-lexicon.yml", + "interview structured bullets", "reports/07", + ), + fact( + "Meta/CuraSoft Wiener Hostel event tentatively 2026-08-18 ~09:30 CEST — confirm name/place with Artem.", + "07-interview-2026-08-12.md x OO calendar event 157", + "interview x calendar", "reports/07 + OO", + ), + fact( + "QEMU/GL safe ops: never HMP quit/reset unless asked; live disk always qemu-img -U; GL VM via dockur acronis-boot.", + "gl-vm-access.md x qemu-monitor-safe rule", + "QEMU/GL access pair", "edelweiss-curasoft/docs + .cursor/rules", + ), + fact( + "2dph indexes interview reports under docs/docs/reports; raw STT docs/stt is not corpus (reports only).", + "2dph-edelweiss-pilot.md x 00-summary.md", + "STT policy pair", "pilot doc + reports", + ), + ] + + +def main() -> int: + lex = yaml.safe_load(LEX.read_text()) + cs = yaml.safe_load(CS.read_text()) if CS.exists() else {} + model = StaticModel.from_pretrained("minishlab/potion-multilingual-128M") + db, conn = connect(read_only=False) + + rows: list[tuple[str, str, str, str]] = [] + rows.extend(from_edelweiss_lex(lex)) + rows.extend(from_cs_lex(cs)) + rows.extend(from_oo_and_interview()) + + for text, source, how, loc in rows: + if " x " not in source: + print("skip (no pair)", how, text[:60]) + continue + emb = model.encode([text])[0].astype(float).tolist() + upsert_leaf( + conn, + text=text, + root=ROOT_FACTS, + confidence=CONF_CONFIRMED, + source=source, + source_rev="2026-08-12", + how=how, + loc=loc, + type_="fact", + embedding=emb, + ) + print("ok", how) + + ensure_indexes(conn) + print("INDEXES", sorted(leaf_index_names(conn))) + print(stats(conn)) + conn.close() + db.close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bin/tools/kblib.py b/bin/tools/kblib.py index 72ddd8c..9d894b9 100644 --- a/bin/tools/kblib.py +++ b/bin/tools/kblib.py @@ -178,9 +178,15 @@ def ensure_indexes(conn: ladybug.Connection) -> None: def drop_indexes(conn: ladybug.Connection) -> None: """No-op. Kept for callers; DROP INDEX is fatal on Ladybug 0.19. - Historical note claimed "drop before bulk MERGE". Measured: upsert while - indexes exist keeps HNSW queryable; DROP leaves ghost catalog tables that - block recreate. Bulk rebuilders must delete `var/kb.lbug` instead. + Historical note claimed "drop before bulk MERGE". Measured on 0.19: + - DROP FTS/VECTOR leaves ghost catalog → CREATE fails permanently until + `var/kb.lbug` is deleted. + - MERGE/upsert while **FTS** exists can corrupt FTS + ("document for node offset N is missing during delete"). + - Upsert while **HNSW** exists stays queryable. + + Bulk rebuilders must delete `var/kb.lbug`, write all leafs (info+facts) + with no indexes, then `ensure_indexes()` once. """ return