eSliderandCursor
b73b4d4f97
Ladybug 0.19 DROP INDEX leaves `_0_Leaf_vec_UPPER` / `0_id_docs` in catalog so CREATE fails while SHOW_INDEXES omits the index; create_fts_and_vector used to swallow that. Never drop FTS/VECTOR; ensure_indexes after upserts; rebuild = delete kb.lbug. Add compose.edelweiss.yml + regression tests. Co-authored-by: Cursor <cursoragent@cursor.com>
76 lines
2.0 KiB
Python
76 lines
2.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Load test: bulk insert performance (writing facts to the brain)."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import time
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "bin" / "tools"))
|
|
|
|
from kblib import connect, init_schema, upsert_leaf, ensure_indexes
|
|
from model2vec import StaticModel
|
|
|
|
|
|
DB_PATH = ROOT / "var" / "kb.lbug"
|
|
|
|
|
|
def run_bulk_tests(count: int = 100, _drop_indexes: bool = True) -> dict:
|
|
results: dict = {}
|
|
|
|
print(f"Bulk insert test: {count} leafs")
|
|
|
|
# Fresh DB for load test — never DROP INDEX on a live catalog.
|
|
if DB_PATH.exists():
|
|
DB_PATH.unlink()
|
|
|
|
db, conn = connect(str(DB_PATH), read_only=False)
|
|
init_schema(conn)
|
|
|
|
model = StaticModel.from_pretrained("minishlab/potion-multilingual-128M")
|
|
|
|
t0 = time.time()
|
|
for i in range(count):
|
|
text = f"bulk load test fact {i:03d} running container on host"
|
|
emb = model.encode([text])[0].astype(float).tolist()
|
|
upsert_leaf(
|
|
conn,
|
|
text=text,
|
|
root="facts",
|
|
confidence="confirmed",
|
|
source="load-test",
|
|
source_rev="2dph",
|
|
how="load_test_bulk",
|
|
loc=f"test:{i}",
|
|
type_="fact",
|
|
embedding=emb,
|
|
)
|
|
t1 = time.time()
|
|
|
|
# Create indexes once after bulk write (DROP+recreate is unsafe on Ladybug 0.19).
|
|
ensure_indexes(conn)
|
|
|
|
conn.close()
|
|
db.close()
|
|
|
|
elapsed = t1 - t0
|
|
results["total_seconds"] = elapsed
|
|
results["throughput"] = count / elapsed # leafs/sec
|
|
results["count"] = count
|
|
return results
|
|
|
|
|
|
def main():
|
|
count_str = sys.argv[1] if len(sys.argv) > 1 else "200"
|
|
count = int(count_str)
|
|
drop_str = sys.argv[2] if len(sys.argv) > 2 else "true"
|
|
drop_indexes = drop_str.lower() in ("1", "true", "yes")
|
|
|
|
r = run_bulk_tests(count=count, _drop_indexes=drop_indexes)
|
|
print(json.dumps(r, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |