feat(mail): full Gmail+OnlyOffice sync, import, and brain indexing

- bin/mail/sync.go: async Go sync engine (8 workers, paginated Gmail via
  API + OnlyOffice IMAP); Gmail attachments key off body.attachmentId, not
  MIME partId; ICS sidecars Latin-1->UTF-8 normalized (TestICSToMarkdownNormalizesLatin1)
- bin/mail/import: message.json -> markdown; PDFs via pdftotext -layout
  fast path with docling subprocess fallback for the ~5% textless files
- bin/mail/index_mail: fresh-rebuild indexer (repo corpus + mail) avoiding
  ladybug WAL corruption on bulk-insert into indexed DBs; split from import
- bin/kb/index: keep FTS/VECTOR indexes across incremental runs (drop+recreate
  leaves stale backing tables killing the vector index)
- docs: README/PLAN/AGENTS cover the mail pipeline

Result: 17,835 messages -> 28,918 info leafs, FTS+HNSW healthy.
This commit is contained in:
2026-08-11 21:57:38 +01:00
parent 8781c0c3eb
commit 678a1d1dba
20 changed files with 5000 additions and 26 deletions
+9 -5
View File
@@ -116,11 +116,13 @@ def main(argv: list[str]) -> int:
db, conn = connect(DB_PATH, read_only=False)
init_schema(conn)
if not (a.rebuild or _already_indexed(conn)):
create_fts_and_vector(conn, force=True)
# Keep the FTS/VECTOR indexes in place across incremental runs: ladybug's
# DROP INDEX leaves the backing tables registered on migrated DBs, so a
# drop+recreate silently kills the vector index. Only create when missing.
embed = embedder()
done, total = index_leafs(conn, leafs, embed, a.limit)
create_fts_and_vector(conn, force=(done > 0 or a.rebuild))
if a.rebuild or not _has_indexes(conn):
create_fts_and_vector(conn, force=True)
s = stats(conn)
conn.close()
db.close()
@@ -130,9 +132,11 @@ def main(argv: list[str]) -> int:
return 0
def _already_indexed(conn) -> bool:
def _has_indexes(conn) -> bool:
try:
return conn.execute("MATCH (l:Leaf) RETURN count(*)").get_all()[0][0] > 0
rows = conn.execute("CALL SHOW_INDEXES() RETURN *").get_all()
names = {row[1] for row in rows if row[0] == "Leaf"}
return "id" in names and "Leaf_vec" in names
except Exception:
return False