feat(mail): full Gmail+OnlyOffice sync, import, and brain indexing
- bin/mail/sync.go: async Go sync engine (8 workers, paginated Gmail via API + OnlyOffice IMAP); Gmail attachments key off body.attachmentId, not MIME partId; ICS sidecars Latin-1->UTF-8 normalized (TestICSToMarkdownNormalizesLatin1) - bin/mail/import: message.json -> markdown; PDFs via pdftotext -layout fast path with docling subprocess fallback for the ~5% textless files - bin/mail/index_mail: fresh-rebuild indexer (repo corpus + mail) avoiding ladybug WAL corruption on bulk-insert into indexed DBs; split from import - bin/kb/index: keep FTS/VECTOR indexes across incremental runs (drop+recreate leaves stale backing tables killing the vector index) - docs: README/PLAN/AGENTS cover the mail pipeline Result: 17,835 messages -> 28,918 info leafs, FTS+HNSW healthy.
This commit is contained in:
Executable
+450
@@ -0,0 +1,450 @@
|
||||
#!/usr/bin/env python3
|
||||
"""mail/import - pull OnlyOffice mails into var/mail/ as markdown.
|
||||
|
||||
bin/mail/import --from-raw var/mail convert Go-synced message.json to md
|
||||
bin/mail/import import newest inbox messages
|
||||
bin/mail/import --folder sent import sent folder
|
||||
bin/mail/import --since 2026-01-01 only messages after a date
|
||||
bin/mail/import --limit 50 cap messages per run
|
||||
bin/mail/import --no-attachments body only, skip attachment conversion
|
||||
bin/mail/import --ocr OCR scanned PDFs/images via docling
|
||||
bin/mail/import --dry-run list messages without writing anything
|
||||
|
||||
Writes one directory per message: var/mail/{folder}/{message_id}/
|
||||
message.md frontmatter + markdown body
|
||||
attachments/ raw attachment files (zips unpacked to _unpacked/)
|
||||
attachments/*.md converted attachment content
|
||||
|
||||
Indexing is a separate step (bin/mail/index_mail): conversion can crash in
|
||||
native docling and must not leave the brain DB mid-transaction.
|
||||
|
||||
Requires ONLYOFFICE_URL/USER/PASS in .env (or env). Idempotent: a message
|
||||
already present (message.md exists) is skipped unless --force.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "bin" / "tools"))
|
||||
|
||||
from mailconv import ( # noqa: E402
|
||||
ARCHIVE_SUFFIXES,
|
||||
IMAGE_SUFFIXES,
|
||||
LEGACY_OFFICE_SUFFIXES,
|
||||
TEXT_SUFFIXES,
|
||||
html_to_markdown,
|
||||
is_convertible,
|
||||
normalize_markdown,
|
||||
subject_to_filename,
|
||||
zip_extract_safe,
|
||||
)
|
||||
|
||||
import requests # noqa: E402
|
||||
|
||||
FOLDER_IDS = {"inbox": 1, "sent": 2, "drafts": 3, "trash": 4, "spam": 5}
|
||||
DEFAULT_LIMIT = 25
|
||||
|
||||
|
||||
def load_env() -> dict:
|
||||
env = {k: v for k, v in os.environ.items()}
|
||||
envfile = ROOT / ".env"
|
||||
if envfile.exists():
|
||||
for line in envfile.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
k, _, v = line.partition("=")
|
||||
env.setdefault(k.strip(), v.strip().strip("\"'"))
|
||||
url = env.get("ONLYOFFICE_URL") or env.get("OO_URL")
|
||||
user = env.get("ONLYOFFICE_USER") or env.get("OO_USER")
|
||||
password = env.get("ONLYOFFICE_PASS") or env.get("OO_PASSWORD")
|
||||
missing = [n for n, v in (("ONLYOFFICE_URL", url), ("ONLYOFFICE_USER", user),
|
||||
("ONLYOFFICE_PASS", password)) if not v]
|
||||
if missing:
|
||||
sys.exit(f"mail/import: missing {', '.join(missing)} (need .env or env)")
|
||||
return {"url": url.rstrip("/"), "user": user, "password": password}
|
||||
|
||||
|
||||
class OOClient:
|
||||
def __init__(self, conf: dict):
|
||||
self.base = conf["url"]
|
||||
self.session = requests.Session()
|
||||
self.token = None
|
||||
self._login(conf)
|
||||
|
||||
def _login(self, conf: dict) -> None:
|
||||
r = self.session.post(f"{self.base}/api/2.0/authentication.json",
|
||||
json={"userName": conf["user"], "password": conf["password"], "type": 0},
|
||||
timeout=30)
|
||||
r.raise_for_status()
|
||||
body = r.json()
|
||||
self.token = (body.get("response") or {}).get("token", "")
|
||||
if not self.token:
|
||||
sys.exit("mail/import: authentication failed (empty token)")
|
||||
|
||||
def _headers(self) -> dict:
|
||||
return {"Authorization": f"Bearer {self.token}", "Accept": "application/json"}
|
||||
|
||||
def get(self, path: str, params: dict | None = None):
|
||||
r = self.session.get(f"{self.base}{path}", params=params, headers=self._headers(), timeout=30)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def list_messages(self, folder: int, page: int = 1, count: int = DEFAULT_LIMIT) -> list[dict]:
|
||||
data = self.get("/api/2.0/mail/messages",
|
||||
params={"folder": folder, "page": page, "count": count})
|
||||
return data.get("response", [])
|
||||
|
||||
def get_message(self, message_id: str) -> dict:
|
||||
data = self.get(f"/api/2.0/mail/messages/{message_id}")
|
||||
return data.get("response", {})
|
||||
|
||||
def download_attachment(self, attach_id, dest: Path) -> bool:
|
||||
"""Download one attachment via the portal session cookie (.ashx handler)."""
|
||||
url = f"{self.base}/addons/mail/httphandlers/download.ashx?attachid={attach_id}"
|
||||
r = self.session.get(url, timeout=60)
|
||||
if r.status_code != 200:
|
||||
return False
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_bytes(r.content)
|
||||
return True
|
||||
|
||||
|
||||
def folder_id(name: str) -> int:
|
||||
if name in FOLDER_IDS:
|
||||
return FOLDER_IDS[name]
|
||||
if name.isdigit():
|
||||
return int(name)
|
||||
sys.exit(f"mail/import: unknown folder '{name}' (use {', '.join(FOLDER_IDS)})")
|
||||
|
||||
|
||||
def safe_attachment_name(att: dict) -> str:
|
||||
name = att.get("fileName") or att.get("storedName") or "attachment"
|
||||
name = re.sub(r"[^\w.\- ]+", "_", name)
|
||||
return name
|
||||
|
||||
|
||||
def convert_file_to_md(path: Path, ocr: bool) -> str | None:
|
||||
"""Convert one attachment file to markdown text; None when not convertible."""
|
||||
suffix = path.suffix.lower()
|
||||
if suffix in TEXT_SUFFIXES:
|
||||
return normalize_markdown(path.read_text(encoding="utf-8", errors="replace"))
|
||||
if suffix in (".docx", ".pptx", ".xlsx", ".html", ".htm", ".epub", ".eml", ".msg"):
|
||||
try:
|
||||
from markitdown import MarkItDown
|
||||
md = MarkItDown()
|
||||
result = md.convert(str(path))
|
||||
return normalize_markdown(result.text_content)
|
||||
except Exception as e:
|
||||
return f"\n<!-- conversion failed: {e} -->\n"
|
||||
if suffix == ".pdf":
|
||||
return _convert_pdf(path, ocr)
|
||||
if suffix in IMAGE_SUFFIXES and ocr:
|
||||
return _convert_pdf(path, ocr)
|
||||
if suffix in LEGACY_OFFICE_SUFFIXES:
|
||||
return _convert_legacy(path)
|
||||
if suffix in ARCHIVE_SUFFIXES:
|
||||
return None # handled by caller (unpack + recurse)
|
||||
return None
|
||||
|
||||
|
||||
def _convert_pdf(path: Path, ocr: bool) -> str:
|
||||
"""Convert one PDF to markdown.
|
||||
|
||||
Fast path: poppler's pdftotext (-layout) extracts exact text from
|
||||
born-digital PDFs in ~15ms vs docling's 1-3s. Only textless PDFs (scanned
|
||||
pages, layout-heavy) fall back to docling, which runs isolated in a
|
||||
subprocess because its native onnx/RT-DETR has segfaulted the main process.
|
||||
"""
|
||||
text = _pdf_fast_text(path)
|
||||
if ocr or text is None or not text.strip():
|
||||
return _convert_pdf_docling(path, ocr)
|
||||
return normalize_markdown(text)
|
||||
|
||||
|
||||
def _pdf_fast_text(path: Path) -> str | None:
|
||||
"""pdftotext -layout; None when poppler is unavailable (or the PDF has no text layer)."""
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["pdftotext", "-layout", str(path), "-"],
|
||||
capture_output=True, timeout=60)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
return None
|
||||
if proc.returncode != 0:
|
||||
return None
|
||||
return proc.stdout.decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def _convert_pdf_docling(path: Path, ocr: bool) -> str:
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[sys.executable, os.path.abspath(__file__), "--pdf-worker", str(path),
|
||||
"--ocr" if ocr else "--no-ocr"],
|
||||
capture_output=True, text=True, timeout=600)
|
||||
except subprocess.TimeoutExpired:
|
||||
return "\n<!-- pdf conversion timed out -->\n"
|
||||
if proc.returncode != 0:
|
||||
tail = proc.stderr.strip().splitlines()[-3:]
|
||||
return f"\n<!-- pdf conversion failed: {proc.returncode}: {' | '.join(tail)} -->\n"
|
||||
return proc.stdout
|
||||
|
||||
|
||||
def _pdf_worker(path: Path, ocr: bool) -> None:
|
||||
"""docling worker entry: prints converted markdown on stdout, exits non-zero on error."""
|
||||
try:
|
||||
from docling.document_converter import DocumentConverter, PdfFormatOption
|
||||
from docling.datamodel.pipeline_options import PdfPipelineOptions
|
||||
opts = PdfPipelineOptions()
|
||||
opts.do_ocr = bool(ocr)
|
||||
opts.do_table_structure = True
|
||||
conv = DocumentConverter(format_options={"pdf": PdfFormatOption(pipeline_options=opts)})
|
||||
res = conv.convert(str(path))
|
||||
sys.stdout.write(normalize_markdown(res.document.export_to_markdown()))
|
||||
sys.exit(0)
|
||||
except Exception as e:
|
||||
# errors/stacktraces to stderr; the caller only reports a one-liner
|
||||
print(f"pdf-worker: {e}", file=sys.stderr)
|
||||
import traceback
|
||||
traceback.print_exc(file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def _convert_legacy(path: Path) -> str:
|
||||
"""Legacy .doc/.xls/.ppt -> md via pandoc (installed) or a stub."""
|
||||
try:
|
||||
out = subprocess.run(["pandoc", str(path), "-t", "markdown"],
|
||||
capture_output=True, text=True, timeout=120)
|
||||
if out.returncode == 0 and out.stdout.strip():
|
||||
return normalize_markdown(out.stdout)
|
||||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||||
pass
|
||||
return f"\n<!-- legacy {path.suffix} not convertible (pandoc unavailable) -->\n"
|
||||
|
||||
|
||||
def write_message_md(msg: dict, folder: str, out_dir: Path, target_dir: Path | None = None) -> Path:
|
||||
import yaml
|
||||
body_html = msg.get("htmlBody") or ""
|
||||
body_text = msg.get("textBody") or ""
|
||||
body_md = ""
|
||||
if body_html.strip():
|
||||
body_md = html_to_markdown(body_html)
|
||||
elif body_text.strip():
|
||||
body_md = normalize_markdown(body_text)
|
||||
# accept both OnlyOffice (receivedDate) and Go-sync (receivedAt) date keys
|
||||
date = msg.get("receivedDate") or msg.get("receivedAt") or ""
|
||||
if date and not isinstance(date, str):
|
||||
date = str(date)
|
||||
meta = {
|
||||
"id": msg.get("id"),
|
||||
"source": msg.get("source"),
|
||||
"folder": folder,
|
||||
"subject": msg.get("subject", ""),
|
||||
"from": msg.get("from", ""),
|
||||
"to": msg.get("to", ""),
|
||||
"cc": msg.get("cc", ""),
|
||||
"date": date,
|
||||
"has_attachments": bool(msg.get("hasAttachments")),
|
||||
"mime_message_id": msg.get("mimeMessageId", ""),
|
||||
"calendar_uid": msg.get("calendarUid", ""),
|
||||
"type": "mail",
|
||||
}
|
||||
meta = {k: v for k, v in meta.items() if v not in (None, "")}
|
||||
frontmatter = "---\n" + yaml.safe_dump(meta, sort_keys=False, allow_unicode=True).strip() + "\n---\n"
|
||||
content = f"{frontmatter}\n# {meta.get('subject','')}\n\n{body_md}".strip() + "\n"
|
||||
if target_dir is not None:
|
||||
msg_dir = target_dir
|
||||
else:
|
||||
msg_dir = out_dir / folder / str(meta.get("id"))
|
||||
msg_dir.mkdir(parents=True, exist_ok=True)
|
||||
md_path = msg_dir / "message.md"
|
||||
md_path.write_text(content, encoding="utf-8")
|
||||
return md_path
|
||||
|
||||
|
||||
def convert_attachments(msg: dict, msg_dir: Path, ocr: bool) -> list[dict]:
|
||||
"""Download + convert each attachment; returns [{name, md, raw}] summaries.
|
||||
|
||||
raw file keeps the API storedName (unique hash, avoids collisions); the
|
||||
markdown is named after the friendly fileName when available.
|
||||
|
||||
In --from-raw mode attachments are already on disk (Go sync wrote them;
|
||||
.ics already has a structured .md sidecar). Files with an existing .md
|
||||
sidecar are left as-is, only unconverted raws are converted here.
|
||||
"""
|
||||
out: list[dict] = []
|
||||
atts = msg.get("attachments") or []
|
||||
att_dir = msg_dir / "attachments"
|
||||
for att in atts:
|
||||
aid = att.get("fileId")
|
||||
display = safe_attachment_name(att)
|
||||
stored = att.get("storedName")
|
||||
raw_name = safe_attachment_name({"storedName": stored}) if stored else display
|
||||
raw = att_dir / raw_name
|
||||
if aid and not raw.exists() and OOCLIENT is not None:
|
||||
if not OOCLIENT.download_attachment(aid, raw):
|
||||
out.append({"name": display, "md": "\n<!-- download failed -->\n", "raw": str(raw)})
|
||||
continue
|
||||
if not raw.exists():
|
||||
out.append({"name": display, "md": "\n<!-- raw missing -->\n", "raw": str(raw)})
|
||||
continue
|
||||
md_stem = Path(display).stem or raw.stem
|
||||
md_path = att_dir / f"{md_stem}.md"
|
||||
# Go sync pre-wrote structured .md for .ics; keep it.
|
||||
if not md_path.exists():
|
||||
md_text = _convert_att_recursive(raw, ocr)
|
||||
md_path.write_text(f"# Attachment: {display}\n\n{md_text}\n", encoding="utf-8")
|
||||
else:
|
||||
md_text = md_path.read_text(encoding="utf-8", errors="replace")
|
||||
out.append({"name": display, "md": md_text, "raw": str(raw), "md_file": str(md_path)})
|
||||
return out
|
||||
|
||||
|
||||
def _convert_att_recursive(path: Path, ocr: bool) -> str:
|
||||
if path.suffix.lower() in ARCHIVE_SUFFIXES:
|
||||
parts: list[str] = []
|
||||
unpack = path.parent / "_unpacked" / path.stem
|
||||
files = zip_extract_safe(path, unpack)
|
||||
for f in files:
|
||||
sub = _convert_att_recursive(f, ocr)
|
||||
if sub and sub.strip():
|
||||
parts.append(f"## {f.name}\n\n{sub}")
|
||||
return "\n\n".join(parts) if parts else "\n<!-- empty zip -->\n"
|
||||
text = convert_file_to_md(path, ocr)
|
||||
return text or "\n<!-- not convertible -->\n"
|
||||
|
||||
|
||||
# module-level client for attachment downloads in convert_attachments
|
||||
OOCLIENT: OOClient | None = None
|
||||
|
||||
|
||||
def convert_one(msg_dir: Path, full: dict, folder: str, out_root: Path,
|
||||
ocr: bool, no_attachments: bool, target_dir: Path | None = None) -> dict:
|
||||
"""Write message.md + convert attachments for one message dict.
|
||||
|
||||
Works for both live API messages and the Go-sync message.json shape
|
||||
(source field optional; attachments read from attachments/ dir).
|
||||
target_dir overrides the derived path (used by --from-raw where the
|
||||
directory layout is authoritative, not the message folder field).
|
||||
"""
|
||||
mid = str(full.get("id"))
|
||||
write_message_md(full, folder, out_root, target_dir=target_dir)
|
||||
converted: list[dict] = []
|
||||
if not no_attachments:
|
||||
converted = convert_attachments(full, msg_dir, ocr)
|
||||
return {"id": mid, "subject": full.get("subject", ""),
|
||||
"date": full.get("receivedDate", "") or full.get("receivedAt", ""),
|
||||
"attachments": len(converted)}
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
global OOCLIENT
|
||||
p = argparse.ArgumentParser(description="pull OnlyOffice mails to var/mail as markdown")
|
||||
p.add_argument("--folder", default="inbox", help="inbox|sent|drafts|trash|spam or numeric id")
|
||||
p.add_argument("--limit", type=int, default=DEFAULT_LIMIT, help="max messages per run")
|
||||
p.add_argument("--offset", type=int, default=0, help="skip N messages")
|
||||
p.add_argument("--since", default="", help="only messages received after YYYY-MM-DD")
|
||||
p.add_argument("--id", action="append", default=[], help="import specific message id (repeatable)")
|
||||
p.add_argument("--from-raw", default="",
|
||||
help="convert Go-synced dirs (var/mail/<folder>/<id>/message.json) to markdown")
|
||||
p.add_argument("--no-attachments", action="store_true", help="skip attachment download+convert")
|
||||
p.add_argument("--ocr", action="store_true", help="OCR scanned PDFs/images via docling")
|
||||
p.add_argument("--force", action="store_true", help="re-import even if message.md exists")
|
||||
p.add_argument("--dry-run", action="store_true", help="list messages, write nothing")
|
||||
p.add_argument("--json", action="store_true")
|
||||
p.add_argument("--pdf-worker", default="", help=argparse.SUPPRESS)
|
||||
p.add_argument("--no-ocr", action="store_true", help=argparse.SUPPRESS)
|
||||
a = p.parse_args(argv)
|
||||
|
||||
if a.pdf_worker:
|
||||
_pdf_worker(Path(a.pdf_worker), ocr=not a.no_ocr)
|
||||
return 0
|
||||
|
||||
conf = load_env()
|
||||
fid = folder_id(a.folder)
|
||||
out_root = ROOT / "var" / "mail"
|
||||
summary: list[dict] = []
|
||||
if a.from_raw:
|
||||
OOCLIENT = None
|
||||
raw_root = Path(a.from_raw)
|
||||
for msg_dir in sorted(raw_root.rglob("message.json")):
|
||||
mid = msg_dir.parent.name
|
||||
entry = {"id": mid, "subject": "", "date": "",
|
||||
"attachments": 0, "skipped": False}
|
||||
md_path = msg_dir.parent / "message.md"
|
||||
if md_path.exists() and not a.force:
|
||||
entry["skipped"] = True
|
||||
summary.append(entry)
|
||||
continue
|
||||
if a.dry_run:
|
||||
entry["skipped"] = "dry-run"
|
||||
summary.append(entry)
|
||||
continue
|
||||
full = json.loads(msg_dir.read_text(encoding="utf-8"))
|
||||
entry.update(convert_one(msg_dir.parent, full, full.get("folder") or a.folder,
|
||||
raw_root, a.ocr, a.no_attachments,
|
||||
target_dir=msg_dir.parent))
|
||||
summary.append(entry)
|
||||
else:
|
||||
OOCLIENT = OOClient(conf)
|
||||
if a.id:
|
||||
messages = [{"id": i} for i in a.id]
|
||||
else:
|
||||
page = 1
|
||||
messages = []
|
||||
want = a.offset + a.limit
|
||||
while len(messages) < want:
|
||||
count = min(DEFAULT_LIMIT, want - len(messages))
|
||||
chunk = OOCLIENT.list_messages(fid, page=page, count=count)
|
||||
if not chunk:
|
||||
break
|
||||
messages.extend(chunk)
|
||||
if len(chunk) < count:
|
||||
break
|
||||
page += 1
|
||||
messages = messages[a.offset:a.offset + a.limit]
|
||||
if a.since:
|
||||
messages = [m for m in messages
|
||||
if (m.get("receivedDate") or "") >= a.since]
|
||||
|
||||
for m in messages:
|
||||
mid = str(m.get("id"))
|
||||
entry = {"id": mid, "subject": m.get("subject", ""),
|
||||
"date": m.get("receivedDate", ""), "attachments": 0, "skipped": False}
|
||||
msg_dir = out_root / a.folder / mid
|
||||
md_path = msg_dir / "message.md"
|
||||
if md_path.exists() and not a.force:
|
||||
entry["skipped"] = True
|
||||
summary.append(entry)
|
||||
continue
|
||||
if a.dry_run:
|
||||
entry["skipped"] = "dry-run"
|
||||
summary.append(entry)
|
||||
continue
|
||||
full = OOCLIENT.get_message(mid)
|
||||
entry.update(convert_one(msg_dir, full, a.folder, out_root, a.ocr, a.no_attachments,
|
||||
target_dir=msg_dir))
|
||||
summary.append(entry)
|
||||
if a.json:
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
imported = [e for e in summary if not e["skipped"]]
|
||||
print(f"mail/import: folder={a.folder} checked={len(summary)} "
|
||||
f"imported={len(imported)} (skipped={sum(e['skipped'] is True for e in summary)})")
|
||||
for e in summary:
|
||||
flag = "skip" if e["skipped"] is True else ("dry" if e["skipped"] == "dry-run" else "ok ")
|
||||
print(f" [{flag}] {e['id']} {e['date'][:10]} {e['subject'][:60]}"
|
||||
f" (atts={e['attachments']})")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
Reference in New Issue
Block a user