feat(mail): full Gmail+OnlyOffice sync, import, and brain indexing
- bin/mail/sync.go: async Go sync engine (8 workers, paginated Gmail via API + OnlyOffice IMAP); Gmail attachments key off body.attachmentId, not MIME partId; ICS sidecars Latin-1->UTF-8 normalized (TestICSToMarkdownNormalizesLatin1) - bin/mail/import: message.json -> markdown; PDFs via pdftotext -layout fast path with docling subprocess fallback for the ~5% textless files - bin/mail/index_mail: fresh-rebuild indexer (repo corpus + mail) avoiding ladybug WAL corruption on bulk-insert into indexed DBs; split from import - bin/kb/index: keep FTS/VECTOR indexes across incremental runs (drop+recreate leaves stale backing tables killing the vector index) - docs: README/PLAN/AGENTS cover the mail pipeline Result: 17,835 messages -> 28,918 info leafs, FTS+HNSW healthy.
This commit is contained in:
@@ -0,0 +1,148 @@
|
||||
"""mailconv - pure helpers for bin/mail/import (mail -> markdown + attachments).
|
||||
|
||||
Shared with unit tests in bin/tools/test_mailconv.py. No network, no OnlyOffice
|
||||
dependencies here: everything is `str -> str` or `Path -> str` so the tests run
|
||||
offline against fixtures.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import re
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
# Body part / attachment file suffixes we know how to turn into markdown text.
|
||||
TEXT_SUFFIXES = {".md", ".markdown", ".txt", ".csv", ".json", ".xml", ".yaml", ".yml", ".log", ".tsv",
|
||||
".ics", ".ical", ".vcf", ".eml"}
|
||||
OFFICE_SUFFIXES = {".docx", ".pptx", ".xlsx", ".html", ".htm", ".epub", ".eml", ".msg"}
|
||||
PDF_SUFFIXES = {".pdf"}
|
||||
IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".tif", ".webp"}
|
||||
ARCHIVE_SUFFIXES = {".zip"}
|
||||
# Legacy binary Office (doc/xls/ppt) — markitdown/docling skip them; we try
|
||||
# pandoc first, else leave a stub.
|
||||
LEGACY_OFFICE_SUFFIXES = {".doc", ".xls", ".ppt"}
|
||||
|
||||
CONVERTIBLE_SUFFIXES = (
|
||||
TEXT_SUFFIXES | OFFICE_SUFFIXES | PDF_SUFFIXES | IMAGE_SUFFIXES | ARCHIVE_SUFFIXES | LEGACY_OFFICE_SUFFIXES
|
||||
)
|
||||
|
||||
|
||||
def clean_email_address(raw: str) -> str:
|
||||
"""Extract the bare email from '"Name" <a@b.c>' and strip control chars."""
|
||||
m = re.search(r"<([^<>@\s]+@[^<>@\s]+)>", raw)
|
||||
return (m.group(1) if m else raw).strip()
|
||||
|
||||
|
||||
def subject_to_filename(subject: str, max_len: int = 80) -> str:
|
||||
"""Turn a mail subject into a filesystem-safe slug (keep first token readable)."""
|
||||
s = re.sub(r"[^\w\-. ]+", "", subject).strip()
|
||||
s = re.sub(r"\s+", "_", s)
|
||||
s = s.strip("._")
|
||||
if not s:
|
||||
s = "untitled"
|
||||
return s[:max_len] or "untitled"
|
||||
|
||||
|
||||
def strip_html(html_text: str) -> str:
|
||||
"""Naive HTML -> plain text fallback (used only if markitdown is missing)."""
|
||||
import re as _re
|
||||
text = _re.sub(r"(?is)<(script|style)[^>]*>.*?</\1>", "", html_text)
|
||||
text = _re.sub(r"(?s)<br\s*/?>", "\n", text)
|
||||
text = _re.sub(r"(?s)</p>", "\n\n", text)
|
||||
text = _re.sub(r"(?s)<[^>]+>", "", text)
|
||||
return html.unescape(text).strip()
|
||||
|
||||
|
||||
def _unwrap_tables(html_text: str) -> str:
|
||||
"""Unwrap mail HTML tables into pipe-joined text lines.
|
||||
|
||||
Outlook/Stripe-style emails wrap content in nested spacer/frame tables that
|
||||
markitdown renders as hundreds of `--- |` cells and duplicated blocks.
|
||||
Every <table> becomes plain "cell1 | cell2" lines (key-value pairs survive),
|
||||
so only headings/paragraphs/links reach markitdown and no table noise is left.
|
||||
"""
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
except Exception:
|
||||
return html_text
|
||||
soup = BeautifulSoup(html_text, "html.parser")
|
||||
for table in reversed(soup.find_all("table")):
|
||||
lines: list[str] = []
|
||||
for row in table.find_all("tr"):
|
||||
cells = [c.get_text(" ", strip=True) for c in row.find_all(["td", "th"])]
|
||||
line = " | ".join(x for x in cells if x)
|
||||
if line:
|
||||
lines.append(line)
|
||||
if lines:
|
||||
table.replace_with(BeautifulSoup("\n".join(lines), "html.parser"))
|
||||
else:
|
||||
table.decompose()
|
||||
return str(soup)
|
||||
|
||||
|
||||
def html_to_markdown(html_text: str) -> str:
|
||||
"""Convert a mail HTML body to markdown using markitdown when available."""
|
||||
html_text = _unwrap_tables(html_text)
|
||||
try:
|
||||
from markitdown import MarkItDown
|
||||
import io
|
||||
md = MarkItDown()
|
||||
result = md.convert_stream(io.BytesIO(html_text.encode("utf-8", errors="replace")),
|
||||
file_extension=".html")
|
||||
text = result.text_content.strip()
|
||||
if text:
|
||||
return normalize_markdown(text)
|
||||
except Exception:
|
||||
pass
|
||||
return normalize_markdown(strip_html(html_text))
|
||||
|
||||
|
||||
def normalize_markdown(text: str) -> str:
|
||||
"""Collapse the pdfminer/markitdown NUL artifacts and stray control chars."""
|
||||
# NUL bytes that pdfminer inserts between digits/letters.
|
||||
text = text.replace("\x00", "")
|
||||
# Email spacer noise: zero-width chars, soft hyphens, figure spaces,
|
||||
# combining grapheme joiner, BOM.
|
||||
for ch in ("\ufeff", "\u200b", "\u034f", "\u00ad", "\u2007", "\u2008", "\u200a", "\u2002"):
|
||||
text = text.replace(ch, "")
|
||||
text = re.sub(r"[ \t]{2,}", " ", text)
|
||||
# Trim trailing whitespace per line so space-only spacer rows collapse.
|
||||
text = "\n".join(l.rstrip() for l in text.split("\n"))
|
||||
# Collapse 3+ blank lines to two.
|
||||
text = re.sub(r"\n{3,}", "\n\n", text)
|
||||
# Remove weird trailing control chars.
|
||||
text = "".join(ch for ch in text if ch >= " " or ch in "\n\t")
|
||||
return text.strip()
|
||||
|
||||
|
||||
def split_zip_members(zip_path: Path) -> list[str]:
|
||||
"""Return safe member names of a zip archive (skips dir entries)."""
|
||||
try:
|
||||
with zipfile.ZipFile(zip_path) as zf:
|
||||
return [m for m in zf.namelist() if not m.endswith("/")]
|
||||
except zipfile.BadZipFile:
|
||||
return []
|
||||
|
||||
|
||||
def zip_extract_safe(zip_path: Path, dest: Path) -> list[Path]:
|
||||
"""Extract a zip into dest guarding against path traversal; returns files."""
|
||||
out: list[Path] = []
|
||||
try:
|
||||
with zipfile.ZipFile(zip_path) as zf:
|
||||
for member in zf.infolist():
|
||||
if member.is_dir():
|
||||
continue
|
||||
target = (dest / member.filename).resolve()
|
||||
if not target.is_relative_to(dest.resolve()):
|
||||
continue
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zf.open(member) as src, open(target, "wb") as dst:
|
||||
dst.write(src.read())
|
||||
out.append(target)
|
||||
except zipfile.BadZipFile:
|
||||
return []
|
||||
return out
|
||||
|
||||
|
||||
def is_convertible(suffix: str) -> bool:
|
||||
return suffix.lower() in CONVERTIBLE_SUFFIXES
|
||||
@@ -0,0 +1,123 @@
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
|
||||
from mailconv import ( # noqa: E402
|
||||
clean_email_address,
|
||||
html_to_markdown,
|
||||
is_convertible,
|
||||
normalize_markdown,
|
||||
split_zip_members,
|
||||
subject_to_filename,
|
||||
zip_extract_safe,
|
||||
)
|
||||
from mailconv import _unwrap_tables # noqa: E402
|
||||
|
||||
|
||||
class TestMailConv(unittest.TestCase):
|
||||
def test_clean_email_address(self):
|
||||
self.assertEqual(clean_email_address('"Ben Baker" <bb@teks.com>'), "bb@teks.com")
|
||||
self.assertEqual(clean_email_address("eslider@gmail.com"), "eslider@gmail.com")
|
||||
self.assertEqual(clean_email_address("<a@b.c>"), "a@b.c")
|
||||
|
||||
def test_subject_to_filename(self):
|
||||
self.assertEqual(subject_to_filename("Your receipt #2422"), "Your_receipt_2422")
|
||||
self.assertEqual(subject_to_filename("a/b\\c:d*e"), "abcde")
|
||||
self.assertEqual(subject_to_filename(" "), "untitled")
|
||||
|
||||
def test_html_to_markdown(self):
|
||||
out = html_to_markdown("<html><body><h1>Hi</h1><p>Some <b>bold</b> text.</p></body></html>")
|
||||
self.assertIn("Hi", out)
|
||||
self.assertIn("**bold**", out)
|
||||
|
||||
def test_html_strip_fallback(self):
|
||||
from mailconv import strip_html
|
||||
self.assertEqual(strip_html("<p>a</p><p>b</p>"), "a\n\nb")
|
||||
|
||||
def test_flatten_layout_tables(self):
|
||||
html = ("<table><tr>"
|
||||
+ "".join(f"<td>spacer{i}</td>" for i in range(12))
|
||||
+ "</tr></table>"
|
||||
+ "<p>real</p>"
|
||||
+ "<table><tr><td>a</td><td>b</td></tr></table>")
|
||||
out = _unwrap_tables(html)
|
||||
# tables unwrapped into pipe text; no <td> left; content preserved
|
||||
self.assertNotIn("<td>spacer0</td>", out)
|
||||
self.assertIn("spacer0 | spacer1", out)
|
||||
self.assertIn("a | b", out)
|
||||
self.assertIn("real", out)
|
||||
|
||||
def test_html_to_markdown_layout_clean(self):
|
||||
html = "<table><tr>" + "".join(f"<td>x{i}</td>" for i in range(12)) + "</tr></table><h1>Hi</h1>"
|
||||
out = html_to_markdown(html)
|
||||
self.assertIn("Hi", out)
|
||||
self.assertNotIn("| ---", out)
|
||||
|
||||
def test_normalize_markdown_removes_nul(self):
|
||||
self.assertEqual(normalize_markdown("Z0\x00A\x00Y\x00B"), "Z0AYB")
|
||||
self.assertEqual(normalize_markdown("a\n\n\n\nb"), "a\n\nb")
|
||||
|
||||
def test_normalize_strips_email_noise(self):
|
||||
noisy = "\ufeffa\u200b\u034f\u00ad\u2007\u2002 b\u200a c\u2008"
|
||||
out = normalize_markdown(noisy)
|
||||
self.assertNotIn("\u200b", out)
|
||||
self.assertNotIn("\ufeff", out)
|
||||
self.assertNotIn("\u034f", out)
|
||||
self.assertIn("a b c", out)
|
||||
|
||||
def test_split_zip_members(self):
|
||||
p = Path(self._mk_zip(["a.txt", "sub/b.txt"]))
|
||||
self.assertEqual(split_zip_members(p), ["a.txt", "sub/b.txt"])
|
||||
|
||||
def test_zip_extract_safe(self):
|
||||
zip_path = self._mk_zip(["a.txt", "dir/b.txt"])
|
||||
dest = Path(self._tmp("x"))
|
||||
files = zip_extract_safe(zip_path, dest)
|
||||
self.assertEqual(len(files), 2)
|
||||
self.assertTrue((dest / "a.txt").exists())
|
||||
self.assertTrue((dest / "dir" / "b.txt").exists())
|
||||
|
||||
def test_zip_extract_safe_blocks_traversal(self):
|
||||
# member "../evil.txt" must not escape dest
|
||||
zip_path = Path(self._tmp("evil.zip"))
|
||||
with zipfile.ZipFile(zip_path, "w") as zf:
|
||||
zf.writestr("../evil.txt", "boom")
|
||||
dest = Path(self._tmp("out"))
|
||||
files = zip_extract_safe(zip_path, dest)
|
||||
self.assertEqual(files, [])
|
||||
self.assertFalse((dest.parent / "evil.txt").exists())
|
||||
|
||||
def test_is_convertible(self):
|
||||
self.assertTrue(is_convertible(".pdf"))
|
||||
self.assertTrue(is_convertible(".zip"))
|
||||
self.assertTrue(is_convertible(".docx"))
|
||||
self.assertTrue(is_convertible(".TXT"))
|
||||
self.assertFalse(is_convertible(".exe"))
|
||||
self.assertFalse(is_convertible(".unknown"))
|
||||
|
||||
def _mk_zip(self, members):
|
||||
zpath = Path(self._tmp("arc.zip"))
|
||||
with zipfile.ZipFile(zpath, "w") as zf:
|
||||
for m in members:
|
||||
zf.writestr(m, "content")
|
||||
return str(zpath)
|
||||
|
||||
def _tmp(self, name):
|
||||
d = self.__class__._td
|
||||
p = Path(d) / name
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
return str(p)
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
import tempfile
|
||||
cls._td = tempfile.mkdtemp(prefix="mailconv_test_")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user