From b612d3cc97a8a2e24bf163a73dd60ab4310ead13 Mon Sep 17 00:00:00 2001 From: Andriy Oblivantsev Date: Fri, 14 Aug 2026 17:59:02 +0100 Subject: [PATCH] feat(mail): adapt M365 ETL loop for OO+Gmail bots. Default compose mail-sync to onlyoffice,gmail every 300s with import on new mail; gate full --rebuild behind MAIL_SYNC_INDEX. Fix mail-build stage order and add bin/stack/start-mail-sync. --- AGENTS.md | 10 +++++++++- Dockerfile | 17 +++++++++-------- PLAN.md | 13 +++++++++---- README.md | 2 ++ bin/docker-entrypoint | 21 ++++++++++++--------- bin/stack/lib.sh | 19 ++++++++++++++++--- bin/stack/start-mail-sync | 21 +++++++++++++++++++++ bin/stack/stop | 2 +- bin/tools/test_stack.py | 21 ++++++++++++++++++--- compose.yaml | 19 ++++++++++++------- docs/runbook.md | 4 +++- 11 files changed, 112 insertions(+), 37 deletions(-) create mode 100755 bin/stack/start-mail-sync diff --git a/AGENTS.md b/AGENTS.md index 60de48d..d74b014 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -66,12 +66,20 @@ var/ kb.lbug, var/mail/*, caches (gitignored) ```bash bin/mail/sync.go --source onlyoffice,gmail --workers 8 --out var/mail # raw message.json + attachments bin/mail/sync.go --source gmail --query 'from:example.com' --out var/mail # Gmail search (default in:inbox) +bin/mail/sync.go --source m365 --env ~/.config/brain/mail.env --out var/mail # Microsoft Graph (delta) bin/mail/import.go --from-raw var/mail # message.json → message.md (convert only) bin/brain/index.go --rebuild --with-facts --with-chats +bin/stack/start-mail-sync # compose ETL: sync→import every 300s ``` - `sync` (Go) downloads messages + attachments; Gmail uses paginated list + - `body.attachmentId` (not partId) for attachments. + `body.attachmentId` (not partId) for attachments. Sources: `onlyoffice`, + `gmail`, `m365` (client-credentials + delta link; commit after success). +- Compose `mail-sync` / `bin/stack/start-mail-sync`: ETL loop (default + `onlyoffice,gmail`, 300s). On `new>0` runs import; full `--rebuild` only if + `MAIL_SYNC_INDEX=1`. Secrets: `~/.config/brain/mail.env` + `~/.gmail-mcp`. + Case wrappers (e.g. family `gmail-sync-la-quinta.sh`) and ai-bot + `gmail-reauth.sh` reuse this sync/OAuth — do not fork corpus download. - `import` converts body + attachments to markdown. PDFs use poppler `pdftotext -layout` fast path (~15ms); textless/scanned PDFs use `pdftoppm` + tesseract `eng+deu` (`bin/mail/ocr.go`). Optional diff --git a/Dockerfile b/Dockerfile index 20cc43b..ad021b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,15 @@ # API: Go + ladybug via Zig CGO (no CPython). # Index: Python write path (profile `index` until brain/add is v2). +# --- mail-sync: standalone M365/OnlyOffice/Gmail puller (pure Go, no CGO) --- +FROM golang:1.26-bookworm AS mail-build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY bin/mail ./bin/mail +COPY internal ./internal +RUN CGO_ENABLED=0 go build -o /mail-sync ./bin/mail/sync.go + # --- Python sidecar (Ladybug write / rebuild) --- FROM python:3.12-slim AS index @@ -38,14 +47,6 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ CMD python -c "import model2vec, ladybug, mistune; print('ok')" || exit 1 ENTRYPOINT ["/app/bin/docker-entrypoint"] -# --- mail-sync: standalone M365/OnlyOffice/Gmail puller (pure Go, no CGO) --- -FROM golang:1.26-bookworm AS mail-build -WORKDIR /src -COPY go.mod go.sum ./ -RUN go mod download -COPY bin/mail ./bin/mail -RUN CGO_ENABLED=0 go build -o /mail-sync ./bin/mail/sync.go - # --- Go API: CGO with Zig, not gcc --- FROM golang:1.26-bookworm AS api-build WORKDIR /src diff --git a/PLAN.md b/PLAN.md index c98a7f0..06f5153 100644 --- a/PLAN.md +++ b/PLAN.md @@ -86,7 +86,7 @@ detective method: **a fact needs ≥2 independent sources or it is mail/ocr.go tesseract eng+deu (pdftoppm scans) md/import (deprecated; bin/markdown/import.go) brain/extract brain/audit brain/deduce (thinking wrapper) - stack/start start-assistant stop status + stack/start start-assistant start-mail-sync stop status web/search (deprecated shim → web/search.go) db/psql-yq (vendored) ssh-tunnel onlyoffice pg tunnel 5433 @@ -141,8 +141,9 @@ Common props on every node/edge: `root`, `confidence`, `evidence[]`, `how`, ## Mail pipeline (done) -1. `bin/mail/sync.go` (Go, 8 workers) — paginated Gmail/OnlyOffice download. - Gmail attachments key off `body.attachmentId`, not MIME `partId`. +1. `bin/mail/sync.go` (Go, 8 workers) — paginated Gmail / OnlyOffice / M365 + Graph download. Gmail attachments key off `body.attachmentId`, not MIME + `partId`. M365 uses client-credentials + delta link (commit after success). 2. `bin/mail/import.go --from-raw` — message.json → message.md; PDFs via `pdftotext -layout` (~15ms); textless/scanned PDFs `pdftoppm` + tesseract `eng+deu`. ICS sidecars @@ -151,7 +152,11 @@ Common props on every node/edge: `root`, `confidence`, `evidence[]`, `how`, corrupts its WAL on bulk-insert into an already-indexed DB. Conversion and indexing stay separate for crash safety. `bin/mail/index_mail` is a deprecation shim. -4. Result: 17,835 messages → 28,918 info leafs, FTS + HNSW healthy, searchable +4. Compose `mail-sync` / `bin/stack/start-mail-sync` — ETL loop (default + `onlyoffice,gmail`, 300s): sync → import on `new>0`; full rebuild only if + `MAIL_SYNC_INDEX=1`. Bot digests (ai-bot) and case wrappers reuse sync/OAuth; + they do not replace the corpus path. +5. Result: 17,835 messages → 28,918 info leafs, FTS + HNSW healthy, searchable via `bin/brain/search.go`. ## CI/CD pipeline (D15) diff --git a/README.md b/README.md index c7b55b9..b95d146 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,8 @@ Mail is a first-class corpus (retrievable through the same search): ```bash bin/mail/sync.go --source onlyoffice,gmail --workers 8 --out var/mail # raw sync (Go) +bin/mail/sync.go --source m365 --env ~/.config/brain/mail.env # Microsoft 365 Graph +bin/stack/start-mail-sync # compose ETL (300s; no auto-rebuild) bin/mail/import.go --from-raw var/mail # JSON → markdown bin/brain/add.go --text T --root facts --source "a.md x b.md" bin/brain/index.go --rebuild --with-facts --with-chats # facts extract + chats md diff --git a/bin/docker-entrypoint b/bin/docker-entrypoint index 9cdacb4..8041408 100755 --- a/bin/docker-entrypoint +++ b/bin/docker-entrypoint @@ -5,7 +5,7 @@ # serve | search | watch # Index image (Python write path, compose profile `index`): # index | extract | audit | search (deprecated python wrapper) -# mail-sync [N] pull loop: sync -> import -> index every N s (default 10) +# mail-sync [N] ETL loop: sync -> import; optional index (default 300s) # # Usage comment starts at line 2 (self-describing convention). set -euo pipefail @@ -36,20 +36,23 @@ case "$CMD" in extract) exec "$KB_PY" /app/bin/facts/extract "$@" ;; audit) exec "$KB_PY" /app/bin/facts/audit "$@" ;; mail-sync) - # loop: pull mail (M365/OnlyOffice/Gmail) every N s, convert to md, - # rebuild the brain index. Index only when something new arrived. - interval="${1:-10}" - [ "$interval" -gt 0 ] 2>/dev/null || interval=10 - : "${MAIL_SYNC_ENV:=/secret/m365.env}" - : "${MAIL_SYNC_SRC:=m365}" + # ETL: pull mail, convert to md when new>0. Full --rebuild is opt-in + # (MAIL_SYNC_INDEX=1) — ~29k leaf rebuild is minutes, not a 10s loop. + interval="${1:-300}" + [ "$interval" -gt 0 ] 2>/dev/null || interval=300 + : "${MAIL_SYNC_ENV:=/secret/mail.env}" + : "${MAIL_SYNC_SRC:=onlyoffice,gmail}" : "${MAIL_SYNC_OUT:=/app/var/mail}" + : "${MAIL_SYNC_INDEX:=0}" while true; do - out="$("/app/bin/mail-sync" --source "$MAIL_SYNC_SRC" --env "$MAIL_SYNC_ENV" --out "$MAIL_SYNC_OUT" 2>&1)" + out="$("/app/bin/mail-sync" --source "$MAIL_SYNC_SRC" --env "$MAIL_SYNC_ENV" --out "$MAIL_SYNC_OUT" 2>&1)" || true echo "$out" new="$(printf '%s\n' "$out" | sed -n 's/.*new=\([0-9]*\).*/\1/p' | tail -1)" if [ -n "$new" ] && [ "$new" -gt 0 ] 2>/dev/null; then "$KB_PY" /app/bin/mail/import --from-raw "$MAIL_SYNC_OUT" 2>&1 | tail -1 - "$KB_PY" /app/bin/kb/index --rebuild --with-mail 2>&1 | tail -1 + if [ "$MAIL_SYNC_INDEX" = "1" ]; then + "$KB_PY" /app/bin/kb/index --rebuild --with-mail 2>&1 | tail -1 + fi fi sleep "$interval" done diff --git a/bin/stack/lib.sh b/bin/stack/lib.sh index 8db3a52..0a761f2 100644 --- a/bin/stack/lib.sh +++ b/bin/stack/lib.sh @@ -143,12 +143,17 @@ ensure_picoclaw() { wait_health "$PICOCLAW_URL/health" || stack_die "picoclaw health failed at $PICOCLAW_URL/health" } +mail_sync_running() { + compose ps --status running --services 2>/dev/null | grep -qx mail-sync +} + stack_status() { - local bh=down mcp=down ph=down present=false + local bh=down mcp=down ph=down present=false ms=down health_ok "$BRAIN_URL/health" && bh=ok mcp_ok && mcp=ok reasoner_has_model && present=true health_ok "$PICOCLAW_URL/health" && ph=ok + mail_sync_running && ms=ok cat <&2 + compose up -d mail-sync +} + stack_attach_agent() { local opts=() if [[ -t 0 && -t 1 ]]; then @@ -230,6 +243,6 @@ stack_stop() { return 0 ;; esac - echo "stack: stop brain brain-mcp reasoner picoclaw (volumes kept)" >&2 - compose --profile picoclaw --profile reasoner stop picoclaw brain-mcp reasoner brain + echo "stack: stop brain brain-mcp reasoner picoclaw mail-sync (volumes kept)" >&2 + compose --profile picoclaw --profile reasoner stop picoclaw brain-mcp reasoner brain mail-sync } diff --git a/bin/stack/start-mail-sync b/bin/stack/start-mail-sync new file mode 100755 index 0000000..d39e34a --- /dev/null +++ b/bin/stack/start-mail-sync @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# bin/stack/start-mail-sync - compose up mail-sync ETL (sync → import; optional index). +# +# bin/stack/start-mail-sync +# +# Default: onlyoffice,gmail every 300s into kb-var. Full --rebuild only if +# MAIL_SYNC_INDEX=1 in compose/env. Secrets: ~/.config/brain/mail.env + +# ~/.gmail-mcp (mounted). Does not start brain/picoclaw. +set -euo pipefail + +STACK_DIR="$(CDPATH= cd -- "$(dirname "$0")" && pwd)" +# shellcheck source=lib.sh +source "$STACK_DIR/lib.sh" +case "${1:-}" in +-h | --help) + stack_usage "$0" + exit 0 + ;; +esac +stack_start_mail_sync "$@" +stack_status diff --git a/bin/stack/stop b/bin/stack/stop index 880c419..70fc77f 100755 --- a/bin/stack/stop +++ b/bin/stack/stop @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# bin/stack/stop - stop compose brain / brain-mcp / reasoner / picoclaw. +# bin/stack/stop - stop compose brain / brain-mcp / reasoner / picoclaw / mail-sync. # # bin/stack/stop # diff --git a/bin/tools/test_stack.py b/bin/tools/test_stack.py index 6ad25ba..f84b0eb 100644 --- a/bin/tools/test_stack.py +++ b/bin/tools/test_stack.py @@ -9,7 +9,7 @@ import unittest from pathlib import Path ROOT = Path(__file__).resolve().parents[2] -METHODS = ("start", "start-assistant", "stop", "status") +METHODS = ("start", "start-assistant", "start-mail-sync", "stop", "status") class StackLayoutTest(unittest.TestCase): @@ -32,8 +32,10 @@ class StackLayoutTest(unittest.TestCase): lib = (ROOT / "bin" / "stack" / "lib.sh").read_text() self.assertIn("stack_start", lib) self.assertIn("stack_start_assistant", lib) + self.assertIn("stack_start_mail_sync", lib) self.assertIn("stack_stop", lib) self.assertIn("stack_status", lib) + self.assertIn("mail-sync", lib) self.assertIn("qwen3.5:9b", lib) self.assertIn("picoclaw agent", lib) self.assertIn("--no-deps", lib) @@ -49,6 +51,16 @@ class StackLayoutTest(unittest.TestCase): self.assertNotIn("stack_start_assistant", start) self.assertNotIn("picoclaw agent", start) + def test_start_mail_sync_is_etl_only(self) -> None: + src = (ROOT / "bin" / "stack" / "start-mail-sync").read_text() + self.assertIn("stack_start_mail_sync", src) + self.assertNotIn("stack_start_assistant", src) + ep = (ROOT / "bin" / "docker-entrypoint").read_text() + self.assertIn('MAIL_SYNC_SRC:=onlyoffice,gmail', ep) + self.assertIn('MAIL_SYNC_INDEX:=0', ep) + self.assertIn('interval="${1:-300}"', ep) + self.assertIn('MAIL_SYNC_INDEX" = "1"', ep) + def test_start_assistant_attaches_agent(self) -> None: src = (ROOT / "bin" / "stack" / "start-assistant").read_text() self.assertIn("stack_start_assistant", src) @@ -151,7 +163,10 @@ exit 0 check=False, ) self.assertEqual(r.returncode, 0, r.stderr) - self.assertFalse(log.exists(), "healthy brain must not docker compose up") + if log.exists(): + logged = log.read_text() + self.assertNotIn("up -d", logged, "healthy brain must not docker compose up") + self.assertIn("ps", logged) # status probes mail-sync via compose ps def test_start_ups_brain_when_unhealthy(self) -> None: with tempfile.TemporaryDirectory() as raw: @@ -190,7 +205,7 @@ exit 0 logged = log.read_text() self.assertIn("stop", logged) self.assertNotIn(" down", logged) - for svc in ("brain", "brain-mcp", "reasoner", "picoclaw"): + for svc in ("brain", "brain-mcp", "reasoner", "picoclaw", "mail-sync"): self.assertIn(svc, logged) def test_start_assistant_no_attach_starts_picoclaw(self) -> None: diff --git a/compose.yaml b/compose.yaml index 434376f..d1d6c5a 100644 --- a/compose.yaml +++ b/compose.yaml @@ -92,11 +92,13 @@ services: tmpfs: - /tmp - # mail-sync: pull M365 (and OnlyOffice/Gmail) mail every 10 s into the shared - # var volume, then rebuild the brain index. Runs on the index image so it can - # convert + index in-process. Needs ~/.config/brain/m365.env for the M365 - # source; override MAIL_SYNC_SRC / MAIL_SYNC_ENV for other providers. + # mail-sync ETL: sync → import every 300s into shared var. Full brain rebuild + # only when MAIL_SYNC_INDEX=1 (expensive on large mail corpora). Default + # sources: onlyoffice,gmail. For M365 set MAIL_SYNC_SRC=m365 and put + # M365_TENANT/CLIENT_ID/CLIENT_SECRET/USERS in ~/.config/brain/mail.env + # (or m365.env + MAIL_SYNC_ENV). Gmail OAuth: mount ~/.gmail-mcp. # docker compose up -d mail-sync + # bin/stack/start-mail-sync mail-sync: image: ghcr.io/eslider/2dph:index build: @@ -106,13 +108,16 @@ services: environment: HF_HOME: /data/hf KB_PY: python3 - MAIL_SYNC_SRC: m365 - MAIL_SYNC_ENV: /secret/m365.env + MAIL_SYNC_SRC: onlyoffice,gmail + MAIL_SYNC_ENV: /secret/mail.env + MAIL_SYNC_INDEX: "0" + HOME: /home/2dph volumes: - kb-model:/data/hf - kb-var:/app/var - ~/.config/brain:/secret:ro - command: ["mail-sync", "10"] + - ~/.gmail-mcp:/home/2dph/.gmail-mcp:ro + command: ["mail-sync", "300"] read_only: true tmpfs: - /tmp diff --git a/docs/runbook.md b/docs/runbook.md index 8bf1d11..6a8f045 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -71,7 +71,8 @@ delete `var/kb.lbug` then `--rebuild`. ```bash bin/stack/start # brain :8630, wait until MCP search/get/audit -bin/stack/status # YAML: brain / reasoner / picoclaw +bin/stack/status # YAML: brain / reasoner / picoclaw / mail_sync +bin/stack/start-mail-sync # compose ETL: OO+Gmail sync→import (300s; no auto-rebuild) bin/stack/start-assistant # + qwen3.5:9b + PicoClaw agent (ask the brain) bin/stack/start-assistant --no-attach bin/stack/stop # compose stop; volumes kept @@ -81,6 +82,7 @@ Same Compose services by hand: ```bash docker compose up -d brain # :8630 Zig CGO serve +docker compose up -d mail-sync # ETL loop into kb-var docker compose --profile index run --rm index # rebuild docker compose --profile picoclaw up brain-mcp # MCP 127.0.0.1:8630 ```