AGENTS.md

@@ -66,12 +66,20 @@ var/          kb.lbug, var/mail/*, caches (gitignored)
 ```bash
 bin/mail/sync.go --source onlyoffice,gmail --workers 8 --out var/mail   # raw message.json + attachments
 bin/mail/sync.go --source gmail --query 'from:example.com' --out var/mail  # Gmail search (default in:inbox)
+bin/mail/sync.go --source m365 --env ~/.config/brain/mail.env --out var/mail  # Microsoft Graph (delta)
 bin/mail/import.go --from-raw var/mail                                  # message.json → message.md (convert only)
 bin/brain/index.go --rebuild --with-facts --with-chats
+bin/stack/start-mail-sync                                               # compose ETL: sync→import every 300s
 ```
 
 - `sync` (Go) downloads messages + attachments; Gmail uses paginated list +
-  `body.attachmentId` (not partId) for attachments.
+  `body.attachmentId` (not partId) for attachments. Sources: `onlyoffice`,
+  `gmail`, `m365` (client-credentials + delta link; commit after success).
+- Compose `mail-sync` / `bin/stack/start-mail-sync`: ETL loop (default
+  `onlyoffice,gmail`, 300s). On `new>0` runs import; full `--rebuild` only if
+  `MAIL_SYNC_INDEX=1`. Secrets: `~/.config/brain/mail.env` + `~/.gmail-mcp`.
+  Case wrappers (e.g. family `gmail-sync-la-quinta.sh`) and ai-bot
+  `gmail-reauth.sh` reuse this sync/OAuth — do not fork corpus download.
 - `import` converts body + attachments to markdown. PDFs use poppler
   `pdftotext -layout` fast path (~15ms); textless/scanned PDFs use
   `pdftoppm` + tesseract `eng+deu` (`bin/mail/ocr.go`). Optional

Dockerfile

@@ -6,6 +6,15 @@
 # API: Go + ladybug via Zig CGO (no CPython).
 # Index: Python write path (profile `index` until brain/add is v2).
 
+# --- mail-sync: standalone M365/OnlyOffice/Gmail puller (pure Go, no CGO) ---
+FROM golang:1.26-bookworm AS mail-build
+WORKDIR /src
+COPY go.mod go.sum ./
+RUN go mod download
+COPY bin/mail ./bin/mail
+COPY internal ./internal
+RUN CGO_ENABLED=0 go build -o /mail-sync ./bin/mail/sync.go
+
 # --- Python sidecar (Ladybug write / rebuild) ---
 FROM python:3.12-slim AS index
 
@@ -38,14 +47,6 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
     CMD python -c "import model2vec, ladybug, mistune; print('ok')" || exit 1
 ENTRYPOINT ["/app/bin/docker-entrypoint"]
 
-# --- mail-sync: standalone M365/OnlyOffice/Gmail puller (pure Go, no CGO) ---
-FROM golang:1.26-bookworm AS mail-build
-WORKDIR /src
-COPY go.mod go.sum ./
-RUN go mod download
-COPY bin/mail ./bin/mail
-RUN CGO_ENABLED=0 go build -o /mail-sync ./bin/mail/sync.go
-
 # --- Go API: CGO with Zig, not gcc ---
 FROM golang:1.26-bookworm AS api-build
 WORKDIR /src

PLAN.md

@@ -86,7 +86,7 @@ detective method: **a fact needs ≥2 independent sources or it is
     mail/ocr.go             tesseract eng+deu (pdftoppm scans)
     md/import               (deprecated; bin/markdown/import.go)
     brain/extract  brain/audit       brain/deduce    (thinking wrapper)
-    stack/start start-assistant stop status
+    stack/start start-assistant start-mail-sync stop status
     web/search               (deprecated shim → web/search.go)
     db/psql-yq               (vendored)
     ssh-tunnel               onlyoffice pg tunnel 5433
@@ -141,8 +141,9 @@ Common props on every node/edge: `root`, `confidence`, `evidence[]`, `how`,
 
 ## Mail pipeline (done)
 
-1. `bin/mail/sync.go` (Go, 8 workers) — paginated Gmail/OnlyOffice download.
+1. `bin/mail/sync.go` (Go, 8 workers) — paginated Gmail / OnlyOffice / M365
-   Gmail attachments key off `body.attachmentId`, not MIME `partId`.
+   Graph download. Gmail attachments key off `body.attachmentId`, not MIME
+   `partId`. M365 uses client-credentials + delta link (commit after success).
 2. `bin/mail/import.go --from-raw` — message.json → message.md; PDFs via
    `pdftotext -layout` (~15ms); textless/scanned PDFs `pdftoppm` + tesseract
    `eng+deu`. ICS sidecars
@@ -151,7 +152,11 @@ Common props on every node/edge: `root`, `confidence`, `evidence[]`, `how`,
    corrupts its WAL on bulk-insert into an already-indexed DB. Conversion and
    indexing stay separate for crash safety. `bin/mail/index_mail` is a
    deprecation shim.
-4. Result: 17,835 messages → 28,918 info leafs, FTS + HNSW healthy, searchable
+4. Compose `mail-sync` / `bin/stack/start-mail-sync` — ETL loop (default
+   `onlyoffice,gmail`, 300s): sync → import on `new>0`; full rebuild only if
+   `MAIL_SYNC_INDEX=1`. Bot digests (ai-bot) and case wrappers reuse sync/OAuth;
+   they do not replace the corpus path.
+5. Result: 17,835 messages → 28,918 info leafs, FTS + HNSW healthy, searchable
    via `bin/brain/search.go`.
 
 ## CI/CD pipeline (D15)

README.md

@@ -119,6 +119,8 @@ Mail is a first-class corpus (retrievable through the same search):
 
 ```bash
 bin/mail/sync.go --source onlyoffice,gmail --workers 8 --out var/mail  # raw sync (Go)
+bin/mail/sync.go --source m365 --env ~/.config/brain/mail.env          # Microsoft 365 Graph
+bin/stack/start-mail-sync                                              # compose ETL (300s; no auto-rebuild)
 bin/mail/import.go --from-raw var/mail                                  # JSON → markdown
 bin/brain/add.go --text T --root facts --source "a.md x b.md"
 bin/brain/index.go --rebuild --with-facts --with-chats                  # facts extract + chats md

bin/docker-entrypoint

@@ -5,7 +5,7 @@
 #   serve | search | watch
 # Index image (Python write path, compose profile `index`):
 #   index | extract | audit | search (deprecated python wrapper)
-#   mail-sync [N]   pull loop: sync -> import -> index every N s (default 10)
+#   mail-sync [N]   ETL loop: sync -> import; optional index (default 300s)
 #
 # Usage comment starts at line 2 (self-describing convention).
 set -euo pipefail
@@ -36,20 +36,23 @@ case "$CMD" in
 	extract) exec "$KB_PY" /app/bin/facts/extract "$@" ;;
 	audit) exec "$KB_PY" /app/bin/facts/audit "$@" ;;
 	mail-sync)
-		# loop: pull mail (M365/OnlyOffice/Gmail) every N s, convert to md,
+		# ETL: pull mail, convert to md when new>0. Full --rebuild is opt-in
-		# rebuild the brain index. Index only when something new arrived.
+		# (MAIL_SYNC_INDEX=1) — ~29k leaf rebuild is minutes, not a 10s loop.
-		interval="${1:-10}"
+		interval="${1:-300}"
-		[ "$interval" -gt 0 ] 2>/dev/null || interval=10
+		[ "$interval" -gt 0 ] 2>/dev/null || interval=300
-		: "${MAIL_SYNC_ENV:=/secret/m365.env}"
+		: "${MAIL_SYNC_ENV:=/secret/mail.env}"
-		: "${MAIL_SYNC_SRC:=m365}"
+		: "${MAIL_SYNC_SRC:=onlyoffice,gmail}"
 		: "${MAIL_SYNC_OUT:=/app/var/mail}"
+		: "${MAIL_SYNC_INDEX:=0}"
 		while true; do
-			out="$("/app/bin/mail-sync" --source "$MAIL_SYNC_SRC" --env "$MAIL_SYNC_ENV" --out "$MAIL_SYNC_OUT" 2>&1)"
+			out="$("/app/bin/mail-sync" --source "$MAIL_SYNC_SRC" --env "$MAIL_SYNC_ENV" --out "$MAIL_SYNC_OUT" 2>&1)" || true
 			echo "$out"
 			new="$(printf '%s\n' "$out" | sed -n 's/.*new=\([0-9]*\).*/\1/p' | tail -1)"
 			if [ -n "$new" ] && [ "$new" -gt 0 ] 2>/dev/null; then
 				"$KB_PY" /app/bin/mail/import --from-raw "$MAIL_SYNC_OUT" 2>&1 | tail -1
-				"$KB_PY" /app/bin/kb/index --rebuild --with-mail 2>&1 | tail -1
+				if [ "$MAIL_SYNC_INDEX" = "1" ]; then
+					"$KB_PY" /app/bin/kb/index --rebuild --with-mail 2>&1 | tail -1
+				fi
 			fi
 			sleep "$interval"
 		done

bin/stack/lib.sh

@@ -143,12 +143,17 @@ ensure_picoclaw() {
 	wait_health "$PICOCLAW_URL/health" || stack_die "picoclaw health failed at $PICOCLAW_URL/health"
 }
 
+mail_sync_running() {
+	compose ps --status running --services 2>/dev/null | grep -qx mail-sync
+}
+
 stack_status() {
-	local bh=down mcp=down ph=down present=false
+	local bh=down mcp=down ph=down present=false ms=down
 	health_ok "$BRAIN_URL/health" && bh=ok
 	mcp_ok && mcp=ok
 	reasoner_has_model && present=true
 	health_ok "$PICOCLAW_URL/health" && ph=ok
+	mail_sync_running && ms=ok
 	cat <<EOF
 brain:
   url: $BRAIN_URL
@@ -161,6 +166,9 @@ reasoner:
 picoclaw:
   url: $PICOCLAW_URL
   health: $ph
+mail_sync:
+  service: mail-sync
+  running: $ms
 EOF
 }
 
@@ -168,6 +176,11 @@ stack_start() {
 	ensure_brain
 }
 
+stack_start_mail_sync() {
+	echo "mail-sync: compose up (ETL sync→import; index only if MAIL_SYNC_INDEX=1)" >&2
+	compose up -d mail-sync
+}
+
 stack_attach_agent() {
 	local opts=()
 	if [[ -t 0 && -t 1 ]]; then
@@ -230,6 +243,6 @@ stack_stop() {
 		return 0
 		;;
 	esac
-	echo "stack: stop brain brain-mcp reasoner picoclaw (volumes kept)" >&2
+	echo "stack: stop brain brain-mcp reasoner picoclaw mail-sync (volumes kept)" >&2
-	compose --profile picoclaw --profile reasoner stop picoclaw brain-mcp reasoner brain
+	compose --profile picoclaw --profile reasoner stop picoclaw brain-mcp reasoner brain mail-sync
 }

bin/stack/start-mail-sync

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# bin/stack/start-mail-sync - compose up mail-sync ETL (sync → import; optional index).
+#
+#   bin/stack/start-mail-sync
+#
+# Default: onlyoffice,gmail every 300s into kb-var. Full --rebuild only if
+# MAIL_SYNC_INDEX=1 in compose/env. Secrets: ~/.config/brain/mail.env +
+# ~/.gmail-mcp (mounted). Does not start brain/picoclaw.
+set -euo pipefail
+
+STACK_DIR="$(CDPATH= cd -- "$(dirname "$0")" && pwd)"
+# shellcheck source=lib.sh
+source "$STACK_DIR/lib.sh"
+case "${1:-}" in
+-h | --help)
+	stack_usage "$0"
+	exit 0
+	;;
+esac
+stack_start_mail_sync "$@"
+stack_status

bin/stack/stop

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# bin/stack/stop - stop compose brain / brain-mcp / reasoner / picoclaw.
+# bin/stack/stop - stop compose brain / brain-mcp / reasoner / picoclaw / mail-sync.
 #
 #   bin/stack/stop
 #

bin/tools/test_stack.py

@@ -9,7 +9,7 @@ import unittest
 from pathlib import Path
 
 ROOT = Path(__file__).resolve().parents[2]
-METHODS = ("start", "start-assistant", "stop", "status")
+METHODS = ("start", "start-assistant", "start-mail-sync", "stop", "status")
 
 
 class StackLayoutTest(unittest.TestCase):
@@ -32,8 +32,10 @@ class StackLayoutTest(unittest.TestCase):
         lib = (ROOT / "bin" / "stack" / "lib.sh").read_text()
         self.assertIn("stack_start", lib)
         self.assertIn("stack_start_assistant", lib)
+        self.assertIn("stack_start_mail_sync", lib)
         self.assertIn("stack_stop", lib)
         self.assertIn("stack_status", lib)
+        self.assertIn("mail-sync", lib)
         self.assertIn("qwen3.5:9b", lib)
         self.assertIn("picoclaw agent", lib)
         self.assertIn("--no-deps", lib)
@@ -49,6 +51,16 @@ class StackLayoutTest(unittest.TestCase):
         self.assertNotIn("stack_start_assistant", start)
         self.assertNotIn("picoclaw agent", start)
 
+    def test_start_mail_sync_is_etl_only(self) -> None:
+        src = (ROOT / "bin" / "stack" / "start-mail-sync").read_text()
+        self.assertIn("stack_start_mail_sync", src)
+        self.assertNotIn("stack_start_assistant", src)
+        ep = (ROOT / "bin" / "docker-entrypoint").read_text()
+        self.assertIn('MAIL_SYNC_SRC:=onlyoffice,gmail', ep)
+        self.assertIn('MAIL_SYNC_INDEX:=0', ep)
+        self.assertIn('interval="${1:-300}"', ep)
+        self.assertIn('MAIL_SYNC_INDEX" = "1"', ep)
+
     def test_start_assistant_attaches_agent(self) -> None:
         src = (ROOT / "bin" / "stack" / "start-assistant").read_text()
         self.assertIn("stack_start_assistant", src)
@@ -151,7 +163,10 @@ exit 0
                 check=False,
             )
             self.assertEqual(r.returncode, 0, r.stderr)
-            self.assertFalse(log.exists(), "healthy brain must not docker compose up")
+            if log.exists():
+                logged = log.read_text()
+                self.assertNotIn("up -d", logged, "healthy brain must not docker compose up")
+                self.assertIn("ps", logged)  # status probes mail-sync via compose ps
 
     def test_start_ups_brain_when_unhealthy(self) -> None:
         with tempfile.TemporaryDirectory() as raw:
@@ -190,7 +205,7 @@ exit 0
             logged = log.read_text()
             self.assertIn("stop", logged)
             self.assertNotIn(" down", logged)
-            for svc in ("brain", "brain-mcp", "reasoner", "picoclaw"):
+            for svc in ("brain", "brain-mcp", "reasoner", "picoclaw", "mail-sync"):
                 self.assertIn(svc, logged)
 
     def test_start_assistant_no_attach_starts_picoclaw(self) -> None:

compose.yaml

@@ -92,11 +92,13 @@ services:
     tmpfs:
       - /tmp
 
-  # mail-sync: pull M365 (and OnlyOffice/Gmail) mail every 10 s into the shared
+  # mail-sync ETL: sync → import every 300s into shared var. Full brain rebuild
-  # var volume, then rebuild the brain index. Runs on the index image so it can
+  # only when MAIL_SYNC_INDEX=1 (expensive on large mail corpora). Default
-  # convert + index in-process. Needs ~/.config/brain/m365.env for the M365
+  # sources: onlyoffice,gmail. For M365 set MAIL_SYNC_SRC=m365 and put
-  # source; override MAIL_SYNC_SRC / MAIL_SYNC_ENV for other providers.
+  # M365_TENANT/CLIENT_ID/CLIENT_SECRET/USERS in ~/.config/brain/mail.env
+  # (or m365.env + MAIL_SYNC_ENV). Gmail OAuth: mount ~/.gmail-mcp.
   #   docker compose up -d mail-sync
+  #   bin/stack/start-mail-sync
   mail-sync:
     image: ghcr.io/eslider/2dph:index
     build:
@@ -106,13 +108,16 @@ services:
     environment:
       HF_HOME: /data/hf
       KB_PY: python3
-      MAIL_SYNC_SRC: m365
+      MAIL_SYNC_SRC: onlyoffice,gmail
-      MAIL_SYNC_ENV: /secret/m365.env
+      MAIL_SYNC_ENV: /secret/mail.env
+      MAIL_SYNC_INDEX: "0"
+      HOME: /home/2dph
     volumes:
       - kb-model:/data/hf
       - kb-var:/app/var
       - ~/.config/brain:/secret:ro
-    command: ["mail-sync", "10"]
+      - ~/.gmail-mcp:/home/2dph/.gmail-mcp:ro
+    command: ["mail-sync", "300"]
     read_only: true
     tmpfs:
       - /tmp

docs/runbook.md

@@ -71,7 +71,8 @@ delete `var/kb.lbug` then `--rebuild`.
 
 ```bash
 bin/stack/start                 # brain :8630, wait until MCP search/get/audit
-bin/stack/status                # YAML: brain / reasoner / picoclaw
+bin/stack/status                # YAML: brain / reasoner / picoclaw / mail_sync
+bin/stack/start-mail-sync       # compose ETL: OO+Gmail sync→import (300s; no auto-rebuild)
 bin/stack/start-assistant       # + qwen3.5:9b + PicoClaw agent (ask the brain)
 bin/stack/start-assistant --no-attach
 bin/stack/stop                  # compose stop; volumes kept
@@ -81,6 +82,7 @@ Same Compose services by hand:
 
 ```bash
 docker compose up -d brain                       # :8630 Zig CGO serve
+docker compose up -d mail-sync                   # ETL loop into kb-var
 docker compose --profile index run --rm index    # rebuild
 docker compose --profile picoclaw up brain-mcp   # MCP 127.0.0.1:8630
 ```