feat(mail): adapt M365 ETL loop for OO+Gmail bots.
Tests / Test (push) Skipped
Tests / OCR (tesseract fixture) (push) Skipped
Tests / Release (semver) (push) Skipped
Tests / Test (pull_request) Failing after 6s
Tests / OCR (tesseract fixture) (pull_request) Failing after 4s
Tests / Release (semver) (pull_request) Skipped

Default compose mail-sync to onlyoffice,gmail every 300s with import on new
mail; gate full --rebuild behind MAIL_SYNC_INDEX. Fix mail-build stage order
and add bin/stack/start-mail-sync.
This commit is contained in:
2026-08-14 17:59:02 +01:00
parent 5ccca2fac9
commit b612d3cc97
11 changed files with 112 additions and 37 deletions
+12 -9
View File
@@ -5,7 +5,7 @@
# serve | search | watch
# Index image (Python write path, compose profile `index`):
# index | extract | audit | search (deprecated python wrapper)
# mail-sync [N] pull loop: sync -> import -> index every N s (default 10)
# mail-sync [N] ETL loop: sync -> import; optional index (default 300s)
#
# Usage comment starts at line 2 (self-describing convention).
set -euo pipefail
@@ -36,20 +36,23 @@ case "$CMD" in
extract) exec "$KB_PY" /app/bin/facts/extract "$@" ;;
audit) exec "$KB_PY" /app/bin/facts/audit "$@" ;;
mail-sync)
# loop: pull mail (M365/OnlyOffice/Gmail) every N s, convert to md,
# rebuild the brain index. Index only when something new arrived.
interval="${1:-10}"
[ "$interval" -gt 0 ] 2>/dev/null || interval=10
: "${MAIL_SYNC_ENV:=/secret/m365.env}"
: "${MAIL_SYNC_SRC:=m365}"
# ETL: pull mail, convert to md when new>0. Full --rebuild is opt-in
# (MAIL_SYNC_INDEX=1) — ~29k leaf rebuild is minutes, not a 10s loop.
interval="${1:-300}"
[ "$interval" -gt 0 ] 2>/dev/null || interval=300
: "${MAIL_SYNC_ENV:=/secret/mail.env}"
: "${MAIL_SYNC_SRC:=onlyoffice,gmail}"
: "${MAIL_SYNC_OUT:=/app/var/mail}"
: "${MAIL_SYNC_INDEX:=0}"
while true; do
out="$("/app/bin/mail-sync" --source "$MAIL_SYNC_SRC" --env "$MAIL_SYNC_ENV" --out "$MAIL_SYNC_OUT" 2>&1)"
out="$("/app/bin/mail-sync" --source "$MAIL_SYNC_SRC" --env "$MAIL_SYNC_ENV" --out "$MAIL_SYNC_OUT" 2>&1)" || true
echo "$out"
new="$(printf '%s\n' "$out" | sed -n 's/.*new=\([0-9]*\).*/\1/p' | tail -1)"
if [ -n "$new" ] && [ "$new" -gt 0 ] 2>/dev/null; then
"$KB_PY" /app/bin/mail/import --from-raw "$MAIL_SYNC_OUT" 2>&1 | tail -1
"$KB_PY" /app/bin/kb/index --rebuild --with-mail 2>&1 | tail -1
if [ "$MAIL_SYNC_INDEX" = "1" ]; then
"$KB_PY" /app/bin/kb/index --rebuild --with-mail 2>&1 | tail -1
fi
fi
sleep "$interval"
done
+16 -3
View File
@@ -143,12 +143,17 @@ ensure_picoclaw() {
wait_health "$PICOCLAW_URL/health" || stack_die "picoclaw health failed at $PICOCLAW_URL/health"
}
mail_sync_running() {
compose ps --status running --services 2>/dev/null | grep -qx mail-sync
}
stack_status() {
local bh=down mcp=down ph=down present=false
local bh=down mcp=down ph=down present=false ms=down
health_ok "$BRAIN_URL/health" && bh=ok
mcp_ok && mcp=ok
reasoner_has_model && present=true
health_ok "$PICOCLAW_URL/health" && ph=ok
mail_sync_running && ms=ok
cat <<EOF
brain:
url: $BRAIN_URL
@@ -161,6 +166,9 @@ reasoner:
picoclaw:
url: $PICOCLAW_URL
health: $ph
mail_sync:
service: mail-sync
running: $ms
EOF
}
@@ -168,6 +176,11 @@ stack_start() {
ensure_brain
}
stack_start_mail_sync() {
echo "mail-sync: compose up (ETL sync→import; index only if MAIL_SYNC_INDEX=1)" >&2
compose up -d mail-sync
}
stack_attach_agent() {
local opts=()
if [[ -t 0 && -t 1 ]]; then
@@ -230,6 +243,6 @@ stack_stop() {
return 0
;;
esac
echo "stack: stop brain brain-mcp reasoner picoclaw (volumes kept)" >&2
compose --profile picoclaw --profile reasoner stop picoclaw brain-mcp reasoner brain
echo "stack: stop brain brain-mcp reasoner picoclaw mail-sync (volumes kept)" >&2
compose --profile picoclaw --profile reasoner stop picoclaw brain-mcp reasoner brain mail-sync
}
+21
View File
@@ -0,0 +1,21 @@
#!/usr/bin/env bash
# bin/stack/start-mail-sync - compose up mail-sync ETL (sync → import; optional index).
#
# bin/stack/start-mail-sync
#
# Default: onlyoffice,gmail every 300s into kb-var. Full --rebuild only if
# MAIL_SYNC_INDEX=1 in compose/env. Secrets: ~/.config/brain/mail.env +
# ~/.gmail-mcp (mounted). Does not start brain/picoclaw.
set -euo pipefail
STACK_DIR="$(CDPATH= cd -- "$(dirname "$0")" && pwd)"
# shellcheck source=lib.sh
source "$STACK_DIR/lib.sh"
case "${1:-}" in
-h | --help)
stack_usage "$0"
exit 0
;;
esac
stack_start_mail_sync "$@"
stack_status
+1 -1
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
# bin/stack/stop - stop compose brain / brain-mcp / reasoner / picoclaw.
# bin/stack/stop - stop compose brain / brain-mcp / reasoner / picoclaw / mail-sync.
#
# bin/stack/stop
#
+18 -3
View File
@@ -9,7 +9,7 @@ import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
METHODS = ("start", "start-assistant", "stop", "status")
METHODS = ("start", "start-assistant", "start-mail-sync", "stop", "status")
class StackLayoutTest(unittest.TestCase):
@@ -32,8 +32,10 @@ class StackLayoutTest(unittest.TestCase):
lib = (ROOT / "bin" / "stack" / "lib.sh").read_text()
self.assertIn("stack_start", lib)
self.assertIn("stack_start_assistant", lib)
self.assertIn("stack_start_mail_sync", lib)
self.assertIn("stack_stop", lib)
self.assertIn("stack_status", lib)
self.assertIn("mail-sync", lib)
self.assertIn("qwen3.5:9b", lib)
self.assertIn("picoclaw agent", lib)
self.assertIn("--no-deps", lib)
@@ -49,6 +51,16 @@ class StackLayoutTest(unittest.TestCase):
self.assertNotIn("stack_start_assistant", start)
self.assertNotIn("picoclaw agent", start)
def test_start_mail_sync_is_etl_only(self) -> None:
src = (ROOT / "bin" / "stack" / "start-mail-sync").read_text()
self.assertIn("stack_start_mail_sync", src)
self.assertNotIn("stack_start_assistant", src)
ep = (ROOT / "bin" / "docker-entrypoint").read_text()
self.assertIn('MAIL_SYNC_SRC:=onlyoffice,gmail', ep)
self.assertIn('MAIL_SYNC_INDEX:=0', ep)
self.assertIn('interval="${1:-300}"', ep)
self.assertIn('MAIL_SYNC_INDEX" = "1"', ep)
def test_start_assistant_attaches_agent(self) -> None:
src = (ROOT / "bin" / "stack" / "start-assistant").read_text()
self.assertIn("stack_start_assistant", src)
@@ -151,7 +163,10 @@ exit 0
check=False,
)
self.assertEqual(r.returncode, 0, r.stderr)
self.assertFalse(log.exists(), "healthy brain must not docker compose up")
if log.exists():
logged = log.read_text()
self.assertNotIn("up -d", logged, "healthy brain must not docker compose up")
self.assertIn("ps", logged) # status probes mail-sync via compose ps
def test_start_ups_brain_when_unhealthy(self) -> None:
with tempfile.TemporaryDirectory() as raw:
@@ -190,7 +205,7 @@ exit 0
logged = log.read_text()
self.assertIn("stop", logged)
self.assertNotIn(" down", logged)
for svc in ("brain", "brain-mcp", "reasoner", "picoclaw"):
for svc in ("brain", "brain-mcp", "reasoner", "picoclaw", "mail-sync"):
self.assertIn(svc, logged)
def test_start_assistant_no_attach_starts_picoclaw(self) -> None: