feat(mail): full Gmail+OnlyOffice sync, import, and brain indexing
- bin/mail/sync.go: async Go sync engine (8 workers, paginated Gmail via API + OnlyOffice IMAP); Gmail attachments key off body.attachmentId, not MIME partId; ICS sidecars Latin-1->UTF-8 normalized (TestICSToMarkdownNormalizesLatin1) - bin/mail/import: message.json -> markdown; PDFs via pdftotext -layout fast path with docling subprocess fallback for the ~5% textless files - bin/mail/index_mail: fresh-rebuild indexer (repo corpus + mail) avoiding ladybug WAL corruption on bulk-insert into indexed DBs; split from import - bin/kb/index: keep FTS/VECTOR indexes across incremental runs (drop+recreate leaves stale backing tables killing the vector index) - docs: README/PLAN/AGENTS cover the mail pipeline Result: 17,835 messages -> 28,918 info leafs, FTS+HNSW healthy.
This commit is contained in:
@@ -0,0 +1,154 @@
|
||||
// Package synccmd wires the sync library to a CLI: reads .env, parses flags,
|
||||
// picks sources, prints stats. Kept separate from the library so unit tests
|
||||
// don't depend on os.Args/env.
|
||||
package sync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// CLIConfig is a superset of SyncConfig plus flag parsing results.
|
||||
type CLIConfig struct {
|
||||
Sync SyncConfig
|
||||
Env string // .env path; default <cwd>/.env
|
||||
Sources string
|
||||
Help bool
|
||||
}
|
||||
|
||||
// ParseCLI reads os.Args into a CLIConfig. Exit codes: 0 ok, 2 usage.
|
||||
func ParseCLI(args []string) (CLIConfig, int, error) {
|
||||
fs := flag.NewFlagSet("mail/sync", flag.ContinueOnError)
|
||||
var (
|
||||
env = fs.String("env", "", ".env file (default: <cwd>/.env)")
|
||||
out = fs.String("out", "", "var/mail root (default: <cwd>/var/mail)")
|
||||
workers = fs.Int("workers", 4, "concurrent downloads")
|
||||
limit = fs.Int("limit", 0, "max messages per source (0 = all)")
|
||||
offset = fs.Int("offset", 0, "skip first N messages per source")
|
||||
force = fs.Bool("force", false, "overwrite existing message.json + attachments")
|
||||
dryRun = fs.Bool("dry-run", false, "list message counts without writing")
|
||||
srcs = fs.String("source", "onlyoffice", "comma list: onlyoffice,gmail (default onlyoffice)")
|
||||
help = fs.Bool("help", false, "usage")
|
||||
)
|
||||
fs.SetOutput(os.Stderr)
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return CLIConfig{}, 2, err
|
||||
}
|
||||
if *help || fs.NArg() > 0 {
|
||||
return CLIConfig{Help: true}, 0, nil
|
||||
}
|
||||
wd, err := os.Getwd()
|
||||
if err != nil {
|
||||
return CLIConfig{}, 2, err
|
||||
}
|
||||
if *env == "" {
|
||||
*env = filepath.Join(wd, ".env")
|
||||
}
|
||||
if *out == "" {
|
||||
*out = filepath.Join(wd, "var", "mail")
|
||||
}
|
||||
envVars := readEnv(*env)
|
||||
cfg := SyncConfig{
|
||||
Out: *out,
|
||||
Workers: *workers,
|
||||
Limit: *limit,
|
||||
Offset: *offset,
|
||||
Force: *force,
|
||||
DryRun: *dryRun,
|
||||
Policy: RetryPolicy{},
|
||||
}
|
||||
cli := CLIConfig{Sync: cfg, Env: *env, Sources: *srcs}
|
||||
for _, s := range strings.Split(*srcs, ",") {
|
||||
switch strings.TrimSpace(s) {
|
||||
case "onlyoffice":
|
||||
u := pick(envVars["ONLYOFFICE_URL"], envVars["OO_URL"])
|
||||
user := pick(envVars["ONLYOFFICE_USER"], envVars["OO_USER"])
|
||||
pass := pick(envVars["ONLYOFFICE_PASS"], envVars["OO_PASSWORD"])
|
||||
if u == "" || user == "" || pass == "" {
|
||||
return CLIConfig{}, 2, fmt.Errorf("onlyoffice source needs ONLYOFFICE_URL/USER/PASS in %s", *env)
|
||||
}
|
||||
cfg.OO = &OOConfig{URL: u, User: user, Password: pass}
|
||||
case "gmail":
|
||||
home, _ := os.UserHomeDir()
|
||||
cfg.Gmail = &GmailCredentials{
|
||||
CredentialsPath: filepath.Join(home, ".gmail-mcp", "credentials.json"),
|
||||
KeysPath: filepath.Join(home, ".gmail-mcp", "gcp-oauth.keys.json"),
|
||||
}
|
||||
default:
|
||||
return CLIConfig{}, 2, fmt.Errorf("unknown source %q", s)
|
||||
}
|
||||
}
|
||||
cli.Sync = cfg
|
||||
return cli, 0, nil
|
||||
}
|
||||
|
||||
// Main is the CLI entry: returns process exit code.
|
||||
func Main(args []string) int {
|
||||
cli, code, err := ParseCLI(args)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "mail/sync:", err)
|
||||
return code
|
||||
}
|
||||
if cli.Help {
|
||||
fmt.Fprintln(os.Stderr, "usage: bin/mail/sync.go [--source onlyoffice,gmail] [--limit N] [--offset N] [--workers N] [--force] [--dry-run]")
|
||||
return 0
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 6*time.Hour)
|
||||
defer cancel()
|
||||
start := time.Now()
|
||||
stats, err := Run(ctx, cli.Sync)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "mail/sync:", err)
|
||||
return 1
|
||||
}
|
||||
if cli.Sync.DryRun {
|
||||
fmt.Printf("mail/sync: dry-run checked=%d (no writes)\n", stats.Checked)
|
||||
return 0
|
||||
}
|
||||
fmt.Printf("mail/sync: checked=%d new=%d skipped=%d failed=%d in %s\n",
|
||||
stats.Checked, stats.New, stats.Skipped, stats.Failed, time.Since(start).Round(time.Millisecond))
|
||||
if stats.Failed > 0 {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// readEnv parses KEY=VALUE lines (ignoring comments) with KEY=PATH override.
|
||||
func readEnv(path string) map[string]string {
|
||||
out := map[string]string{}
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
for _, line := range strings.Split(string(b), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") || !strings.Contains(line, "=") {
|
||||
continue
|
||||
}
|
||||
k, v, _ := strings.Cut(line, "=")
|
||||
out[strings.TrimSpace(k)] = strings.Trim(strings.TrimSpace(v), "\"'")
|
||||
}
|
||||
// env overrides file
|
||||
for _, kv := range os.Environ() {
|
||||
k, v, ok := strings.Cut(kv, "=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(k, "ONLYOFFICE_") || strings.HasPrefix(k, "OO_") {
|
||||
out[k] = v
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func pick(a, b string) string {
|
||||
if a != "" {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
Reference in New Issue
Block a user