feat(mail): full Gmail+OnlyOffice sync, import, and brain indexing
- bin/mail/sync.go: async Go sync engine (8 workers, paginated Gmail via API + OnlyOffice IMAP); Gmail attachments key off body.attachmentId, not MIME partId; ICS sidecars Latin-1->UTF-8 normalized (TestICSToMarkdownNormalizesLatin1) - bin/mail/import: message.json -> markdown; PDFs via pdftotext -layout fast path with docling subprocess fallback for the ~5% textless files - bin/mail/index_mail: fresh-rebuild indexer (repo corpus + mail) avoiding ladybug WAL corruption on bulk-insert into indexed DBs; split from import - bin/kb/index: keep FTS/VECTOR indexes across incremental runs (drop+recreate leaves stale backing tables killing the vector index) - docs: README/PLAN/AGENTS cover the mail pipeline Result: 17,835 messages -> 28,918 info leafs, FTS+HNSW healthy.
This commit is contained in:
@@ -0,0 +1,154 @@
|
||||
// Package synccmd wires the sync library to a CLI: reads .env, parses flags,
|
||||
// picks sources, prints stats. Kept separate from the library so unit tests
|
||||
// don't depend on os.Args/env.
|
||||
package sync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// CLIConfig is a superset of SyncConfig plus flag parsing results.
|
||||
type CLIConfig struct {
|
||||
Sync SyncConfig
|
||||
Env string // .env path; default <cwd>/.env
|
||||
Sources string
|
||||
Help bool
|
||||
}
|
||||
|
||||
// ParseCLI reads os.Args into a CLIConfig. Exit codes: 0 ok, 2 usage.
|
||||
func ParseCLI(args []string) (CLIConfig, int, error) {
|
||||
fs := flag.NewFlagSet("mail/sync", flag.ContinueOnError)
|
||||
var (
|
||||
env = fs.String("env", "", ".env file (default: <cwd>/.env)")
|
||||
out = fs.String("out", "", "var/mail root (default: <cwd>/var/mail)")
|
||||
workers = fs.Int("workers", 4, "concurrent downloads")
|
||||
limit = fs.Int("limit", 0, "max messages per source (0 = all)")
|
||||
offset = fs.Int("offset", 0, "skip first N messages per source")
|
||||
force = fs.Bool("force", false, "overwrite existing message.json + attachments")
|
||||
dryRun = fs.Bool("dry-run", false, "list message counts without writing")
|
||||
srcs = fs.String("source", "onlyoffice", "comma list: onlyoffice,gmail (default onlyoffice)")
|
||||
help = fs.Bool("help", false, "usage")
|
||||
)
|
||||
fs.SetOutput(os.Stderr)
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return CLIConfig{}, 2, err
|
||||
}
|
||||
if *help || fs.NArg() > 0 {
|
||||
return CLIConfig{Help: true}, 0, nil
|
||||
}
|
||||
wd, err := os.Getwd()
|
||||
if err != nil {
|
||||
return CLIConfig{}, 2, err
|
||||
}
|
||||
if *env == "" {
|
||||
*env = filepath.Join(wd, ".env")
|
||||
}
|
||||
if *out == "" {
|
||||
*out = filepath.Join(wd, "var", "mail")
|
||||
}
|
||||
envVars := readEnv(*env)
|
||||
cfg := SyncConfig{
|
||||
Out: *out,
|
||||
Workers: *workers,
|
||||
Limit: *limit,
|
||||
Offset: *offset,
|
||||
Force: *force,
|
||||
DryRun: *dryRun,
|
||||
Policy: RetryPolicy{},
|
||||
}
|
||||
cli := CLIConfig{Sync: cfg, Env: *env, Sources: *srcs}
|
||||
for _, s := range strings.Split(*srcs, ",") {
|
||||
switch strings.TrimSpace(s) {
|
||||
case "onlyoffice":
|
||||
u := pick(envVars["ONLYOFFICE_URL"], envVars["OO_URL"])
|
||||
user := pick(envVars["ONLYOFFICE_USER"], envVars["OO_USER"])
|
||||
pass := pick(envVars["ONLYOFFICE_PASS"], envVars["OO_PASSWORD"])
|
||||
if u == "" || user == "" || pass == "" {
|
||||
return CLIConfig{}, 2, fmt.Errorf("onlyoffice source needs ONLYOFFICE_URL/USER/PASS in %s", *env)
|
||||
}
|
||||
cfg.OO = &OOConfig{URL: u, User: user, Password: pass}
|
||||
case "gmail":
|
||||
home, _ := os.UserHomeDir()
|
||||
cfg.Gmail = &GmailCredentials{
|
||||
CredentialsPath: filepath.Join(home, ".gmail-mcp", "credentials.json"),
|
||||
KeysPath: filepath.Join(home, ".gmail-mcp", "gcp-oauth.keys.json"),
|
||||
}
|
||||
default:
|
||||
return CLIConfig{}, 2, fmt.Errorf("unknown source %q", s)
|
||||
}
|
||||
}
|
||||
cli.Sync = cfg
|
||||
return cli, 0, nil
|
||||
}
|
||||
|
||||
// Main is the CLI entry: returns process exit code.
|
||||
func Main(args []string) int {
|
||||
cli, code, err := ParseCLI(args)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "mail/sync:", err)
|
||||
return code
|
||||
}
|
||||
if cli.Help {
|
||||
fmt.Fprintln(os.Stderr, "usage: bin/mail/sync.go [--source onlyoffice,gmail] [--limit N] [--offset N] [--workers N] [--force] [--dry-run]")
|
||||
return 0
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 6*time.Hour)
|
||||
defer cancel()
|
||||
start := time.Now()
|
||||
stats, err := Run(ctx, cli.Sync)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "mail/sync:", err)
|
||||
return 1
|
||||
}
|
||||
if cli.Sync.DryRun {
|
||||
fmt.Printf("mail/sync: dry-run checked=%d (no writes)\n", stats.Checked)
|
||||
return 0
|
||||
}
|
||||
fmt.Printf("mail/sync: checked=%d new=%d skipped=%d failed=%d in %s\n",
|
||||
stats.Checked, stats.New, stats.Skipped, stats.Failed, time.Since(start).Round(time.Millisecond))
|
||||
if stats.Failed > 0 {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// readEnv parses KEY=VALUE lines (ignoring comments) with KEY=PATH override.
|
||||
func readEnv(path string) map[string]string {
|
||||
out := map[string]string{}
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
for _, line := range strings.Split(string(b), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") || !strings.Contains(line, "=") {
|
||||
continue
|
||||
}
|
||||
k, v, _ := strings.Cut(line, "=")
|
||||
out[strings.TrimSpace(k)] = strings.Trim(strings.TrimSpace(v), "\"'")
|
||||
}
|
||||
// env overrides file
|
||||
for _, kv := range os.Environ() {
|
||||
k, v, ok := strings.Cut(kv, "=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(k, "ONLYOFFICE_") || strings.HasPrefix(k, "OO_") {
|
||||
out[k] = v
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func pick(a, b string) string {
|
||||
if a != "" {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
@@ -0,0 +1,356 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// GmailCredentials holds the OAuth files produced by the gmail MCP
|
||||
// (@gongrzhe/server-gmail-autoauth-mcp) auto-auth flow.
|
||||
type GmailCredentials struct {
|
||||
CredentialsPath string // ~/.gmail-mcp/credentials.json
|
||||
KeysPath string // ~/.gmail-mcp/gcp-oauth.keys.json
|
||||
User string // fixed: the authed account
|
||||
}
|
||||
|
||||
// gmailToken is the JSON shape of credentials.json + refresh response.
|
||||
type gmailToken struct {
|
||||
AccessToken string `json:"access_token"`
|
||||
RefreshToken string `json:"refresh_token"`
|
||||
Expiry int64 `json:"expiry_date"` // ms epoch
|
||||
}
|
||||
|
||||
type gmailKeys struct {
|
||||
Installed *gmailKeyBlock `json:"installed"`
|
||||
Web *gmailKeyBlock `json:"web"`
|
||||
}
|
||||
type gmailKeyBlock struct {
|
||||
ClientID string `json:"client_id"`
|
||||
ClientSecret string `json:"client_secret"`
|
||||
}
|
||||
|
||||
// GmailClient talks to the Gmail REST API using the OAuth refresh token from
|
||||
// ~/.gmail-mcp/. Token is refreshed lazily with a mutex-guarded cache.
|
||||
type GmailClient struct {
|
||||
creds GmailCredentials
|
||||
client *http.Client
|
||||
mu chan struct{}
|
||||
token *gmailToken
|
||||
user string
|
||||
}
|
||||
|
||||
func NewGmailClient(creds GmailCredentials) (*GmailClient, error) {
|
||||
if creds.CredentialsPath == "" {
|
||||
home, _ := os.UserHomeDir()
|
||||
creds.CredentialsPath = filepath.Join(home, ".gmail-mcp", "credentials.json")
|
||||
creds.KeysPath = filepath.Join(home, ".gmail-mcp", "gcp-oauth.keys.json")
|
||||
}
|
||||
g := &GmailClient{
|
||||
creds: creds,
|
||||
client: &http.Client{Timeout: 60 * time.Second},
|
||||
mu: make(chan struct{}, 1),
|
||||
}
|
||||
g.mu <- struct{}{}
|
||||
return g, nil
|
||||
}
|
||||
|
||||
// accessToken returns a fresh bearer token, refreshing via the Google token
|
||||
// endpoint when the cached one is missing or about to expire.
|
||||
func (g *GmailClient) accessToken(ctx context.Context) (string, error) {
|
||||
select {
|
||||
case <-g.mu:
|
||||
case <-ctx.Done():
|
||||
return "", ctx.Err()
|
||||
}
|
||||
defer func() { g.mu <- struct{}{} }()
|
||||
if g.token != nil && g.token.AccessToken != "" && g.token.Expiry > time.Now().UnixMilli()+300_000 {
|
||||
return g.token.AccessToken, nil
|
||||
}
|
||||
return g.refreshLocked(ctx)
|
||||
}
|
||||
|
||||
func (g *GmailClient) refreshLocked(ctx context.Context) (string, error) {
|
||||
cred, err := os.ReadFile(g.creds.CredentialsPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read gmail credentials %s: %w", g.creds.CredentialsPath, err)
|
||||
}
|
||||
var t gmailToken
|
||||
if err := json.Unmarshal(cred, &t); err != nil {
|
||||
return "", fmt.Errorf("parse gmail credentials: %w", err)
|
||||
}
|
||||
if t.RefreshToken == "" {
|
||||
return "", errors.New("gmail credentials.json has no refresh_token (run the gmail MCP auth flow)")
|
||||
}
|
||||
keys, err := os.ReadFile(g.creds.KeysPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read gmail keys %s: %w", g.creds.KeysPath, err)
|
||||
}
|
||||
var k gmailKeys
|
||||
if err := json.Unmarshal(keys, &k); err != nil {
|
||||
return "", fmt.Errorf("parse gmail keys: %w", err)
|
||||
}
|
||||
block := k.Installed
|
||||
if block == nil {
|
||||
block = k.Web
|
||||
}
|
||||
if block == nil {
|
||||
return "", errors.New("gmail gcp-oauth.keys.json has no installed/web block")
|
||||
}
|
||||
|
||||
form := url.Values{}
|
||||
form.Set("client_id", block.ClientID)
|
||||
form.Set("client_secret", block.ClientSecret)
|
||||
form.Set("refresh_token", t.RefreshToken)
|
||||
form.Set("grant_type", "refresh_token")
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://oauth2.googleapis.com/token",
|
||||
strings.NewReader(form.Encode()))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
resp, err := g.client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("gmail token refresh: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
var e struct {
|
||||
Error string `json:"error"`
|
||||
Desc string `json:"error_description"`
|
||||
}
|
||||
_ = json.Unmarshal(body, &e)
|
||||
if e.Error == "invalid_grant" {
|
||||
return "", fmt.Errorf("gmail OAuth token invalid/expired - re-auth via: npx -y @gongrzhe/server-gmail-autoauth-mcp auth (uses ~/.gmail-mcp)")
|
||||
}
|
||||
return "", fmt.Errorf("gmail token refresh status %d: %s", resp.StatusCode, truncate(string(body), 300))
|
||||
}
|
||||
var out struct {
|
||||
AccessToken string `json:"access_token"`
|
||||
ExpiresIn int64 `json:"expires_in"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &out); err != nil {
|
||||
return "", fmt.Errorf("gmail token refresh parse: %w", err)
|
||||
}
|
||||
g.token = &gmailToken{
|
||||
AccessToken: out.AccessToken,
|
||||
RefreshToken: t.RefreshToken,
|
||||
Expiry: time.Now().UnixMilli() + out.ExpiresIn*1000,
|
||||
}
|
||||
return out.AccessToken, nil
|
||||
}
|
||||
|
||||
// ListIDs returns message ids matching q, walking nextPageToken up to maxIDs
|
||||
// (0 = unlimited). Thread-level pagination via the messages.list endpoint.
|
||||
func (g *GmailClient) ListIDs(ctx context.Context, q string, maxIDs int, pageToken string) (ids []string, next string, err error) {
|
||||
for {
|
||||
params := url.Values{}
|
||||
params.Set("q", q)
|
||||
params.Set("maxResults", "100")
|
||||
if pageToken != "" {
|
||||
params.Set("pageToken", pageToken)
|
||||
}
|
||||
var out struct {
|
||||
Messages []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"messages"`
|
||||
NextPageToken string `json:"nextPageToken"`
|
||||
}
|
||||
if err := g.getJSON(ctx, "/gmail/v1/users/me/messages?"+params.Encode(), &out); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
for _, m := range out.Messages {
|
||||
ids = append(ids, m.ID)
|
||||
if maxIDs > 0 && len(ids) >= maxIDs {
|
||||
return ids, out.NextPageToken, nil
|
||||
}
|
||||
}
|
||||
if out.NextPageToken == "" {
|
||||
break
|
||||
}
|
||||
pageToken = out.NextPageToken
|
||||
}
|
||||
return ids, "", nil
|
||||
}
|
||||
|
||||
// GetMessage fetches a message in format=full and normalizes it.
|
||||
func (g *GmailClient) GetMessage(ctx context.Context, id string) (*Message, error) {
|
||||
var raw struct {
|
||||
ID string `json:"id"`
|
||||
ThreadID string `json:"threadId"`
|
||||
InternalDate string `json:"internalDate"` // ms epoch string
|
||||
Payload gmailPart
|
||||
}
|
||||
path := "/gmail/v1/users/me/messages/" + url.PathEscape(id) + "?format=full"
|
||||
if err := g.getJSON(ctx, path, &raw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m := &Message{
|
||||
Source: "gmail",
|
||||
ID: raw.ID,
|
||||
Folder: "gmail",
|
||||
}
|
||||
for _, h := range raw.Payload.Headers {
|
||||
switch strings.ToLower(h.Name) {
|
||||
case "subject":
|
||||
m.Subject = h.Value
|
||||
case "from":
|
||||
m.From = h.Value
|
||||
case "to":
|
||||
m.To = h.Value
|
||||
case "cc":
|
||||
m.CC = h.Value
|
||||
case "bcc":
|
||||
m.BCC = h.Value
|
||||
case "message-id":
|
||||
m.MimeMessageID = h.Value
|
||||
case "date":
|
||||
if t, err := time.Parse(time.RFC1123Z, h.Value); err == nil {
|
||||
m.ReceivedAt = t
|
||||
}
|
||||
}
|
||||
}
|
||||
if ms, err := parseMS(raw.InternalDate); err == nil {
|
||||
m.ReceivedAt = ms
|
||||
}
|
||||
m.TextBody, m.HTMLBody, m.Attachments = collectParts(raw.Payload, "root", m.ID, 0)
|
||||
m.HasAttachments = len(m.Attachments) > 0
|
||||
return m, nil
|
||||
}
|
||||
|
||||
type gmailPart struct {
|
||||
PartID string `json:"partId"`
|
||||
MimeType string `json:"mimeType"`
|
||||
Filename string `json:"filename"`
|
||||
Body gmailBody `json:"body"`
|
||||
Headers []gmailHeader `json:"headers"`
|
||||
Parts []gmailPart `json:"parts"`
|
||||
}
|
||||
type gmailHeader struct {
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
type gmailBody struct {
|
||||
Size int64 `json:"size"`
|
||||
Data string `json:"data"`
|
||||
AttachmentID string `json:"attachmentId"`
|
||||
}
|
||||
|
||||
// collectParts walks the MIME tree: text bodies into plain/html, anything with
|
||||
// a filename into attachments (returned with base64 ids for later download).
|
||||
func collectParts(p gmailPart, mime string, msgID string, depth int) (text, html string, atts []Attachment) {
|
||||
if depth > 16 {
|
||||
return
|
||||
}
|
||||
mt := strings.ToLower(p.MimeType)
|
||||
if p.Filename != "" && mt != "text/plain" && mt != "text/html" {
|
||||
pid := p.PartID
|
||||
if pid == "" {
|
||||
pid = fmt.Sprintf("%d", depth)
|
||||
}
|
||||
// Gmail's attachments API keys off body.attachmentId, not partId.
|
||||
attID := p.Body.AttachmentID
|
||||
if attID == "" {
|
||||
attID = pid
|
||||
}
|
||||
atts = append(atts, Attachment{
|
||||
FileID: msgID + ":" + attID,
|
||||
FileName: p.Filename,
|
||||
StoredName: p.Filename,
|
||||
Size: p.Body.Size,
|
||||
ContentType: p.MimeType,
|
||||
})
|
||||
} else if data, err := base64.URLEncoding.DecodeString(p.Body.Data); err == nil && len(p.Body.Data) > 0 {
|
||||
s := string(data)
|
||||
if mt == "text/html" && html == "" {
|
||||
html = s
|
||||
} else if (mt == "text/plain" || mt == "") && text == "" {
|
||||
text = s
|
||||
}
|
||||
}
|
||||
for _, child := range p.Parts {
|
||||
t, h, a := collectParts(child, mt, msgID, depth+1)
|
||||
if text == "" {
|
||||
text = t
|
||||
}
|
||||
if html == "" {
|
||||
html = h
|
||||
}
|
||||
atts = append(atts, a...)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// DownloadAttachment fetches an attachment's bytes from the Gmail API.
|
||||
func (g *GmailClient) DownloadAttachment(ctx context.Context, msgID, attID string) ([]byte, error) {
|
||||
// attID format is "<msgId>:<partId>"; the API needs the bare attachment id.
|
||||
partID := attID
|
||||
if i := strings.Index(attID, ":"); i >= 0 {
|
||||
partID = attID[i+1:]
|
||||
}
|
||||
var out struct {
|
||||
Data string `json:"data"`
|
||||
}
|
||||
path := "/gmail/v1/users/me/messages/" + url.PathEscape(msgID) + "/attachments/" + url.PathEscape(partID)
|
||||
if err := g.getJSON(ctx, path, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return base64.URLEncoding.DecodeString(out.Data)
|
||||
}
|
||||
|
||||
func (g *GmailClient) getJSON(ctx context.Context, path string, out any) error {
|
||||
tok, err := g.accessToken(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
u := "https://gmail.googleapis.com" + path
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+tok)
|
||||
resp, err := g.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 16<<20))
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("gmail %s: status %d: %s", path, resp.StatusCode, truncate(string(body), 300))
|
||||
}
|
||||
if out != nil {
|
||||
return json.Unmarshal(body, out)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseMS(s string) (time.Time, error) {
|
||||
if s == "" {
|
||||
return time.Time{}, errors.New("empty")
|
||||
}
|
||||
var ms int64
|
||||
if _, err := fmt.Sscanf(s, "%d", &ms); err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
return time.UnixMilli(ms), nil
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
return s[:n] + "…"
|
||||
}
|
||||
|
||||
var _ = bytes.MinRead
|
||||
@@ -0,0 +1,258 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
ics "github.com/arran4/golang-ical"
|
||||
"golang.org/x/text/encoding/charmap"
|
||||
)
|
||||
|
||||
// ICSToMarkdown parses a VCALENDAR/VEVENT payload and renders a compact
|
||||
// structured markdown block: what / when / where / organizer / attendees.
|
||||
// Returns the raw text when the payload is not a calendar.
|
||||
func ICSToMarkdown(data []byte) string {
|
||||
data = normalizeEncoding(data)
|
||||
cal, err := ics.ParseCalendar(strings.NewReader(string(data)))
|
||||
if err != nil {
|
||||
return normalizeMarkdown(string(data))
|
||||
}
|
||||
method := ""
|
||||
for _, p := range cal.CalendarProperties {
|
||||
if p.IANAToken == string(ics.ComponentPropertyMethod) {
|
||||
method = p.Value
|
||||
break
|
||||
}
|
||||
}
|
||||
method = strings.TrimSpace(method)
|
||||
var out []string
|
||||
for _, ev := range cal.Events() {
|
||||
summary := strings.TrimSpace(propValue(ev, ics.ComponentPropertySummary))
|
||||
if summary != "" {
|
||||
out = append(out, "# "+summary)
|
||||
}
|
||||
if when := eventWhen(ev); when != "" {
|
||||
out = append(out, "- **When:** "+when)
|
||||
}
|
||||
if loc := strings.TrimSpace(propValue(ev, ics.ComponentPropertyLocation)); loc != "" {
|
||||
out = append(out, "- **Where:** "+loc)
|
||||
}
|
||||
if desc := strings.TrimSpace(stripHTML(propValue(ev, ics.ComponentPropertyDescription))); desc != "" {
|
||||
out = append(out, "- **What:** "+desc)
|
||||
}
|
||||
if org := propValue(ev, ics.ComponentPropertyOrganizer); org != "" {
|
||||
out = append(out, "- **Organizer:** "+attendeeFmt(org))
|
||||
}
|
||||
for _, a := range ev.Attendees() {
|
||||
cn := strings.TrimSpace(firstParam(a.ICalParameters, "CN"))
|
||||
partstat := string(a.ParticipationStatus())
|
||||
name := cn
|
||||
if name == "" {
|
||||
name = a.Email()
|
||||
}
|
||||
line := name
|
||||
if email := a.Email(); email != "" && email != name {
|
||||
line = name + " <" + email + ">"
|
||||
}
|
||||
if partstat != "" && !strings.EqualFold(partstat, "NEEDS-ACTION") {
|
||||
line += " (" + strings.Title(strings.ToLower(strings.ReplaceAll(partstat, "_", " "))) + ")"
|
||||
}
|
||||
out = append(out, "- **Attendee:** "+line)
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return normalizeMarkdown(string(data))
|
||||
}
|
||||
if method != "" {
|
||||
out = append([]string{"*Calendar method: " + method + "*"}, out...)
|
||||
}
|
||||
return normalizeMarkdown(strings.Join(out, "\n\n"))
|
||||
}
|
||||
|
||||
func eventWhen(ev *ics.VEvent) string {
|
||||
start, errStart := ev.GetStartAt()
|
||||
end, errEnd := ev.GetEndAt()
|
||||
// All-day events: golang-ical has dedicated getters.
|
||||
if errStart != nil {
|
||||
if allDay, err := ev.GetAllDayStartAt(); err == nil {
|
||||
start = allDay
|
||||
errStart = nil
|
||||
}
|
||||
}
|
||||
if errEnd != nil {
|
||||
if allDay, err := ev.GetAllDayEndAt(); err == nil {
|
||||
end = allDay
|
||||
errEnd = nil
|
||||
}
|
||||
}
|
||||
if errStart != nil {
|
||||
// Non-IANA TZID (e.g. "W. Europe Standard Time"): parse the raw
|
||||
// property text instead of failing.
|
||||
return rawWhen(ev)
|
||||
}
|
||||
if errEnd != nil || end.Equal(start) {
|
||||
return dtFmt(start)
|
||||
}
|
||||
return dtFmt(start) + " → " + dtFmt(end)
|
||||
}
|
||||
|
||||
// rawWhen parses DTSTART/DTEND property values that golang-ical cannot resolve
|
||||
// because the TZID is not an IANA zone. Formats: 20260812T120000 or 20260812.
|
||||
func rawWhen(ev *ics.VEvent) string {
|
||||
start := rawPropValue(ev, ics.ComponentPropertyDtStart)
|
||||
end := rawPropValue(ev, ics.ComponentPropertyDtEnd)
|
||||
if start == "" {
|
||||
return ""
|
||||
}
|
||||
if end == "" || end == start {
|
||||
return rawDTFmt(start)
|
||||
}
|
||||
return rawDTFmt(start) + " → " + rawDTFmt(end)
|
||||
}
|
||||
|
||||
func rawPropValue(ev *ics.VEvent, prop ics.ComponentProperty) string {
|
||||
p := ev.GetProperty(prop)
|
||||
if p == nil {
|
||||
return ""
|
||||
}
|
||||
return p.Value
|
||||
}
|
||||
|
||||
// rawDTFmt turns 20260812T120000 into 2026-08-12 12:00; 20260812 into 2026-08-12.
|
||||
func rawDTFmt(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if len(s) >= 8 && isDigits(s[:8]) {
|
||||
y, m, d := s[:4], s[4:6], s[6:8]
|
||||
if len(s) > 8 && (s[8] == 'T' || s[8] == 't') && len(s) >= 15 && isDigits(s[9:15]) {
|
||||
h, mi := s[9:11], s[11:13]
|
||||
return fmt.Sprintf("%s-%s-%s %s:%s", y, m, d, h, mi)
|
||||
}
|
||||
return fmt.Sprintf("%s-%s-%s", y, m, d)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func isDigits(s string) bool {
|
||||
for _, c := range s {
|
||||
if c < '0' || c > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return s != ""
|
||||
}
|
||||
|
||||
// dtFmt renders a time as local "2006-01-02 15:04" (tz label when meaningful).
|
||||
func dtFmt(t time.Time) string {
|
||||
loc := t.Local()
|
||||
label := ""
|
||||
if loc.Location() != time.Local {
|
||||
label = " " + loc.Location().String()
|
||||
}
|
||||
return loc.Format("2006-01-02 15:04") + label
|
||||
}
|
||||
|
||||
// propertyGetter is satisfied by both *ics.Calendar and *ics.VEvent.
|
||||
type propertyGetter interface {
|
||||
GetProperty(ics.ComponentProperty) *ics.IANAProperty
|
||||
}
|
||||
|
||||
func propValue(ev propertyGetter, prop ics.ComponentProperty) string {
|
||||
p := ev.GetProperty(prop)
|
||||
if p == nil {
|
||||
return ""
|
||||
}
|
||||
return p.Value
|
||||
}
|
||||
|
||||
func firstParam(params map[string][]string, key string) string {
|
||||
if vs, ok := params[key]; ok && len(vs) > 0 {
|
||||
return vs[0]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func attendeeFmt(raw string) string {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if i := strings.Index(raw, ":"); i >= 0 {
|
||||
raw = raw[i+1:]
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
// stripHTML removes tags and decodes entities from an ics DESCRIPTION that may
|
||||
// carry HTML (Outlook/Exchange style), keeping text lines readable.
|
||||
func stripHTML(s string) string {
|
||||
if !strings.Contains(s, "<") {
|
||||
return s
|
||||
}
|
||||
lines := strings.Split(s, "\n")
|
||||
for i, l := range lines {
|
||||
var b strings.Builder
|
||||
depth := 0
|
||||
for j := 0; j < len(l); j++ {
|
||||
c := l[j]
|
||||
if c == '<' {
|
||||
if j+1 < len(l) && l[j+1] == '/' {
|
||||
depth--
|
||||
} else {
|
||||
depth++
|
||||
}
|
||||
for j < len(l) && l[j] != '>' {
|
||||
j++
|
||||
}
|
||||
continue
|
||||
}
|
||||
if c == '>' {
|
||||
continue
|
||||
}
|
||||
if depth == 0 {
|
||||
b.WriteByte(c)
|
||||
}
|
||||
}
|
||||
lines[i] = strings.TrimSpace(b.String())
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
// normalizeMarkdown collapses blank-line runs and strips control chars.
|
||||
func normalizeMarkdown(s string) string {
|
||||
s = strings.ReplaceAll(s, "\x00", "")
|
||||
for _, ch := range []string{"\ufeff", "\u200b", "\u034f", "\u00ad", "\u2007", "\u2008", "\u200a", "\u2002"} {
|
||||
s = strings.ReplaceAll(s, ch, "")
|
||||
}
|
||||
lines := strings.Split(s, "\n")
|
||||
var out []string
|
||||
blank := 0
|
||||
for _, l := range lines {
|
||||
if strings.TrimSpace(l) == "" {
|
||||
blank++
|
||||
if blank > 1 {
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
blank = 0
|
||||
}
|
||||
out = append(out, l)
|
||||
}
|
||||
return strings.Join(out, "\n")
|
||||
}
|
||||
|
||||
// normalizeEncoding re-encodes legacy single-byte text as UTF-8. ICS files
|
||||
// exported by some portals are Latin-1 (e.g. "N\xfcrnberg"); golang-ical
|
||||
// passes the bytes through, producing invalid UTF-8 in the markdown output.
|
||||
// Valid UTF-8 is returned untouched.
|
||||
func normalizeEncoding(data []byte) []byte {
|
||||
if utf8.Valid(data) {
|
||||
return data
|
||||
}
|
||||
dec := charmap.ISO8859_1.NewDecoder()
|
||||
out, err := dec.Bytes(data)
|
||||
if err != nil {
|
||||
return data
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var _ = fmt.Sprintf // keep fmt import if helpers change
|
||||
@@ -0,0 +1,222 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// OOConfig mirrors the .env / environment used by bin/mail/import.
|
||||
type OOConfig struct {
|
||||
URL string
|
||||
User string
|
||||
Password string
|
||||
}
|
||||
|
||||
// OOClient is a minimal OnlyOffice API client: authentication.json for the
|
||||
// bearer token plus the session cookie jar required by the .ashx download
|
||||
// handler. It mirrors the endpoint contract bin/mail/import already uses.
|
||||
type OOClient struct {
|
||||
cfg OOConfig
|
||||
client *http.Client
|
||||
mu chan struct{}
|
||||
token string
|
||||
folderID int
|
||||
}
|
||||
|
||||
func NewOOClient(cfg OOConfig, folderID int) (*OOClient, error) {
|
||||
jar, err := cookiejar.New(nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c := &OOClient{
|
||||
cfg: cfg,
|
||||
client: &http.Client{Jar: jar, Timeout: 60 * time.Second},
|
||||
mu: make(chan struct{}, 1),
|
||||
folderID: folderID,
|
||||
}
|
||||
c.mu <- struct{}{}
|
||||
if err := c.authenticate(context.Background()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (o *OOClient) authenticate(ctx context.Context) error {
|
||||
select {
|
||||
case <-o.mu:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
defer func() { o.mu <- struct{}{} }()
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"userName": o.cfg.User, "password": o.cfg.Password, "type": 0,
|
||||
})
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
strings.TrimRight(o.cfg.URL, "/")+"/api/2.0/authentication.json",
|
||||
strings.NewReader(string(body)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := o.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
data, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
||||
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
||||
return fmt.Errorf("oo authenticate status %d: %s", resp.StatusCode, truncate(string(data), 200))
|
||||
}
|
||||
var out struct {
|
||||
Response struct {
|
||||
Token string `json:"token"`
|
||||
} `json:"response"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
if out.Response.Token == "" {
|
||||
return fmt.Errorf("oo authenticate: empty token")
|
||||
}
|
||||
o.token = out.Response.Token
|
||||
return nil
|
||||
}
|
||||
|
||||
// get performs an authenticated GET and decodes the JSON body into out.
|
||||
func (o *OOClient) get(ctx context.Context, path string, out any) error {
|
||||
u := strings.TrimRight(o.cfg.URL, "/") + path
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+o.token)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
resp, err := o.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
data, _ := io.ReadAll(io.LimitReader(resp.Body, 16<<20))
|
||||
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
||||
return fmt.Errorf("oo %s: status %d: %s", path, resp.StatusCode, truncate(string(data), 300))
|
||||
}
|
||||
if out != nil {
|
||||
return json.Unmarshal(data, out)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ooMessage mirrors the OnlyOffice mail message JSON (subset we need).
|
||||
type ooMessage struct {
|
||||
ID int `json:"id"`
|
||||
Subject string `json:"subject"`
|
||||
From string `json:"from"`
|
||||
To string `json:"to"`
|
||||
CC string `json:"cc"`
|
||||
BCC string `json:"bcc"`
|
||||
ReceivedDate string `json:"receivedDate"`
|
||||
HTMLBody string `json:"htmlBody"`
|
||||
TextBody string `json:"textBody"`
|
||||
HasAttachments bool `json:"hasAttachments"`
|
||||
MimeMessageID string `json:"mimeMessageId"`
|
||||
Attachments []struct {
|
||||
FileID int `json:"fileId"`
|
||||
FileName string `json:"fileName"`
|
||||
StoredName string `json:"storedName"`
|
||||
Size int64 `json:"size"`
|
||||
ContentType string `json:"contentType"`
|
||||
} `json:"attachments"`
|
||||
}
|
||||
|
||||
// ListIDs returns message ids in the configured folder, paginating pages until
|
||||
// maxIDs is reached (0 = all).
|
||||
func (o *OOClient) ListIDs(ctx context.Context, maxIDs int, page int) (ids []int, next int, err error) {
|
||||
var out struct {
|
||||
Response []ooMessage `json:"response"`
|
||||
}
|
||||
count := 100
|
||||
if maxIDs > 0 && maxIDs < count {
|
||||
count = maxIDs
|
||||
}
|
||||
path := fmt.Sprintf("/api/2.0/mail/messages?folder=%d&page=%d&count=%d", o.folderID, page, count)
|
||||
if err := o.get(ctx, path, &out); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
for _, m := range out.Response {
|
||||
ids = append(ids, m.ID)
|
||||
if maxIDs > 0 && len(ids) >= maxIDs {
|
||||
break
|
||||
}
|
||||
}
|
||||
next = page + 1
|
||||
return ids, next, nil
|
||||
}
|
||||
|
||||
// GetMessage fetches the full message by id and normalizes into Message.
|
||||
func (o *OOClient) GetMessage(ctx context.Context, id int) (*Message, error) {
|
||||
var out struct {
|
||||
Response ooMessage `json:"response"`
|
||||
}
|
||||
path := fmt.Sprintf("/api/2.0/mail/messages/%d", id)
|
||||
if err := o.get(ctx, path, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m := out.Response
|
||||
msg := &Message{
|
||||
Source: "onlyoffice",
|
||||
ID: fmt.Sprintf("%d", m.ID),
|
||||
Folder: "oo",
|
||||
Subject: m.Subject,
|
||||
From: m.From,
|
||||
To: m.To,
|
||||
CC: m.CC,
|
||||
BCC: m.BCC,
|
||||
HTMLBody: m.HTMLBody,
|
||||
TextBody: m.TextBody,
|
||||
HasAttachments: m.HasAttachments,
|
||||
MimeMessageID: m.MimeMessageID,
|
||||
}
|
||||
if t, err := time.Parse(time.RFC3339Nano, m.ReceivedDate); err == nil {
|
||||
msg.ReceivedAt = t
|
||||
}
|
||||
for _, a := range m.Attachments {
|
||||
msg.Attachments = append(msg.Attachments, Attachment{
|
||||
FileID: fmt.Sprintf("%d", a.FileID),
|
||||
FileName: a.FileName,
|
||||
StoredName: a.StoredName,
|
||||
Size: a.Size,
|
||||
ContentType: a.ContentType,
|
||||
})
|
||||
}
|
||||
return msg, nil
|
||||
}
|
||||
|
||||
// DownloadAttachment fetches attachment bytes via the .ashx handler, which
|
||||
// requires the session cookie (client.Jar) captured during authenticate().
|
||||
func (o *OOClient) DownloadAttachment(ctx context.Context, fileID string) ([]byte, error) {
|
||||
u := strings.TrimRight(o.cfg.URL, "/") + "/addons/mail/httphandlers/download.ashx?attachid=" + url.QueryEscape(fileID)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := o.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, 64<<20))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("oo download attach %s: status %d", fileID, resp.StatusCode)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RetryPolicy is the exponential-backoff strategy applied to transient HTTP
|
||||
// failures (5xx, timeouts, network errors). Callers wrap transient errors with
|
||||
// retryWrap; everything else aborts immediately.
|
||||
type RetryPolicy struct {
|
||||
MaxAttempts int // total attempts (>=1); 0 => 5
|
||||
BaseDelay time.Duration // first backoff; 0 => 250ms
|
||||
MaxDelay time.Duration // cap; 0 => 15s
|
||||
Jitter float64 // 0..1 multiplier; 0 => 0.2
|
||||
}
|
||||
|
||||
func (p RetryPolicy) withDefaults() RetryPolicy {
|
||||
if p.MaxAttempts <= 0 {
|
||||
p.MaxAttempts = 5
|
||||
}
|
||||
if p.BaseDelay <= 0 {
|
||||
p.BaseDelay = 250 * time.Millisecond
|
||||
}
|
||||
if p.MaxDelay <= 0 {
|
||||
p.MaxDelay = 15 * time.Second
|
||||
}
|
||||
if p.Jitter <= 0 {
|
||||
p.Jitter = 0.2
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// delay returns the wait before attempt n (1-based): base * 2^(n-2) + jitter,
|
||||
// capped at MaxDelay. Attempt 1 waits 0, attempt 2 waits base, then doubles.
|
||||
func (p RetryPolicy) delay(attempt int) time.Duration {
|
||||
if attempt <= 1 {
|
||||
return 0
|
||||
}
|
||||
exp := math.Min(float64(attempt-2), 10)
|
||||
d := float64(p.BaseDelay) * math.Pow(2, exp)
|
||||
if p.Jitter > 0 {
|
||||
d *= 1 - p.Jitter + 2*p.Jitter*rand.Float64()
|
||||
}
|
||||
if d > float64(p.MaxDelay) {
|
||||
d = float64(p.MaxDelay)
|
||||
}
|
||||
return time.Duration(d)
|
||||
}
|
||||
|
||||
type errRetry struct{ err error }
|
||||
|
||||
func (e *errRetry) Error() string { return e.err.Error() }
|
||||
func (e *errRetry) Unwrap() error { return e.err }
|
||||
|
||||
func isRetriable(err error) bool {
|
||||
var r *errRetry
|
||||
return errors.As(err, &r)
|
||||
}
|
||||
|
||||
func retryWrap(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if isRetriable(err) {
|
||||
return err
|
||||
}
|
||||
return &errRetry{err: err}
|
||||
}
|
||||
|
||||
// Retry runs fn up to MaxAttempts times with exponential backoff between
|
||||
// attempts. Non-retriable errors abort immediately. Returns the last error.
|
||||
func Retry(ctx context.Context, policy RetryPolicy, fn func() error) error {
|
||||
policy = policy.withDefaults()
|
||||
var err error
|
||||
for attempt := 1; attempt <= policy.MaxAttempts; attempt++ {
|
||||
if err = fn(); err == nil {
|
||||
return nil
|
||||
}
|
||||
if !isRetriable(err) {
|
||||
return err
|
||||
}
|
||||
if attempt == policy.MaxAttempts {
|
||||
return fmt.Errorf("after %d attempts: %w", policy.MaxAttempts, err)
|
||||
}
|
||||
select {
|
||||
case <-time.After(policy.delay(attempt)):
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// SyncConfig wires up a sync run.
|
||||
type SyncConfig struct {
|
||||
OO *OOConfig // OnlyOffice source (optional)
|
||||
Gmail *GmailCredentials // Gmail source (optional)
|
||||
Out string // var/mail root; default <repo>/var/mail
|
||||
Workers int // concurrency; default 4
|
||||
Limit int // max messages per source (0 = all)
|
||||
Offset int // skip first N messages per source
|
||||
Force bool // overwrite existing message.json + attachments
|
||||
DryRun bool // list without writing
|
||||
Policy RetryPolicy
|
||||
}
|
||||
|
||||
// SyncStats is returned by Run.
|
||||
type SyncStats struct {
|
||||
Checked int
|
||||
New int32
|
||||
Failed int32
|
||||
Skipped int32
|
||||
}
|
||||
|
||||
// Source abstracts the two backends for the worker pool.
|
||||
type Source interface {
|
||||
// ListIDs yields ids (string form) to fetch. cursor resumes pagination.
|
||||
ListIDs(ctx context.Context, limit int, cursor string) (ids []string, next string, err error)
|
||||
Get(ctx context.Context, id string) (*Message, error)
|
||||
DownloadAttachment(ctx context.Context, msg *Message, att Attachment) ([]byte, error)
|
||||
Folder() string
|
||||
}
|
||||
|
||||
type ooSource struct {
|
||||
c *OOClient
|
||||
page int
|
||||
}
|
||||
type gmailSource struct {
|
||||
c *GmailClient
|
||||
cur string
|
||||
}
|
||||
|
||||
func (s *ooSource) Folder() string { return "inbox" }
|
||||
func (s *gmailSource) Folder() string { return "gmail" }
|
||||
|
||||
func (s *ooSource) ListIDs(ctx context.Context, limit int, cursor string) ([]string, string, error) {
|
||||
page := s.page
|
||||
if page == 0 {
|
||||
page = 1
|
||||
}
|
||||
ids, next, err := s.c.ListIDs(ctx, limit, page)
|
||||
s.page = next
|
||||
strs := make([]string, len(ids))
|
||||
for i, id := range ids {
|
||||
strs[i] = fmt.Sprintf("%d", id)
|
||||
}
|
||||
return strs, "", err
|
||||
}
|
||||
|
||||
func (s *ooSource) Get(ctx context.Context, id string) (*Message, error) {
|
||||
var mid int
|
||||
if _, err := fmt.Sscanf(id, "%d", &mid); err != nil {
|
||||
return nil, fmt.Errorf("oo id %q: %w", id, err)
|
||||
}
|
||||
return s.c.GetMessage(ctx, mid)
|
||||
}
|
||||
|
||||
func (s *ooSource) DownloadAttachment(ctx context.Context, msg *Message, att Attachment) ([]byte, error) {
|
||||
return s.c.DownloadAttachment(ctx, att.FileID)
|
||||
}
|
||||
|
||||
func (s *gmailSource) ListIDs(ctx context.Context, limit int, cursor string) ([]string, string, error) {
|
||||
ids, next, err := s.c.ListIDs(ctx, "in:inbox", limit, cursor)
|
||||
return ids, next, err
|
||||
}
|
||||
|
||||
func (s *gmailSource) Get(ctx context.Context, id string) (*Message, error) {
|
||||
return s.c.GetMessage(ctx, id)
|
||||
}
|
||||
|
||||
func (s *gmailSource) DownloadAttachment(ctx context.Context, msg *Message, att Attachment) ([]byte, error) {
|
||||
return s.c.DownloadAttachment(ctx, msg.ID, att.FileID)
|
||||
}
|
||||
|
||||
// Run executes the sync across the configured sources with a worker pool.
|
||||
func Run(ctx context.Context, cfg SyncConfig) (*SyncStats, error) {
|
||||
if cfg.Out == "" {
|
||||
cfg.Out = "var/mail"
|
||||
}
|
||||
if cfg.Workers <= 0 {
|
||||
cfg.Workers = 4
|
||||
}
|
||||
if err := os.MkdirAll(cfg.Out, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var sources []Source
|
||||
if cfg.OO != nil {
|
||||
oo, err := NewOOClient(*cfg.OO, 1) // folder inbox
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("onlyoffice auth: %w", err)
|
||||
}
|
||||
sources = append(sources, &ooSource{c: oo})
|
||||
}
|
||||
if cfg.Gmail != nil {
|
||||
gm, err := NewGmailClient(*cfg.Gmail)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("gmail init: %w", err)
|
||||
}
|
||||
sources = append(sources, &gmailSource{c: gm})
|
||||
}
|
||||
if len(sources) == 0 {
|
||||
return nil, errors.New("sync: no source configured (need OO, Gmail, or both)")
|
||||
}
|
||||
|
||||
stats := &SyncStats{}
|
||||
var jobs []struct {
|
||||
src Source
|
||||
id string
|
||||
}
|
||||
for _, src := range sources {
|
||||
ids, _, err := src.ListIDs(ctx, cfg.Offset+cfg.Limit, "")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list %s: %w", src.Folder(), err)
|
||||
}
|
||||
if cfg.Offset > 0 {
|
||||
if cfg.Offset >= len(ids) {
|
||||
ids = nil
|
||||
} else {
|
||||
ids = ids[cfg.Offset:]
|
||||
}
|
||||
}
|
||||
if cfg.Limit > 0 && len(ids) > cfg.Limit {
|
||||
ids = ids[:cfg.Limit]
|
||||
}
|
||||
stats.Checked += len(ids)
|
||||
for _, id := range ids {
|
||||
jobs = append(jobs, struct {
|
||||
src Source
|
||||
id string
|
||||
}{src: src, id: id})
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
mu sync.Mutex
|
||||
failures []string
|
||||
)
|
||||
jobsCh := make(chan struct {
|
||||
src Source
|
||||
id string
|
||||
})
|
||||
for i := 0; i < cfg.Workers; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for j := range jobsCh {
|
||||
status, err := processOne(ctx, j.src, j.id, cfg)
|
||||
switch status {
|
||||
case statusFailed:
|
||||
mu.Lock()
|
||||
failures = append(failures, j.src.Folder()+"/"+j.id+": "+err.Error())
|
||||
mu.Unlock()
|
||||
atomic.AddInt32(&stats.Failed, 1)
|
||||
case statusNew:
|
||||
atomic.AddInt32(&stats.New, 1)
|
||||
case statusSkipped:
|
||||
atomic.AddInt32(&stats.Skipped, 1)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
for _, j := range jobs {
|
||||
select {
|
||||
case jobsCh <- j:
|
||||
case <-ctx.Done():
|
||||
close(jobsCh)
|
||||
wg.Wait()
|
||||
return stats, ctx.Err()
|
||||
}
|
||||
}
|
||||
close(jobsCh)
|
||||
wg.Wait()
|
||||
|
||||
if len(failures) > 0 {
|
||||
fmt.Fprintf(os.Stderr, "sync: %d failures:\n %s\n", len(failures), strings.Join(failures, "\n "))
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
type status int
|
||||
|
||||
const (
|
||||
statusNew status = iota
|
||||
statusSkipped
|
||||
statusFailed
|
||||
)
|
||||
|
||||
func processOne(ctx context.Context, src Source, id string, cfg SyncConfig) (status, error) {
|
||||
if cfg.DryRun {
|
||||
return statusNew, nil
|
||||
}
|
||||
dir := filepath.Join(cfg.Out, src.Folder(), id)
|
||||
jsonPath := filepath.Join(dir, "message.json")
|
||||
if !cfg.Force {
|
||||
if _, err := os.Stat(jsonPath); err == nil {
|
||||
return statusSkipped, nil
|
||||
}
|
||||
}
|
||||
var msg *Message
|
||||
err := Retry(ctx, cfg.Policy, func() error {
|
||||
m, err := src.Get(ctx, id)
|
||||
if err != nil {
|
||||
return retryWrap(err)
|
||||
}
|
||||
m.Folder = src.Folder() // directory layout is authoritative
|
||||
if err := writeMessage(jsonPath, m); err != nil {
|
||||
return err
|
||||
}
|
||||
msg = m
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return statusFailed, err
|
||||
}
|
||||
for _, att := range msg.Attachments {
|
||||
attDir := filepath.Join(dir, "attachments")
|
||||
if err := os.MkdirAll(attDir, 0o755); err != nil {
|
||||
return statusFailed, err
|
||||
}
|
||||
attPath := filepath.Join(attDir, sanitize(att.StoredName))
|
||||
if _, err := os.Stat(attPath); err == nil && !cfg.Force {
|
||||
continue
|
||||
}
|
||||
var data []byte
|
||||
err := Retry(ctx, cfg.Policy, func() error {
|
||||
b, err := src.DownloadAttachment(ctx, msg, att)
|
||||
if err != nil {
|
||||
return retryWrap(err)
|
||||
}
|
||||
data = b
|
||||
return os.WriteFile(attPath, b, 0o644)
|
||||
})
|
||||
if err != nil {
|
||||
return statusFailed, fmt.Errorf("attachment %s: %w", att.FileName, err)
|
||||
}
|
||||
// ICS attachments get structured markdown immediately (same name the
|
||||
// Python converter would use: <display stem>.md).
|
||||
if isICS(att.FileName) {
|
||||
stem := att.FileName
|
||||
if i := strings.LastIndex(stem, "."); i >= 0 {
|
||||
stem = stem[:i]
|
||||
}
|
||||
mdPath := filepath.Join(attDir, sanitize(stem)+".md")
|
||||
if err := os.WriteFile(mdPath, []byte(ICSToMarkdown(data)), 0o644); err != nil {
|
||||
return statusFailed, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return statusNew, nil
|
||||
}
|
||||
|
||||
func writeMessage(path string, m *Message) error {
|
||||
b, err := json.MarshalIndent(m, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, b, 0o644)
|
||||
}
|
||||
|
||||
func sanitize(name string) string {
|
||||
r := strings.NewReplacer("/", "_", "\\", "_", ":", "_", "*", "_", "?", "_", "\"", "_",
|
||||
"<", "_", ">", "_", "|", "_", " ", "_")
|
||||
return r.Replace(name)
|
||||
}
|
||||
|
||||
func isICS(name string) bool {
|
||||
n := strings.ToLower(name)
|
||||
return strings.HasSuffix(n, ".ics") || strings.HasSuffix(n, ".ical")
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
func TestRetrySucceedsOnSecondTry(t *testing.T) {
|
||||
attempts := 0
|
||||
err := Retry(context.Background(), RetryPolicy{BaseDelay: time.Millisecond, MaxDelay: 5 * time.Millisecond}, func() error {
|
||||
attempts++
|
||||
if attempts == 1 {
|
||||
return retryWrap(errors.New("boom"))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if attempts != 2 {
|
||||
t.Fatalf("expected 2 attempts, got %d", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryExhaustsAttempts(t *testing.T) {
|
||||
attempts := 0
|
||||
err := Retry(context.Background(), RetryPolicy{MaxAttempts: 3, BaseDelay: time.Millisecond, MaxDelay: 5 * time.Millisecond}, func() error {
|
||||
attempts++
|
||||
return retryWrap(errors.New("nope"))
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected error after exhaustion")
|
||||
}
|
||||
if attempts != 3 {
|
||||
t.Fatalf("expected 3 attempts, got %d", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryNonRetriableAbortsImmediately(t *testing.T) {
|
||||
attempts := 0
|
||||
err := Retry(context.Background(), RetryPolicy{MaxAttempts: 5, BaseDelay: time.Millisecond}, func() error {
|
||||
attempts++
|
||||
return errors.New("permanent")
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
if attempts != 1 {
|
||||
t.Fatalf("expected 1 attempt for non-retriable, got %d", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryRespectsContext(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
err := Retry(ctx, RetryPolicy{MaxAttempts: 5, BaseDelay: time.Millisecond}, func() error {
|
||||
return retryWrap(errors.New("x"))
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("expected context.Canceled, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelayGrows(t *testing.T) {
|
||||
p := RetryPolicy{BaseDelay: time.Second, MaxDelay: 30 * time.Second, Jitter: 0}
|
||||
d1 := p.delay(1) // attempt 1 => 0
|
||||
d2 := p.delay(2)
|
||||
d3 := p.delay(3)
|
||||
if d1 != 0 {
|
||||
t.Fatalf("attempt 1 delay should be 0, got %v", d1)
|
||||
}
|
||||
if d2 != time.Second {
|
||||
t.Fatalf("attempt 2 delay should be 1s, got %v", d2)
|
||||
}
|
||||
if d3 != 2*time.Second {
|
||||
t.Fatalf("attempt 3 delay should be 2s, got %v", d3)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitize(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"a/b\\c:d*e": "a_b_c_d_e",
|
||||
"normal.txt": "normal.txt",
|
||||
"../evil": ".._evil",
|
||||
"a b c.pdf": "a_b_c.pdf",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := sanitize(in); got != want {
|
||||
t.Errorf("sanitize(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsICS(t *testing.T) {
|
||||
if !isICS("reply.ics") || !isICS("x.ICAL") {
|
||||
t.Fatal("ics extensions not detected")
|
||||
}
|
||||
if isICS("invoice.pdf") {
|
||||
t.Fatal("pdf misdetected as ics")
|
||||
}
|
||||
}
|
||||
|
||||
const fixtureReplyICS = `BEGIN:VCALENDAR
|
||||
METHOD:REPLY
|
||||
PRODID:Microsoft Exchange Server 2010
|
||||
VERSION:2.0
|
||||
BEGIN:VTIMEZONE
|
||||
TZID:W. Europe Standard Time
|
||||
BEGIN:STANDARD
|
||||
DTSTART:16010101T030000
|
||||
TZOFFSETFROM:+0200
|
||||
TZOFFSETTO:+0100
|
||||
RRULE:FREQ=YEARLY;INTERVAL=1;BYDAY=-1SU;BYMONTH=10
|
||||
END:STANDARD
|
||||
BEGIN:DAYLIGHT
|
||||
DTSTART:16010101T020000
|
||||
TZOFFSETFROM:+0100
|
||||
TZOFFSETTO:+0200
|
||||
RRULE:FREQ=YEARLY;INTERVAL=1;BYDAY=-1SU;BYMONTH=3
|
||||
END:DAYLIGHT
|
||||
END:VTIMEZONE
|
||||
BEGIN:VEVENT
|
||||
ATTENDEE;PARTSTAT=ACCEPTED;CN="Baker, Ben":mailto:bbaker1@teksystems.com
|
||||
UID:bvlnr1i35ug30kn6rvu9dop00g@google.com
|
||||
SUMMARY;LANGUAGE=en-US:Accepted: Appointment (Ben Baker)
|
||||
DTSTART;TZID=W. Europe Standard Time:20260812T120000
|
||||
DTEND;TZID=W. Europe Standard Time:20260812T123000
|
||||
CLASS:PUBLIC
|
||||
STATUS:CONFIRMED
|
||||
LOCATION;LANGUAGE=en-US:https://meet.google.com/sxh-ubud-jrd
|
||||
END:VEVENT
|
||||
END:VCALENDAR`
|
||||
|
||||
func TestICSToMarkdown(t *testing.T) {
|
||||
out := ICSToMarkdown([]byte(fixtureReplyICS))
|
||||
for _, want := range []string{
|
||||
"Accepted: Appointment",
|
||||
"When:",
|
||||
"Where:",
|
||||
"meet.google.com",
|
||||
"Attendee:",
|
||||
"Baker, Ben",
|
||||
"Accepted",
|
||||
"Calendar method: REPLY",
|
||||
} {
|
||||
if !contains(out, want) {
|
||||
t.Errorf("output missing %q:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
if contains(out, "BEGIN:VCALENDAR") {
|
||||
t.Errorf("raw ICS leaked into markdown:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestICSToMarkdownFallback(t *testing.T) {
|
||||
out := ICSToMarkdown([]byte("not a calendar"))
|
||||
if !contains(out, "not a calendar") {
|
||||
t.Fatalf("expected raw fallback, got %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestICSToMarkdownNormalizesLatin1(t *testing.T) {
|
||||
// Real-world ICS from a rental portal: summary in UTF-8, location Latin-1
|
||||
// ("N\xfcrnberg"). The markdown output must be valid UTF-8 everywhere.
|
||||
raw := "BEGIN:VCALENDAR\r\nVERSION:2.0\r\nBEGIN:VEVENT\r\n" +
|
||||
"SUMMARY:Mietwagen-Buchung: N\xc3\xbcrnberg\r\n" +
|
||||
"LOCATION:N\xfcrnberg\r\nDTSTART:20200101T090000Z\r\nDTEND:20200101T180000Z\r\n" +
|
||||
"END:VEVENT\r\nEND:VCALENDAR\r\n"
|
||||
out := ICSToMarkdown([]byte(raw))
|
||||
if !utf8.ValidString(out) {
|
||||
t.Fatalf("output is not valid UTF-8:\n%q", out)
|
||||
}
|
||||
if !contains(out, "Nürnberg") {
|
||||
t.Errorf("expected Nürnberg in output:\n%s", out)
|
||||
}
|
||||
if contains(out, "N\xfcrnberg") {
|
||||
t.Errorf("Latin-1 bytes leaked into output:\n%q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestICSToMarkdownAllDay(t *testing.T) {
|
||||
ics := `BEGIN:VCALENDAR
|
||||
VERSION:2.0
|
||||
BEGIN:VEVENT
|
||||
UID:y@google.com
|
||||
SUMMARY:All day thing
|
||||
DTSTART;VALUE=DATE:20260815
|
||||
DTEND;VALUE=DATE:20260816
|
||||
END:VEVENT
|
||||
END:VCALENDAR`
|
||||
out := ICSToMarkdown([]byte(ics))
|
||||
if !contains(out, "All day thing") || !contains(out, "2026-08-15") {
|
||||
t.Errorf("all-day event not parsed:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectParts(t *testing.T) {
|
||||
p := gmailPart{
|
||||
MimeType: "multipart/mixed",
|
||||
Parts: []gmailPart{
|
||||
{MimeType: "multipart/alternative", Parts: []gmailPart{
|
||||
{MimeType: "text/plain", Body: gmailBody{Data: b64("plain text")}},
|
||||
{MimeType: "text/html", Body: gmailBody{Data: b64("<p>html</p>")}},
|
||||
}},
|
||||
{PartID: "2", MimeType: "application/pdf", Filename: "invoice.pdf", Body: gmailBody{Size: 100}},
|
||||
},
|
||||
}
|
||||
text, html, atts := collectParts(p, "root", "abc123", 0)
|
||||
if text != "plain text" {
|
||||
t.Errorf("text = %q", text)
|
||||
}
|
||||
if html != "<p>html</p>" {
|
||||
t.Errorf("html = %q", html)
|
||||
}
|
||||
if len(atts) != 1 || atts[0].FileName != "invoice.pdf" || atts[0].FileID != "abc123:2" {
|
||||
t.Errorf("atts = %+v", atts)
|
||||
}
|
||||
}
|
||||
|
||||
func b64(s string) string {
|
||||
return base64.URLEncoding.EncodeToString([]byte(s))
|
||||
}
|
||||
|
||||
func contains(s, sub string) bool {
|
||||
return len(s) >= len(sub) && (s == sub || len(sub) == 0 ||
|
||||
indexOf(s, sub) >= 0)
|
||||
}
|
||||
|
||||
func indexOf(s, sub string) int {
|
||||
for i := 0; i+len(sub) <= len(s); i++ {
|
||||
if s[i:i+len(sub)] == sub {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
// Package sync downloads OnlyOffice and Gmail messages to var/mail/ as raw
|
||||
// JSON + attachment files, then hands off to bin/mail/import --from-raw for
|
||||
// markdown conversion.
|
||||
//
|
||||
// On-disk schema (per message):
|
||||
//
|
||||
// var/mail/<folder>/<id>/message.json # Message (this package)
|
||||
// var/mail/<folder>/<id>/attachments/ # raw attachment bytes (storedName)
|
||||
//
|
||||
// The Message JSON is the contract shared with the Python converter. Fields
|
||||
// deliberately mirror what bin/mail/import already reads from the OnlyOffice
|
||||
// API, so conversion is source-agnostic.
|
||||
package sync
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// Attachment describes one attachment of a Message. FileID/FileName/StoredName
|
||||
// mirror OnlyOffice; Gmail fills them from its own ids. StoredName is always
|
||||
// unique (hash/attachment id) so raw files never collide.
|
||||
type Attachment struct {
|
||||
FileID string `json:"fileId,omitempty"`
|
||||
FileName string `json:"fileName"`
|
||||
StoredName string `json:"storedName"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
ContentType string `json:"contentType,omitempty"`
|
||||
}
|
||||
|
||||
// Message is the normalized record written to var/mail/<folder>/<id>/message.json.
|
||||
type Message struct {
|
||||
Source string `json:"source"` // "onlyoffice" | "gmail"
|
||||
ID string `json:"id"`
|
||||
Folder string `json:"folder"`
|
||||
Subject string `json:"subject,omitempty"`
|
||||
From string `json:"from,omitempty"`
|
||||
To string `json:"to,omitempty"`
|
||||
CC string `json:"cc,omitempty"`
|
||||
BCC string `json:"bcc,omitempty"`
|
||||
ReceivedAt time.Time `json:"receivedAt,omitempty"`
|
||||
HTMLBody string `json:"htmlBody,omitempty"`
|
||||
TextBody string `json:"textBody,omitempty"`
|
||||
HasAttachments bool `json:"hasAttachments,omitempty"`
|
||||
Attachments []Attachment `json:"attachments,omitempty"`
|
||||
MimeMessageID string `json:"mimeMessageId,omitempty"`
|
||||
}
|
||||
Reference in New Issue
Block a user