Files
2dph/bin/mail/sync/onlyoffice.go
T
eSlider 678a1d1dba feat(mail): full Gmail+OnlyOffice sync, import, and brain indexing
- bin/mail/sync.go: async Go sync engine (8 workers, paginated Gmail via
  API + OnlyOffice IMAP); Gmail attachments key off body.attachmentId, not
  MIME partId; ICS sidecars Latin-1->UTF-8 normalized (TestICSToMarkdownNormalizesLatin1)
- bin/mail/import: message.json -> markdown; PDFs via pdftotext -layout
  fast path with docling subprocess fallback for the ~5% textless files
- bin/mail/index_mail: fresh-rebuild indexer (repo corpus + mail) avoiding
  ladybug WAL corruption on bulk-insert into indexed DBs; split from import
- bin/kb/index: keep FTS/VECTOR indexes across incremental runs (drop+recreate
  leaves stale backing tables killing the vector index)
- docs: README/PLAN/AGENTS cover the mail pipeline

Result: 17,835 messages -> 28,918 info leafs, FTS+HNSW healthy.
2026-08-11 21:57:38 +01:00

223 lines
6.2 KiB
Go

package sync
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/cookiejar"
"net/url"
"strings"
"time"
)
// OOConfig mirrors the .env / environment used by bin/mail/import.
type OOConfig struct {
URL string
User string
Password string
}
// OOClient is a minimal OnlyOffice API client: authentication.json for the
// bearer token plus the session cookie jar required by the .ashx download
// handler. It mirrors the endpoint contract bin/mail/import already uses.
type OOClient struct {
cfg OOConfig
client *http.Client
mu chan struct{}
token string
folderID int
}
func NewOOClient(cfg OOConfig, folderID int) (*OOClient, error) {
jar, err := cookiejar.New(nil)
if err != nil {
return nil, err
}
c := &OOClient{
cfg: cfg,
client: &http.Client{Jar: jar, Timeout: 60 * time.Second},
mu: make(chan struct{}, 1),
folderID: folderID,
}
c.mu <- struct{}{}
if err := c.authenticate(context.Background()); err != nil {
return nil, err
}
return c, nil
}
func (o *OOClient) authenticate(ctx context.Context) error {
select {
case <-o.mu:
case <-ctx.Done():
return ctx.Err()
}
defer func() { o.mu <- struct{}{} }()
body, _ := json.Marshal(map[string]any{
"userName": o.cfg.User, "password": o.cfg.Password, "type": 0,
})
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
strings.TrimRight(o.cfg.URL, "/")+"/api/2.0/authentication.json",
strings.NewReader(string(body)))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
resp, err := o.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
data, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
if resp.StatusCode < 200 || resp.StatusCode > 299 {
return fmt.Errorf("oo authenticate status %d: %s", resp.StatusCode, truncate(string(data), 200))
}
var out struct {
Response struct {
Token string `json:"token"`
} `json:"response"`
}
if err := json.Unmarshal(data, &out); err != nil {
return err
}
if out.Response.Token == "" {
return fmt.Errorf("oo authenticate: empty token")
}
o.token = out.Response.Token
return nil
}
// get performs an authenticated GET and decodes the JSON body into out.
func (o *OOClient) get(ctx context.Context, path string, out any) error {
u := strings.TrimRight(o.cfg.URL, "/") + path
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
if err != nil {
return err
}
req.Header.Set("Authorization", "Bearer "+o.token)
req.Header.Set("Accept", "application/json")
resp, err := o.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
data, _ := io.ReadAll(io.LimitReader(resp.Body, 16<<20))
if resp.StatusCode < 200 || resp.StatusCode > 299 {
return fmt.Errorf("oo %s: status %d: %s", path, resp.StatusCode, truncate(string(data), 300))
}
if out != nil {
return json.Unmarshal(data, out)
}
return nil
}
// ooMessage mirrors the OnlyOffice mail message JSON (subset we need).
type ooMessage struct {
ID int `json:"id"`
Subject string `json:"subject"`
From string `json:"from"`
To string `json:"to"`
CC string `json:"cc"`
BCC string `json:"bcc"`
ReceivedDate string `json:"receivedDate"`
HTMLBody string `json:"htmlBody"`
TextBody string `json:"textBody"`
HasAttachments bool `json:"hasAttachments"`
MimeMessageID string `json:"mimeMessageId"`
Attachments []struct {
FileID int `json:"fileId"`
FileName string `json:"fileName"`
StoredName string `json:"storedName"`
Size int64 `json:"size"`
ContentType string `json:"contentType"`
} `json:"attachments"`
}
// ListIDs returns message ids in the configured folder, paginating pages until
// maxIDs is reached (0 = all).
func (o *OOClient) ListIDs(ctx context.Context, maxIDs int, page int) (ids []int, next int, err error) {
var out struct {
Response []ooMessage `json:"response"`
}
count := 100
if maxIDs > 0 && maxIDs < count {
count = maxIDs
}
path := fmt.Sprintf("/api/2.0/mail/messages?folder=%d&page=%d&count=%d", o.folderID, page, count)
if err := o.get(ctx, path, &out); err != nil {
return nil, 0, err
}
for _, m := range out.Response {
ids = append(ids, m.ID)
if maxIDs > 0 && len(ids) >= maxIDs {
break
}
}
next = page + 1
return ids, next, nil
}
// GetMessage fetches the full message by id and normalizes into Message.
func (o *OOClient) GetMessage(ctx context.Context, id int) (*Message, error) {
var out struct {
Response ooMessage `json:"response"`
}
path := fmt.Sprintf("/api/2.0/mail/messages/%d", id)
if err := o.get(ctx, path, &out); err != nil {
return nil, err
}
m := out.Response
msg := &Message{
Source: "onlyoffice",
ID: fmt.Sprintf("%d", m.ID),
Folder: "oo",
Subject: m.Subject,
From: m.From,
To: m.To,
CC: m.CC,
BCC: m.BCC,
HTMLBody: m.HTMLBody,
TextBody: m.TextBody,
HasAttachments: m.HasAttachments,
MimeMessageID: m.MimeMessageID,
}
if t, err := time.Parse(time.RFC3339Nano, m.ReceivedDate); err == nil {
msg.ReceivedAt = t
}
for _, a := range m.Attachments {
msg.Attachments = append(msg.Attachments, Attachment{
FileID: fmt.Sprintf("%d", a.FileID),
FileName: a.FileName,
StoredName: a.StoredName,
Size: a.Size,
ContentType: a.ContentType,
})
}
return msg, nil
}
// DownloadAttachment fetches attachment bytes via the .ashx handler, which
// requires the session cookie (client.Jar) captured during authenticate().
func (o *OOClient) DownloadAttachment(ctx context.Context, fileID string) ([]byte, error) {
u := strings.TrimRight(o.cfg.URL, "/") + "/addons/mail/httphandlers/download.ashx?attachid=" + url.QueryEscape(fileID)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
if err != nil {
return nil, err
}
resp, err := o.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
data, err := io.ReadAll(io.LimitReader(resp.Body, 64<<20))
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("oo download attach %s: status %d", fileID, resp.StatusCode)
}
return data, nil
}