diff --git a/catalog/apply.go b/catalog/apply.go index 6589c13..bed3990 100644 --- a/catalog/apply.go +++ b/catalog/apply.go @@ -19,9 +19,21 @@ type ApplyResult struct { } // ApplyApproved creates/updates OO contacts for entries with approve=true. +// Companies are applied before persons so Org links resolve in the same run. func ApplyApproved(ctx context.Context, client *onlyoffice.Client, doc *Document, dryRun bool) (*ApplyResult, error) { res := &ApplyResult{DryRun: dryRun} - for i := range doc.Entries { + order := make([]int, 0, len(doc.Entries)) + for i, e := range doc.Entries { + if e.Kind == "company" { + order = append(order, i) + } + } + for i, e := range doc.Entries { + if e.Kind != "company" { + order = append(order, i) + } + } + for _, i := range order { e := &doc.Entries[i] if !e.Approve { res.Skipped++ diff --git a/catalog/mbox.go b/catalog/mbox.go new file mode 100644 index 0000000..e4810aa --- /dev/null +++ b/catalog/mbox.go @@ -0,0 +1,238 @@ +package catalog + +import ( + "bufio" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +// ScanOptions controls Thunderbird / mbox extraction depth. +type ScanOptions struct { + // MboxHeaders, when true, also walks mbox-like files under the root and + // extracts From/To/Cc/Reply-To addresses (headers only, no bodies). + MboxHeaders bool + // MboxMaxBytes skips individual mbox files larger than this (0 = 256 MiB default). + MboxMaxBytes int64 +} + +// ScanThunderbirdRoot finds Thunderbird profiles under root and emits person rows +// from address books (*.mab) and Gloda global-messages-db.sqlite. +func ScanThunderbirdRoot(root string) (*Document, error) { + return ScanThunderbirdRootOpts(root, ScanOptions{}) +} + +// ScanThunderbirdRootOpts is ScanThunderbirdRoot with optional mbox header pass. +func ScanThunderbirdRootOpts(root string, opts ScanOptions) (*Document, error) { + root = filepath.Clean(root) + st, err := os.Stat(root) + if err != nil { + return nil, err + } + if !st.IsDir() { + return nil, fmt.Errorf("not a directory: %s", root) + } + if opts.MboxMaxBytes <= 0 { + opts.MboxMaxBytes = 256 << 20 + } + + var entries []Entry + seenDB := map[string]struct{}{} + seenMAB := map[string]struct{}{} + seenMbox := map[string]struct{}{} + + err = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return nil + } + name := d.Name() + if d.IsDir() { + switch name { + case "Cache", "cache2", "startupCache", "OfflineCache", "minidumps", + "crashes", "safebrowsing", "thumbnails", "chrome", "extensions", + "node_modules", ".git": + return filepath.SkipDir + } + return nil + } + lower := strings.ToLower(name) + switch { + case lower == "global-messages-db.sqlite": + if _, ok := seenDB[path]; ok { + return nil + } + seenDB[path] = struct{}{} + parsed, perr := parseGlodaContacts(path) + if perr != nil { + entries = append(entries, Entry{ + ID: EntryID("person", "", filepath.Base(path)), + Kind: "person", + Name: "gloda", + Sources: []string{path}, + Zone: "private", + Role: "unknown", + Notes: "gloda_parse_error: " + perr.Error(), + Status: "new", + }) + return nil + } + entries = append(entries, parsed...) + case strings.HasSuffix(lower, ".mab"): + if _, ok := seenMAB[path]; ok { + return nil + } + seenMAB[path] = struct{}{} + parsed, perr := parseMABEmails(path) + if perr != nil { + return nil + } + entries = append(entries, parsed...) + case opts.MboxHeaders && isLikelyMboxFile(name, path): + if _, ok := seenMbox[path]; ok { + return nil + } + seenMbox[path] = struct{}{} + info, ierr := d.Info() + if ierr != nil { + return nil + } + if info.Size() > opts.MboxMaxBytes { + return nil + } + parsed, perr := parseMboxHeaderEmails(path) + if perr != nil { + return nil + } + entries = append(entries, parsed...) + } + return nil + }) + if err != nil { + return nil, err + } + + return MergeDocs(&Document{Entries: entries}), nil +} + +func isLikelyMboxFile(name, path string) bool { + lower := strings.ToLower(name) + if strings.HasSuffix(lower, ".msf") || strings.HasSuffix(lower, ".sqlite") || + strings.HasSuffix(lower, ".mab") || strings.HasSuffix(lower, ".json") || + strings.HasSuffix(lower, ".dat") || strings.HasSuffix(lower, ".ini") || + strings.HasSuffix(lower, ".log") { + return false + } + if strings.HasSuffix(lower, ".mbox") || strings.HasSuffix(lower, ".mbx") { + return true + } + // Thunderbird: …/ImapMail//INBOX or …/Mail/Local Folders/Inbox + // or …/INBOX.sbd/ + sep := string(filepath.Separator) + norm := filepath.ToSlash(path) + if strings.Contains(norm, "/ImapMail/") || strings.Contains(norm, "/Mail/") { + if !strings.Contains(name, ".") { + return true + } + known := map[string]struct{}{ + "inbox": {}, "sent": {}, "drafts": {}, "trash": {}, "archives": {}, "junk": {}, + } + if _, ok := known[lower]; ok { + return true + } + } + if strings.Contains(filepath.Base(filepath.Dir(path)), ".sbd") && !strings.Contains(name, ".") { + return true + } + _ = sep + return false +} + +// parseMboxHeaderEmails extracts addresses from From/To/Cc/Reply-To headers only. +func parseMboxHeaderEmails(path string) ([]Entry, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + emails := map[string]struct{}{} + br := bufio.NewReaderSize(f, 1<<20) + inHeaders := false + var headerBuf strings.Builder + + flushHeaders := func() { + if headerBuf.Len() == 0 { + return + } + block := headerBuf.String() + headerBuf.Reset() + for _, line := range strings.Split(block, "\n") { + lower := strings.ToLower(strings.TrimSpace(line)) + if strings.HasPrefix(lower, "from:") || strings.HasPrefix(lower, "to:") || + strings.HasPrefix(lower, "cc:") || strings.HasPrefix(lower, "reply-to:") || + strings.HasPrefix(lower, "sender:") { + for _, m := range emailRE.FindAllString(line, -1) { + em := NormalizeEmail(m) + if !noisyEmail(em) { + emails[em] = struct{}{} + } + } + } + } + } + + for { + line, err := br.ReadString('\n') + if len(line) > 0 { + // Classic mbox separator + if strings.HasPrefix(line, "From ") && (len(line) == 5 || line[5] != ':') { + flushHeaders() + inHeaders = true + continue + } + if inHeaders { + trimmed := strings.TrimRight(line, "\r\n") + if trimmed == "" { + flushHeaders() + inHeaders = false + continue + } + headerBuf.WriteString(trimmed) + headerBuf.WriteByte('\n') + // cap pathological headers + if headerBuf.Len() > 64<<10 { + flushHeaders() + inHeaders = false + } + } + } + if err == io.EOF { + flushHeaders() + break + } + if err != nil { + flushHeaders() + return nil, err + } + } + + var out []Entry + for em := range emails { + org, zone, role := classifyMailIdentity("", em) + out = append(out, Entry{ + ID: EntryID("person", em, ""), + Kind: "person", + Emails: []string{em}, + Org: org, + Sources: []string{path}, + Zone: zone, + Role: role, + Approve: false, + Status: "new", + Notes: "thunderbird_mbox_header", + }) + } + return out, nil +} diff --git a/catalog/mbox_test.go b/catalog/mbox_test.go new file mode 100644 index 0000000..67ad2f4 --- /dev/null +++ b/catalog/mbox_test.go @@ -0,0 +1,67 @@ +package catalog + +import ( + "os" + "path/filepath" + "testing" +) + +func TestParseMboxHeaderEmails(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "INBOX") + body := `From - Mon Jul 1 00:00:00 2016 +From: Axel Schaefer +To: Andriy Oblivantsev +Cc: noreply@example.com, client@stadt-example.de +Subject: test + +Body line ignored +From - Mon Jul 2 00:00:00 2016 +From: Someone +To: list@wheregroup.com + +more body +` + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + ents, err := parseMboxHeaderEmails(path) + if err != nil { + t.Fatal(err) + } + got := map[string]bool{} + for _, e := range ents { + if len(e.Emails) > 0 { + got[e.Emails[0]] = true + } + } + if !got["axel.schaefer@wheregroup.com"] || !got["andriy.oblivantsev@wheregroup.com"] { + t.Fatalf("%v", got) + } + if got["noreply@example.com"] { + t.Fatal("noreply should be filtered") + } + if !got["client@stadt-example.de"] { + t.Fatal("missing client") + } +} + +func TestScanThunderbirdRootOptsMbox(t *testing.T) { + root := t.TempDir() + imap := filepath.Join(root, "ImapMail", "mail.example.com") + if err := os.MkdirAll(imap, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(imap, "INBOX"), []byte( + "From - x\nFrom: a@wheregroup.com\nTo: b@wheregroup.com\n\nbody\n", + ), 0o644); err != nil { + t.Fatal(err) + } + doc, err := ScanThunderbirdRootOpts(root, ScanOptions{MboxHeaders: true}) + if err != nil { + t.Fatal(err) + } + if len(doc.Entries) < 2 { + t.Fatalf("%+v", doc.Entries) + } +} diff --git a/catalog/thunderbird.go b/catalog/thunderbird.go index 6f1ce0e..6b8265f 100644 --- a/catalog/thunderbird.go +++ b/catalog/thunderbird.go @@ -2,9 +2,7 @@ package catalog import ( "database/sql" - "fmt" "os" - "path/filepath" "regexp" "strings" @@ -26,12 +24,25 @@ func noisyEmail(email string) bool { noiseLocal := []string{ "noreply", "no-reply", "donotreply", "mailer-daemon", "postmaster", "bounce", "notifications", "newsletter", "unsubscribe", + "root", "admin", "abuse", "webmaster", "hostmaster", } for _, p := range noiseLocal { - if strings.Contains(local, p) { + if local == p || strings.Contains(local, p) { return true } } + // role / shared mailboxes (keep human dotted names) + roleExact := map[string]struct{}{ + "info": {}, "alle": {}, "all": {}, "office": {}, "office@": {}, + "wartung": {}, "starface": {}, "gitlab": {}, "pl": {}, "gf": {}, + "umsetzung": {}, "support": {}, "sales": {}, "billing": {}, + } + if _, ok := roleExact[local]; ok { + return true + } + if strings.HasSuffix(local, "-request") || strings.HasSuffix(local, "-owner") { + return true + } noiseDomain := []string{ "marketplace.amazon.", "reply.github.com", "users.noreply.github.com", "groups.facebook.com", "googlegroups.com", "yahoogroups.", @@ -53,78 +64,6 @@ func noisyEmail(email string) bool { return false } -// ScanThunderbirdRoot finds Thunderbird profiles under root and emits person rows -// from address books (*.mab) and Gloda global-messages-db.sqlite. -func ScanThunderbirdRoot(root string) (*Document, error) { - root = filepath.Clean(root) - st, err := os.Stat(root) - if err != nil { - return nil, err - } - if !st.IsDir() { - return nil, fmt.Errorf("not a directory: %s", root) - } - - var entries []Entry - seenDB := map[string]struct{}{} - seenMAB := map[string]struct{}{} - - err = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { - if err != nil { - return nil - } - name := d.Name() - if d.IsDir() { - switch name { - case "Cache", "cache2", "startupCache", "OfflineCache", "minidumps", - "crashes", "safebrowsing", "thumbnails", "chrome", "extensions", - "node_modules", ".git": - return filepath.SkipDir - } - return nil - } - lower := strings.ToLower(name) - switch { - case lower == "global-messages-db.sqlite": - if _, ok := seenDB[path]; ok { - return nil - } - seenDB[path] = struct{}{} - parsed, perr := parseGlodaContacts(path) - if perr != nil { - entries = append(entries, Entry{ - ID: EntryID("person", "", filepath.Base(path)), - Kind: "person", - Name: "gloda", - Sources: []string{path}, - Zone: "private", - Role: "unknown", - Notes: "gloda_parse_error: " + perr.Error(), - Status: "new", - }) - return nil - } - entries = append(entries, parsed...) - case strings.HasSuffix(lower, ".mab"): - if _, ok := seenMAB[path]; ok { - return nil - } - seenMAB[path] = struct{}{} - parsed, perr := parseMABEmails(path) - if perr != nil { - return nil - } - entries = append(entries, parsed...) - } - return nil - }) - if err != nil { - return nil, err - } - - return MergeDocs(&Document{Entries: entries}), nil -} - func parseMABEmails(path string) ([]Entry, error) { b, err := os.ReadFile(path) if err != nil { diff --git a/cmd/oo/catalog.go b/cmd/oo/catalog.go index 7b01c04..c43e92f 100644 --- a/cmd/oo/catalog.go +++ b/cmd/oo/catalog.go @@ -88,12 +88,14 @@ func catalogScanProjectsCmd() *cobra.Command { func catalogScanThunderbirdCmd() *cobra.Command { var outPath string + var mboxHeaders bool cmd := &cobra.Command{ Use: "scan-thunderbird", Short: "Thunderbird profiles: abook/history.mab + Gloda SQLite contacts", Long: `Walk a directory tree for Thunderbird profiles and extract person rows from: - *.mab address books (email regex) - global-messages-db.sqlite Gloda contacts/identities + - optional --mbox-headers: From/To/Cc/Reply-To from mbox folder files (no bodies) Noisy senders (noreply, Amazon marketplace, GitHub reply, …) are skipped. Default zone is private; known work domains (e.g. wheregroup.com) get zone=warm role=work.`, @@ -102,7 +104,7 @@ Default zone is private; known work domains (e.g. wheregroup.com) get zone=warm if root == "" { return fmt.Errorf("--root is required") } - doc, err := catalog.ScanThunderbirdRoot(root) + doc, err := catalog.ScanThunderbirdRootOpts(root, catalog.ScanOptions{MboxHeaders: mboxHeaders}) if err != nil { return err } @@ -110,6 +112,7 @@ Default zone is private; known work domains (e.g. wheregroup.com) get zone=warm }, } cmd.Flags().String("root", "", "Thunderbird profile or parent directory (local path)") + cmd.Flags().BoolVar(&mboxHeaders, "mbox-headers", false, "also extract emails from mbox From/To/Cc headers") cmd.Flags().StringVarP(&outPath, "out", "O", "", "write YAML to this path") _ = cmd.MarkFlagRequired("root") return cmd diff --git a/crm.go b/crm.go index 0807c6b..54501c5 100644 --- a/crm.go +++ b/crm.go @@ -46,9 +46,12 @@ func (c *Client) GetContact(ctx context.Context, contactID string) (map[string]a } // FindCompany searches for a company contact with an exact (case-insensitive) -// displayName match. Returns nil when not found. +// displayName match after slogan/legal-suffix normalization. Returns nil when not found. func (c *Client) FindCompany(ctx context.Context, name string) (map[string]any, error) { needle := CompanyGroupingKey(name) + if needle == "" { + return nil, nil + } const page = 50 for start := 0; ; start += page { items, total, err := c.ListContacts(ctx, page, start, name) @@ -67,6 +70,19 @@ func (c *Client) FindCompany(ctx context.Context, name string) (map[string]any, break } } + // Search filter may miss longer legal names; fall back to a full company scan. + all, err := c.ListAllContacts(ctx) + if err != nil { + return nil, err + } + for _, co := range all { + if !isCompany(co) { + continue + } + if CompanyGroupingKey(fmt.Sprint(co["displayName"])) == needle { + return co, nil + } + } return nil, nil } diff --git a/crm_normalize.go b/crm_normalize.go index 08bef81..31e0bcc 100644 --- a/crm_normalize.go +++ b/crm_normalize.go @@ -10,6 +10,9 @@ var multiSpace = regexp.MustCompile(`\s+`) // sloganSeparators split a company name from a trailing tagline/slogan. var sloganSeparators = []string{" — ", " – ", " - ", "—", "–"} +// companyLegalSuffix matches trailing legal-form tokens (GmbH, & Co. KG, …). +var companyLegalSuffix = regexp.MustCompile(`(?i)[\s,]*(&\s*co\.?\s*kg\.?|co\.?\s*kg\.?|gmbh|ag|ltd\.?|llc\.?|inc\.?|plc\.?|s\.?l\.?|sarl|b\.?v\.?)\s*$`) + // StripSloganSuffix returns the part before an em/en dash tagline, e.g. // "Affirm — Fraud Engineering" → "Affirm". func StripSloganSuffix(s string) string { @@ -22,9 +25,23 @@ func StripSloganSuffix(s string) string { return s } -// CompanyGroupingKey normalizes a company name for dedupe (ignores slogans). +// StripCompanyLegalSuffix removes trailing GmbH / & Co. KG / Ltd / … tokens. +func StripCompanyLegalSuffix(s string) string { + s = strings.TrimSpace(s) + s = strings.TrimRight(s, ",; ") + for { + next := companyLegalSuffix.ReplaceAllString(s, "") + next = strings.TrimRight(strings.TrimSpace(next), ",; ") + if next == s || next == "" { + return s + } + s = next + } +} + +// CompanyGroupingKey normalizes a company name for dedupe (ignores slogans and legal suffixes). func CompanyGroupingKey(s string) string { - return NormalizeCompanyName(StripSloganSuffix(s)) + return NormalizeCompanyName(StripCompanyLegalSuffix(StripSloganSuffix(s))) } // NormalizeCompanyName lowercases and collapses whitespace for grouping. diff --git a/crm_normalize_test.go b/crm_normalize_test.go index 8ce33e6..96b7fcd 100644 --- a/crm_normalize_test.go +++ b/crm_normalize_test.go @@ -65,6 +65,10 @@ func TestCompanyGroupingKey(t *testing.T) { if a != b { t.Fatalf("%q != %q", a, b) } + if CompanyGroupingKey("WhereGroup") != CompanyGroupingKey("WhereGroup GmbH & Co. KG,") { + t.Fatalf("legal suffix should not change grouping key: %q vs %q", + CompanyGroupingKey("WhereGroup"), CompanyGroupingKey("WhereGroup GmbH & Co. KG,")) + } } func TestOpportunityTitlesMatchSlogan(t *testing.T) {