feat(catalog): mbox header scan, company legal-suffix match, apply order

Add --mbox-headers for Thunderbird folder files; strip GmbH/& Co. KG in
CompanyGroupingKey; FindCompany falls back to full scan; apply companies
before persons. Tightens role-mailbox noise filters.

catalog/apply.go

@@ -19,9 +19,21 @@ type ApplyResult struct {
 }
 
 // ApplyApproved creates/updates OO contacts for entries with approve=true.
+// Companies are applied before persons so Org links resolve in the same run.
 func ApplyApproved(ctx context.Context, client *onlyoffice.Client, doc *Document, dryRun bool) (*ApplyResult, error) {
 	res := &ApplyResult{DryRun: dryRun}
-	for i := range doc.Entries {
+	order := make([]int, 0, len(doc.Entries))
+	for i, e := range doc.Entries {
+		if e.Kind == "company" {
+			order = append(order, i)
+		}
+	}
+	for i, e := range doc.Entries {
+		if e.Kind != "company" {
+			order = append(order, i)
+		}
+	}
+	for _, i := range order {
 		e := &doc.Entries[i]
 		if !e.Approve {
 			res.Skipped++

catalog/mbox.go

@@ -0,0 +1,238 @@
+package catalog
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// ScanOptions controls Thunderbird / mbox extraction depth.
+type ScanOptions struct {
+	// MboxHeaders, when true, also walks mbox-like files under the root and
+	// extracts From/To/Cc/Reply-To addresses (headers only, no bodies).
+	MboxHeaders bool
+	// MboxMaxBytes skips individual mbox files larger than this (0 = 256 MiB default).
+	MboxMaxBytes int64
+}
+
+// ScanThunderbirdRoot finds Thunderbird profiles under root and emits person rows
+// from address books (*.mab) and Gloda global-messages-db.sqlite.
+func ScanThunderbirdRoot(root string) (*Document, error) {
+	return ScanThunderbirdRootOpts(root, ScanOptions{})
+}
+
+// ScanThunderbirdRootOpts is ScanThunderbirdRoot with optional mbox header pass.
+func ScanThunderbirdRootOpts(root string, opts ScanOptions) (*Document, error) {
+	root = filepath.Clean(root)
+	st, err := os.Stat(root)
+	if err != nil {
+		return nil, err
+	}
+	if !st.IsDir() {
+		return nil, fmt.Errorf("not a directory: %s", root)
+	}
+	if opts.MboxMaxBytes <= 0 {
+		opts.MboxMaxBytes = 256 << 20
+	}
+
+	var entries []Entry
+	seenDB := map[string]struct{}{}
+	seenMAB := map[string]struct{}{}
+	seenMbox := map[string]struct{}{}
+
+	err = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
+		if err != nil {
+			return nil
+		}
+		name := d.Name()
+		if d.IsDir() {
+			switch name {
+			case "Cache", "cache2", "startupCache", "OfflineCache", "minidumps",
+				"crashes", "safebrowsing", "thumbnails", "chrome", "extensions",
+				"node_modules", ".git":
+				return filepath.SkipDir
+			}
+			return nil
+		}
+		lower := strings.ToLower(name)
+		switch {
+		case lower == "global-messages-db.sqlite":
+			if _, ok := seenDB[path]; ok {
+				return nil
+			}
+			seenDB[path] = struct{}{}
+			parsed, perr := parseGlodaContacts(path)
+			if perr != nil {
+				entries = append(entries, Entry{
+					ID:      EntryID("person", "", filepath.Base(path)),
+					Kind:    "person",
+					Name:    "gloda",
+					Sources: []string{path},
+					Zone:    "private",
+					Role:    "unknown",
+					Notes:   "gloda_parse_error: " + perr.Error(),
+					Status:  "new",
+				})
+				return nil
+			}
+			entries = append(entries, parsed...)
+		case strings.HasSuffix(lower, ".mab"):
+			if _, ok := seenMAB[path]; ok {
+				return nil
+			}
+			seenMAB[path] = struct{}{}
+			parsed, perr := parseMABEmails(path)
+			if perr != nil {
+				return nil
+			}
+			entries = append(entries, parsed...)
+		case opts.MboxHeaders && isLikelyMboxFile(name, path):
+			if _, ok := seenMbox[path]; ok {
+				return nil
+			}
+			seenMbox[path] = struct{}{}
+			info, ierr := d.Info()
+			if ierr != nil {
+				return nil
+			}
+			if info.Size() > opts.MboxMaxBytes {
+				return nil
+			}
+			parsed, perr := parseMboxHeaderEmails(path)
+			if perr != nil {
+				return nil
+			}
+			entries = append(entries, parsed...)
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return MergeDocs(&Document{Entries: entries}), nil
+}
+
+func isLikelyMboxFile(name, path string) bool {
+	lower := strings.ToLower(name)
+	if strings.HasSuffix(lower, ".msf") || strings.HasSuffix(lower, ".sqlite") ||
+		strings.HasSuffix(lower, ".mab") || strings.HasSuffix(lower, ".json") ||
+		strings.HasSuffix(lower, ".dat") || strings.HasSuffix(lower, ".ini") ||
+		strings.HasSuffix(lower, ".log") {
+		return false
+	}
+	if strings.HasSuffix(lower, ".mbox") || strings.HasSuffix(lower, ".mbx") {
+		return true
+	}
+	// Thunderbird: …/ImapMail/<server>/INBOX or …/Mail/Local Folders/Inbox
+	// or …/INBOX.sbd/<folder>
+	sep := string(filepath.Separator)
+	norm := filepath.ToSlash(path)
+	if strings.Contains(norm, "/ImapMail/") || strings.Contains(norm, "/Mail/") {
+		if !strings.Contains(name, ".") {
+			return true
+		}
+		known := map[string]struct{}{
+			"inbox": {}, "sent": {}, "drafts": {}, "trash": {}, "archives": {}, "junk": {},
+		}
+		if _, ok := known[lower]; ok {
+			return true
+		}
+	}
+	if strings.Contains(filepath.Base(filepath.Dir(path)), ".sbd") && !strings.Contains(name, ".") {
+		return true
+	}
+	_ = sep
+	return false
+}
+
+// parseMboxHeaderEmails extracts addresses from From/To/Cc/Reply-To headers only.
+func parseMboxHeaderEmails(path string) ([]Entry, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	emails := map[string]struct{}{}
+	br := bufio.NewReaderSize(f, 1<<20)
+	inHeaders := false
+	var headerBuf strings.Builder
+
+	flushHeaders := func() {
+		if headerBuf.Len() == 0 {
+			return
+		}
+		block := headerBuf.String()
+		headerBuf.Reset()
+		for _, line := range strings.Split(block, "\n") {
+			lower := strings.ToLower(strings.TrimSpace(line))
+			if strings.HasPrefix(lower, "from:") || strings.HasPrefix(lower, "to:") ||
+				strings.HasPrefix(lower, "cc:") || strings.HasPrefix(lower, "reply-to:") ||
+				strings.HasPrefix(lower, "sender:") {
+				for _, m := range emailRE.FindAllString(line, -1) {
+					em := NormalizeEmail(m)
+					if !noisyEmail(em) {
+						emails[em] = struct{}{}
+					}
+				}
+			}
+		}
+	}
+
+	for {
+		line, err := br.ReadString('\n')
+		if len(line) > 0 {
+			// Classic mbox separator
+			if strings.HasPrefix(line, "From ") && (len(line) == 5 || line[5] != ':') {
+				flushHeaders()
+				inHeaders = true
+				continue
+			}
+			if inHeaders {
+				trimmed := strings.TrimRight(line, "\r\n")
+				if trimmed == "" {
+					flushHeaders()
+					inHeaders = false
+					continue
+				}
+				headerBuf.WriteString(trimmed)
+				headerBuf.WriteByte('\n')
+				// cap pathological headers
+				if headerBuf.Len() > 64<<10 {
+					flushHeaders()
+					inHeaders = false
+				}
+			}
+		}
+		if err == io.EOF {
+			flushHeaders()
+			break
+		}
+		if err != nil {
+			flushHeaders()
+			return nil, err
+		}
+	}
+
+	var out []Entry
+	for em := range emails {
+		org, zone, role := classifyMailIdentity("", em)
+		out = append(out, Entry{
+			ID:      EntryID("person", em, ""),
+			Kind:    "person",
+			Emails:  []string{em},
+			Org:     org,
+			Sources: []string{path},
+			Zone:    zone,
+			Role:    role,
+			Approve: false,
+			Status:  "new",
+			Notes:   "thunderbird_mbox_header",
+		})
+	}
+	return out, nil
+}

catalog/mbox_test.go

@@ -0,0 +1,67 @@
+package catalog
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestParseMboxHeaderEmails(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "INBOX")
+	body := `From - Mon Jul 1 00:00:00 2016
+From: Axel Schaefer <axel.schaefer@wheregroup.com>
+To: Andriy Oblivantsev <andriy.oblivantsev@wheregroup.com>
+Cc: noreply@example.com, client@stadt-example.de
+Subject: test
+
+Body line ignored
+From - Mon Jul 2 00:00:00 2016
+From: Someone <paul.schmidt@wheregroup.com>
+To: list@wheregroup.com
+
+more body
+`
+	if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	ents, err := parseMboxHeaderEmails(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := map[string]bool{}
+	for _, e := range ents {
+		if len(e.Emails) > 0 {
+			got[e.Emails[0]] = true
+		}
+	}
+	if !got["axel.schaefer@wheregroup.com"] || !got["andriy.oblivantsev@wheregroup.com"] {
+		t.Fatalf("%v", got)
+	}
+	if got["noreply@example.com"] {
+		t.Fatal("noreply should be filtered")
+	}
+	if !got["client@stadt-example.de"] {
+		t.Fatal("missing client")
+	}
+}
+
+func TestScanThunderbirdRootOptsMbox(t *testing.T) {
+	root := t.TempDir()
+	imap := filepath.Join(root, "ImapMail", "mail.example.com")
+	if err := os.MkdirAll(imap, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(imap, "INBOX"), []byte(
+		"From - x\nFrom: a@wheregroup.com\nTo: b@wheregroup.com\n\nbody\n",
+	), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	doc, err := ScanThunderbirdRootOpts(root, ScanOptions{MboxHeaders: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(doc.Entries) < 2 {
+		t.Fatalf("%+v", doc.Entries)
+	}
+}

catalog/thunderbird.go

@@ -2,9 +2,7 @@ package catalog
 
 import (
 	"database/sql"
-	"fmt"
 	"os"
-	"path/filepath"
 	"regexp"
 	"strings"
 
@@ -26,12 +24,25 @@ func noisyEmail(email string) bool {
 	noiseLocal := []string{
 		"noreply", "no-reply", "donotreply", "mailer-daemon", "postmaster",
 		"bounce", "notifications", "newsletter", "unsubscribe",
+		"root", "admin", "abuse", "webmaster", "hostmaster",
 	}
 	for _, p := range noiseLocal {
-		if strings.Contains(local, p) {
+		if local == p || strings.Contains(local, p) {
 			return true
 		}
 	}
+	// role / shared mailboxes (keep human dotted names)
+	roleExact := map[string]struct{}{
+		"info": {}, "alle": {}, "all": {}, "office": {}, "office@": {},
+		"wartung": {}, "starface": {}, "gitlab": {}, "pl": {}, "gf": {},
+		"umsetzung": {}, "support": {}, "sales": {}, "billing": {},
+	}
+	if _, ok := roleExact[local]; ok {
+		return true
+	}
+	if strings.HasSuffix(local, "-request") || strings.HasSuffix(local, "-owner") {
+		return true
+	}
 	noiseDomain := []string{
 		"marketplace.amazon.", "reply.github.com", "users.noreply.github.com",
 		"groups.facebook.com", "googlegroups.com", "yahoogroups.",
@@ -53,78 +64,6 @@ func noisyEmail(email string) bool {
 	return false
 }
 
-// ScanThunderbirdRoot finds Thunderbird profiles under root and emits person rows
-// from address books (*.mab) and Gloda global-messages-db.sqlite.
-func ScanThunderbirdRoot(root string) (*Document, error) {
-	root = filepath.Clean(root)
-	st, err := os.Stat(root)
-	if err != nil {
-		return nil, err
-	}
-	if !st.IsDir() {
-		return nil, fmt.Errorf("not a directory: %s", root)
-	}
-
-	var entries []Entry
-	seenDB := map[string]struct{}{}
-	seenMAB := map[string]struct{}{}
-
-	err = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
-		if err != nil {
-			return nil
-		}
-		name := d.Name()
-		if d.IsDir() {
-			switch name {
-			case "Cache", "cache2", "startupCache", "OfflineCache", "minidumps",
-				"crashes", "safebrowsing", "thumbnails", "chrome", "extensions",
-				"node_modules", ".git":
-				return filepath.SkipDir
-			}
-			return nil
-		}
-		lower := strings.ToLower(name)
-		switch {
-		case lower == "global-messages-db.sqlite":
-			if _, ok := seenDB[path]; ok {
-				return nil
-			}
-			seenDB[path] = struct{}{}
-			parsed, perr := parseGlodaContacts(path)
-			if perr != nil {
-				entries = append(entries, Entry{
-					ID:      EntryID("person", "", filepath.Base(path)),
-					Kind:    "person",
-					Name:    "gloda",
-					Sources: []string{path},
-					Zone:    "private",
-					Role:    "unknown",
-					Notes:   "gloda_parse_error: " + perr.Error(),
-					Status:  "new",
-				})
-				return nil
-			}
-			entries = append(entries, parsed...)
-		case strings.HasSuffix(lower, ".mab"):
-			if _, ok := seenMAB[path]; ok {
-				return nil
-			}
-			seenMAB[path] = struct{}{}
-			parsed, perr := parseMABEmails(path)
-			if perr != nil {
-				return nil
-			}
-			entries = append(entries, parsed...)
-		}
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return MergeDocs(&Document{Entries: entries}), nil
-}
-
 func parseMABEmails(path string) ([]Entry, error) {
 	b, err := os.ReadFile(path)
 	if err != nil {

cmd/oo/catalog.go

@@ -88,12 +88,14 @@ func catalogScanProjectsCmd() *cobra.Command {
 
 func catalogScanThunderbirdCmd() *cobra.Command {
 	var outPath string
+	var mboxHeaders bool
 	cmd := &cobra.Command{
 		Use:   "scan-thunderbird",
 		Short: "Thunderbird profiles: abook/history.mab + Gloda SQLite contacts",
 		Long: `Walk a directory tree for Thunderbird profiles and extract person rows from:
   - *.mab address books (email regex)
   - global-messages-db.sqlite Gloda contacts/identities
+  - optional --mbox-headers: From/To/Cc/Reply-To from mbox folder files (no bodies)
 
 Noisy senders (noreply, Amazon marketplace, GitHub reply, …) are skipped.
 Default zone is private; known work domains (e.g. wheregroup.com) get zone=warm role=work.`,
@@ -102,7 +104,7 @@ Default zone is private; known work domains (e.g. wheregroup.com) get zone=warm
 			if root == "" {
 				return fmt.Errorf("--root is required")
 			}
-			doc, err := catalog.ScanThunderbirdRoot(root)
+			doc, err := catalog.ScanThunderbirdRootOpts(root, catalog.ScanOptions{MboxHeaders: mboxHeaders})
 			if err != nil {
 				return err
 			}
@@ -110,6 +112,7 @@ Default zone is private; known work domains (e.g. wheregroup.com) get zone=warm
 		},
 	}
 	cmd.Flags().String("root", "", "Thunderbird profile or parent directory (local path)")
+	cmd.Flags().BoolVar(&mboxHeaders, "mbox-headers", false, "also extract emails from mbox From/To/Cc headers")
 	cmd.Flags().StringVarP(&outPath, "out", "O", "", "write YAML to this path")
 	_ = cmd.MarkFlagRequired("root")
 	return cmd

crm.go

@@ -46,9 +46,12 @@ func (c *Client) GetContact(ctx context.Context, contactID string) (map[string]a
 }
 
 // FindCompany searches for a company contact with an exact (case-insensitive)
-// displayName match. Returns nil when not found.
+// displayName match after slogan/legal-suffix normalization. Returns nil when not found.
 func (c *Client) FindCompany(ctx context.Context, name string) (map[string]any, error) {
 	needle := CompanyGroupingKey(name)
+	if needle == "" {
+		return nil, nil
+	}
 	const page = 50
 	for start := 0; ; start += page {
 		items, total, err := c.ListContacts(ctx, page, start, name)
@@ -67,6 +70,19 @@ func (c *Client) FindCompany(ctx context.Context, name string) (map[string]any,
 			break
 		}
 	}
+	// Search filter may miss longer legal names; fall back to a full company scan.
+	all, err := c.ListAllContacts(ctx)
+	if err != nil {
+		return nil, err
+	}
+	for _, co := range all {
+		if !isCompany(co) {
+			continue
+		}
+		if CompanyGroupingKey(fmt.Sprint(co["displayName"])) == needle {
+			return co, nil
+		}
+	}
 	return nil, nil
 }
 

crm_normalize.go

@@ -10,6 +10,9 @@ var multiSpace = regexp.MustCompile(`\s+`)
 // sloganSeparators split a company name from a trailing tagline/slogan.
 var sloganSeparators = []string{" — ", " – ", " - ", "—", "–"}
 
+// companyLegalSuffix matches trailing legal-form tokens (GmbH, & Co. KG, …).
+var companyLegalSuffix = regexp.MustCompile(`(?i)[\s,]*(&\s*co\.?\s*kg\.?|co\.?\s*kg\.?|gmbh|ag|ltd\.?|llc\.?|inc\.?|plc\.?|s\.?l\.?|sarl|b\.?v\.?)\s*$`)
+
 // StripSloganSuffix returns the part before an em/en dash tagline, e.g.
 // "Affirm — Fraud Engineering" → "Affirm".
 func StripSloganSuffix(s string) string {
@@ -22,9 +25,23 @@ func StripSloganSuffix(s string) string {
 	return s
 }
 
-// CompanyGroupingKey normalizes a company name for dedupe (ignores slogans).
+// StripCompanyLegalSuffix removes trailing GmbH / & Co. KG / Ltd / … tokens.
+func StripCompanyLegalSuffix(s string) string {
+	s = strings.TrimSpace(s)
+	s = strings.TrimRight(s, ",; ")
+	for {
+		next := companyLegalSuffix.ReplaceAllString(s, "")
+		next = strings.TrimRight(strings.TrimSpace(next), ",; ")
+		if next == s || next == "" {
+			return s
+		}
+		s = next
+	}
+}
+
+// CompanyGroupingKey normalizes a company name for dedupe (ignores slogans and legal suffixes).
 func CompanyGroupingKey(s string) string {
-	return NormalizeCompanyName(StripSloganSuffix(s))
+	return NormalizeCompanyName(StripCompanyLegalSuffix(StripSloganSuffix(s)))
 }
 
 // NormalizeCompanyName lowercases and collapses whitespace for grouping.

crm_normalize_test.go

@@ -65,6 +65,10 @@ func TestCompanyGroupingKey(t *testing.T) {
 	if a != b {
 		t.Fatalf("%q != %q", a, b)
 	}
+	if CompanyGroupingKey("WhereGroup") != CompanyGroupingKey("WhereGroup GmbH & Co. KG,") {
+		t.Fatalf("legal suffix should not change grouping key: %q vs %q",
+			CompanyGroupingKey("WhereGroup"), CompanyGroupingKey("WhereGroup GmbH & Co. KG,"))
+	}
 }
 
 func TestOpportunityTitlesMatchSlogan(t *testing.T) {