From 256864f613f31570cf995dddc51255a9a91a38f8 Mon Sep 17 00:00:00 2001 From: Andriy Oblivantsev Date: Tue, 18 Aug 2026 12:41:58 +0100 Subject: [PATCH 1/5] feat(catalog): port oo catalog (scan/merge/match/apply) from legacy branch The catalog feature (approved ASR-0014) was lost during the GitHub history rewrite: the merge commit on main is empty and the code only survived on the old gitea-history branches (no merge base with main). Port catalog/* and the oo catalog CLI verbatim; drop the library-only applications package whose Sync() created the 'Write application'/'Apply application'/'Send CV' CRM task spam (23k+ tasks deleted 2026-08-18). --- catalog/apply.go | 199 ++++++++++++++++++++++++ catalog/catalog_test.go | 132 ++++++++++++++++ catalog/match.go | 133 ++++++++++++++++ catalog/match_test.go | 26 ++++ catalog/mbox.go | 238 ++++++++++++++++++++++++++++ catalog/mbox_test.go | 67 ++++++++ catalog/scan_projects.go | 127 +++++++++++++++ catalog/thunderbird.go | 191 +++++++++++++++++++++++ catalog/thunderbird_test.go | 97 ++++++++++++ catalog/types.go | 211 +++++++++++++++++++++++++ catalog/vcf.go | 218 ++++++++++++++++++++++++++ cmd/oo/catalog.go | 298 ++++++++++++++++++++++++++++++++++++ go.mod | 14 +- go.sum | 57 ++++++- 14 files changed, 2002 insertions(+), 6 deletions(-) create mode 100644 catalog/apply.go create mode 100644 catalog/catalog_test.go create mode 100644 catalog/match.go create mode 100644 catalog/match_test.go create mode 100644 catalog/mbox.go create mode 100644 catalog/mbox_test.go create mode 100644 catalog/scan_projects.go create mode 100644 catalog/thunderbird.go create mode 100644 catalog/thunderbird_test.go create mode 100644 catalog/types.go create mode 100644 catalog/vcf.go create mode 100644 cmd/oo/catalog.go diff --git a/catalog/apply.go b/catalog/apply.go new file mode 100644 index 0000000..bed3990 --- /dev/null +++ b/catalog/apply.go @@ -0,0 +1,199 @@ +package catalog + +import ( + "context" + "fmt" + "strconv" + "strings" + + onlyoffice "github.com/eslider/go-onlyoffice" +) + +// ApplyResult summarizes one apply pass. +type ApplyResult struct { + Created int `json:"created"` + Updated int `json:"updated"` + Skipped int `json:"skipped"` + Errors []string `json:"errors,omitempty"` + DryRun bool `json:"dry_run"` +} + +// ApplyApproved creates/updates OO contacts for entries with approve=true. +// Companies are applied before persons so Org links resolve in the same run. +func ApplyApproved(ctx context.Context, client *onlyoffice.Client, doc *Document, dryRun bool) (*ApplyResult, error) { + res := &ApplyResult{DryRun: dryRun} + order := make([]int, 0, len(doc.Entries)) + for i, e := range doc.Entries { + if e.Kind == "company" { + order = append(order, i) + } + } + for i, e := range doc.Entries { + if e.Kind != "company" { + order = append(order, i) + } + } + for _, i := range order { + e := &doc.Entries[i] + if !e.Approve { + res.Skipped++ + continue + } + if e.Status == "conflict" { + res.Skipped++ + res.Errors = append(res.Errors, e.ID+": conflict — resolve manually") + continue + } + if dryRun { + if e.OOID != "" || e.Status == "exists" { + res.Updated++ + } else { + res.Created++ + } + continue + } + created, err := applyOne(ctx, client, e) + if err != nil { + res.Errors = append(res.Errors, e.ID+": "+err.Error()) + continue + } + if created { + res.Created++ + } else { + res.Updated++ + } + } + return res, nil +} + +func applyOne(ctx context.Context, client *onlyoffice.Client, e *Entry) (created bool, err error) { + switch e.Kind { + case "company": + return applyCompany(ctx, client, e) + case "person": + return applyPerson(ctx, client, e) + default: + return false, fmt.Errorf("unknown kind %q", e.Kind) + } +} + +func applyCompany(ctx context.Context, client *onlyoffice.Client, e *Entry) (bool, error) { + name := strings.TrimSpace(e.Name) + if name == "" { + return false, fmt.Errorf("company missing name") + } + var co map[string]any + var err error + if e.OOID != "" { + co, err = client.GetContact(ctx, e.OOID) + } else { + co, err = client.FindCompany(ctx, name) + } + if err != nil { + return false, err + } + created := false + if co == nil { + co, err = client.CreateCompany(ctx, name) + if err != nil { + return false, err + } + created = true + } + id := contactIDString(co) + e.OOID = id + if err := ensureContactInfos(ctx, client, id, e); err != nil { + return created, err + } + e.Status = "applied" + if created { + e.Notes = "created" + } else { + e.Notes = "updated" + } + return created, nil +} + +func applyPerson(ctx context.Context, client *onlyoffice.Client, e *Entry) (bool, error) { + first := strings.TrimSpace(e.First) + last := strings.TrimSpace(e.Last) + if first == "" && last == "" { + first, last = SplitDisplayName(e.Name) + } + if first == "" { + first = strings.TrimSpace(e.Name) + } + if first == "" { + return false, fmt.Errorf("person missing name") + } + if last == "" { + last = "-" + } + + var p map[string]any + var err error + if e.OOID != "" { + p, err = client.GetContact(ctx, e.OOID) + } else if len(e.Emails) > 0 { + p, err = client.FindPersonByEmail(ctx, e.Emails[0]) + } else { + p, err = client.FindPerson(ctx, first, last) + } + if err != nil { + return false, err + } + created := false + companyID := 0 + if e.Org != "" { + if co, ferr := client.FindCompany(ctx, e.Org); ferr == nil && co != nil { + companyID, _ = strconv.Atoi(contactIDString(co)) + } + } + if p == nil { + about := "" + if e.Org != "" { + about = "org: " + e.Org + } + p, err = client.CreatePerson(ctx, first, last, companyID, "", about) + if err != nil { + return false, err + } + created = true + } + id := contactIDString(p) + e.OOID = id + if err := ensureContactInfos(ctx, client, id, e); err != nil { + return created, err + } + e.Status = "applied" + if created { + e.Notes = "created" + } else { + e.Notes = "updated" + } + return created, nil +} + +func ensureContactInfos(ctx context.Context, client *onlyoffice.Client, contactID string, e *Entry) error { + existing, err := client.GetContact(ctx, contactID) + if err != nil { + return err + } + for i, em := range e.Emails { + if onlyoffice.HasContactInfo(existing, "Email", em) { + continue + } + if _, err := client.AddContactInfo(ctx, contactID, "Email", em, "Work", i == 0); err != nil { + return fmt.Errorf("add email %s: %w", em, err) + } + } + for _, ph := range e.Phones { + if onlyoffice.HasContactInfo(existing, "Phone", ph) { + continue + } + if _, err := client.AddContactInfo(ctx, contactID, "Phone", ph, "Work", false); err != nil { + return fmt.Errorf("add phone %s: %w", ph, err) + } + } + return nil +} diff --git a/catalog/catalog_test.go b/catalog/catalog_test.go new file mode 100644 index 0000000..47ed1e6 --- /dev/null +++ b/catalog/catalog_test.go @@ -0,0 +1,132 @@ +package catalog + +import ( + "os" + "path/filepath" + "testing" +) + +func TestParseVCFFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "sample.vcf") + body := `BEGIN:VCARD +VERSION:3.0 +FN:Ada Lovelace +N:Lovelace;Ada;;; +EMAIL;TYPE=INTERNET:ada@example.com +TEL;TYPE=CELL:+1-555-0100 +ORG:Analytical Engines +END:VCARD +` + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + entries, err := ParseVCFFile(path) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 { + t.Fatalf("got %d entries", len(entries)) + } + e := entries[0] + if e.Kind != "person" || e.First != "Ada" || e.Last != "Lovelace" { + t.Fatalf("unexpected entry: %+v", e) + } + if len(e.Emails) != 1 || e.Emails[0] != "ada@example.com" { + t.Fatalf("emails: %v", e.Emails) + } + if e.Org != "Analytical Engines" { + t.Fatalf("org: %q", e.Org) + } + if e.Approve { + t.Fatal("approve should default false") + } +} + +func TestMergeDocs(t *testing.T) { + a := &Document{Entries: []Entry{{ + ID: "person:ada@example.com", Kind: "person", First: "Ada", Emails: []string{"ada@example.com"}, + Sources: []string{"a.vcf"}, Zone: "private", Role: "unknown", + }}} + b := &Document{Entries: []Entry{{ + ID: "person:ada@example.com", Kind: "person", Last: "Lovelace", Phones: []string{"+1"}, + Sources: []string{"folder/Ada"}, Zone: "warm", Role: "work", Approve: true, + }}} + m := MergeDocs(a, b) + if len(m.Entries) != 1 { + t.Fatalf("len=%d", len(m.Entries)) + } + e := m.Entries[0] + if e.First != "Ada" || e.Last != "Lovelace" { + t.Fatalf("%+v", e) + } + if len(e.Sources) != 2 || len(e.Phones) != 1 || !e.Approve { + t.Fatalf("%+v", e) + } + if e.Zone != "warm" || e.Role != "work" { + t.Fatalf("zone/role: %s/%s", e.Zone, e.Role) + } +} + +func TestScanContactsRoot(t *testing.T) { + root := t.TempDir() + vcfDir := filepath.Join(root, "Contacts VCF's") + if err := os.MkdirAll(vcfDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(vcfDir, "bob.vcf"), []byte(`BEGIN:VCARD +VERSION:3.0 +FN:Bob Builder +EMAIL:bob@work.test +END:VCARD +`), 0o644); err != nil { + t.Fatal(err) + } + personDir := filepath.Join(root, "Carol Smith") + if err := os.MkdirAll(personDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(personDir, "carol@example.com.txt"), []byte(""), 0o644); err != nil { + t.Fatal(err) + } + doc, err := ScanContactsRoot(root) + if err != nil { + t.Fatal(err) + } + if len(doc.Entries) < 2 { + t.Fatalf("expected >=2 entries, got %d: %+v", len(doc.Entries), doc.Entries) + } +} + +func TestScanProjectsRoot(t *testing.T) { + root := t.TempDir() + repo := filepath.Join(root, "produktor-demo") + if err := os.MkdirAll(filepath.Join(repo, ".git"), 0o755); err != nil { + t.Fatal(err) + } + doc, err := ScanProjectsRoot(root, 3) + if err != nil { + t.Fatal(err) + } + found := false + for _, e := range doc.Entries { + if e.Kind == "company" && e.Name == "produktor-demo" { + found = true + if e.Role != "work" { + t.Fatalf("role=%q", e.Role) + } + } + } + if !found { + t.Fatalf("missing company: %+v", doc.Entries) + } +} + +func TestEntryID(t *testing.T) { + if got := EntryID("person", "A@B.COM", ""); got != "person:a@b.com" { + t.Fatal(got) + } + if got := EntryID("company", "", "Acme Corp"); got != "company:acme corp" { + t.Fatal(got) + } +} diff --git a/catalog/match.go b/catalog/match.go new file mode 100644 index 0000000..c398c5e --- /dev/null +++ b/catalog/match.go @@ -0,0 +1,133 @@ +package catalog + +import ( + "context" + "fmt" + "strconv" + "strings" + + onlyoffice "github.com/eslider/go-onlyoffice" +) + +// MatchAgainstOO sets status/oo_id by email then normalized name. +func MatchAgainstOO(ctx context.Context, client *onlyoffice.Client, doc *Document) error { + all, err := client.ListAllContacts(ctx) + if err != nil { + return err + } + + byEmail := map[string]map[string]any{} + byPersonName := map[string]map[string]any{} + byCompanyName := map[string]map[string]any{} + + for _, c := range all { + id := contactIDString(c) + isCo, _ := c["isCompany"].(bool) + for _, em := range contactEmails(c) { + byEmail[NormalizeEmail(em)] = c + } + if isCo { + key := NormalizeName(fmt.Sprint(c["displayName"])) + if key != "" { + byCompanyName[key] = c + } + continue + } + first := strings.TrimSpace(fmt.Sprint(c["firstName"])) + last := strings.TrimSpace(fmt.Sprint(c["lastName"])) + key := NormalizeName(first + " " + last) + if key == "" { + key = NormalizeName(fmt.Sprint(c["displayName"])) + } + if key != "" { + byPersonName[key] = c + } + _ = id + } + + for i := range doc.Entries { + e := &doc.Entries[i] + // preserve approve + matched := false + conflict := false + var oo map[string]any + + for _, em := range e.Emails { + if c, ok := byEmail[NormalizeEmail(em)]; ok { + if oo != nil && contactIDString(oo) != contactIDString(c) { + conflict = true + } + oo = c + matched = true + } + } + if !matched { + if e.Kind == "company" { + if c, ok := byCompanyName[NormalizeName(e.Name)]; ok { + oo = c + matched = true + } + } else { + key := NormalizeName(strings.TrimSpace(e.First + " " + e.Last)) + if key == "" { + key = NormalizeName(e.Name) + } + if c, ok := byPersonName[key]; ok { + oo = c + matched = true + } + } + } + + if conflict { + e.Status = "conflict" + e.Notes = strings.TrimSpace(e.Notes + " email_matches_multiple_oo") + if oo != nil { + e.OOID = contactIDString(oo) + } + continue + } + if matched && oo != nil { + e.Status = "exists" + e.OOID = contactIDString(oo) + continue + } + e.Status = "new" + e.OOID = "" + } + return nil +} + +func contactIDString(c map[string]any) string { + switch v := c["id"].(type) { + case float64: + return strconv.FormatInt(int64(v), 10) + case int: + return strconv.Itoa(v) + case string: + return v + default: + return fmt.Sprint(v) + } +} + +func contactEmails(c map[string]any) []string { + var out []string + if em := strings.TrimSpace(fmt.Sprint(c["email"])); em != "" && em != "" { + out = append(out, em) + } + if em := strings.TrimSpace(fmt.Sprint(c["primaryEmail"])); em != "" && em != "" { + out = append(out, em) + } + for _, row := range onlyoffice.ContactInfoRows(c) { + t := strings.ToLower(fmt.Sprint(row["infoType"])) + if t != "email" { + continue + } + data := strings.TrimSpace(fmt.Sprint(row["data"])) + if data != "" { + out = append(out, data) + } + } + return uniqueEmails(out) +} diff --git a/catalog/match_test.go b/catalog/match_test.go new file mode 100644 index 0000000..16f7e7c --- /dev/null +++ b/catalog/match_test.go @@ -0,0 +1,26 @@ +package catalog + +import ( + "testing" +) + +func TestMatchAgainstOO_emailThenName(t *testing.T) { + // MatchAgainstOO needs a live client; unit-test the helpers used by matching. + c := map[string]any{ + "id": float64(42), + "isCompany": false, + "firstName": "Ada", + "lastName": "Lovelace", + "email": "ada@example.com", + "commonData": []any{ + map[string]any{"infoType": "Email", "data": "ada.alt@example.com"}, + }, + } + if contactIDString(c) != "42" { + t.Fatal(contactIDString(c)) + } + emails := contactEmails(c) + if len(emails) < 2 { + t.Fatalf("%v", emails) + } +} diff --git a/catalog/mbox.go b/catalog/mbox.go new file mode 100644 index 0000000..e4810aa --- /dev/null +++ b/catalog/mbox.go @@ -0,0 +1,238 @@ +package catalog + +import ( + "bufio" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +// ScanOptions controls Thunderbird / mbox extraction depth. +type ScanOptions struct { + // MboxHeaders, when true, also walks mbox-like files under the root and + // extracts From/To/Cc/Reply-To addresses (headers only, no bodies). + MboxHeaders bool + // MboxMaxBytes skips individual mbox files larger than this (0 = 256 MiB default). + MboxMaxBytes int64 +} + +// ScanThunderbirdRoot finds Thunderbird profiles under root and emits person rows +// from address books (*.mab) and Gloda global-messages-db.sqlite. +func ScanThunderbirdRoot(root string) (*Document, error) { + return ScanThunderbirdRootOpts(root, ScanOptions{}) +} + +// ScanThunderbirdRootOpts is ScanThunderbirdRoot with optional mbox header pass. +func ScanThunderbirdRootOpts(root string, opts ScanOptions) (*Document, error) { + root = filepath.Clean(root) + st, err := os.Stat(root) + if err != nil { + return nil, err + } + if !st.IsDir() { + return nil, fmt.Errorf("not a directory: %s", root) + } + if opts.MboxMaxBytes <= 0 { + opts.MboxMaxBytes = 256 << 20 + } + + var entries []Entry + seenDB := map[string]struct{}{} + seenMAB := map[string]struct{}{} + seenMbox := map[string]struct{}{} + + err = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return nil + } + name := d.Name() + if d.IsDir() { + switch name { + case "Cache", "cache2", "startupCache", "OfflineCache", "minidumps", + "crashes", "safebrowsing", "thumbnails", "chrome", "extensions", + "node_modules", ".git": + return filepath.SkipDir + } + return nil + } + lower := strings.ToLower(name) + switch { + case lower == "global-messages-db.sqlite": + if _, ok := seenDB[path]; ok { + return nil + } + seenDB[path] = struct{}{} + parsed, perr := parseGlodaContacts(path) + if perr != nil { + entries = append(entries, Entry{ + ID: EntryID("person", "", filepath.Base(path)), + Kind: "person", + Name: "gloda", + Sources: []string{path}, + Zone: "private", + Role: "unknown", + Notes: "gloda_parse_error: " + perr.Error(), + Status: "new", + }) + return nil + } + entries = append(entries, parsed...) + case strings.HasSuffix(lower, ".mab"): + if _, ok := seenMAB[path]; ok { + return nil + } + seenMAB[path] = struct{}{} + parsed, perr := parseMABEmails(path) + if perr != nil { + return nil + } + entries = append(entries, parsed...) + case opts.MboxHeaders && isLikelyMboxFile(name, path): + if _, ok := seenMbox[path]; ok { + return nil + } + seenMbox[path] = struct{}{} + info, ierr := d.Info() + if ierr != nil { + return nil + } + if info.Size() > opts.MboxMaxBytes { + return nil + } + parsed, perr := parseMboxHeaderEmails(path) + if perr != nil { + return nil + } + entries = append(entries, parsed...) + } + return nil + }) + if err != nil { + return nil, err + } + + return MergeDocs(&Document{Entries: entries}), nil +} + +func isLikelyMboxFile(name, path string) bool { + lower := strings.ToLower(name) + if strings.HasSuffix(lower, ".msf") || strings.HasSuffix(lower, ".sqlite") || + strings.HasSuffix(lower, ".mab") || strings.HasSuffix(lower, ".json") || + strings.HasSuffix(lower, ".dat") || strings.HasSuffix(lower, ".ini") || + strings.HasSuffix(lower, ".log") { + return false + } + if strings.HasSuffix(lower, ".mbox") || strings.HasSuffix(lower, ".mbx") { + return true + } + // Thunderbird: …/ImapMail//INBOX or …/Mail/Local Folders/Inbox + // or …/INBOX.sbd/ + sep := string(filepath.Separator) + norm := filepath.ToSlash(path) + if strings.Contains(norm, "/ImapMail/") || strings.Contains(norm, "/Mail/") { + if !strings.Contains(name, ".") { + return true + } + known := map[string]struct{}{ + "inbox": {}, "sent": {}, "drafts": {}, "trash": {}, "archives": {}, "junk": {}, + } + if _, ok := known[lower]; ok { + return true + } + } + if strings.Contains(filepath.Base(filepath.Dir(path)), ".sbd") && !strings.Contains(name, ".") { + return true + } + _ = sep + return false +} + +// parseMboxHeaderEmails extracts addresses from From/To/Cc/Reply-To headers only. +func parseMboxHeaderEmails(path string) ([]Entry, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + emails := map[string]struct{}{} + br := bufio.NewReaderSize(f, 1<<20) + inHeaders := false + var headerBuf strings.Builder + + flushHeaders := func() { + if headerBuf.Len() == 0 { + return + } + block := headerBuf.String() + headerBuf.Reset() + for _, line := range strings.Split(block, "\n") { + lower := strings.ToLower(strings.TrimSpace(line)) + if strings.HasPrefix(lower, "from:") || strings.HasPrefix(lower, "to:") || + strings.HasPrefix(lower, "cc:") || strings.HasPrefix(lower, "reply-to:") || + strings.HasPrefix(lower, "sender:") { + for _, m := range emailRE.FindAllString(line, -1) { + em := NormalizeEmail(m) + if !noisyEmail(em) { + emails[em] = struct{}{} + } + } + } + } + } + + for { + line, err := br.ReadString('\n') + if len(line) > 0 { + // Classic mbox separator + if strings.HasPrefix(line, "From ") && (len(line) == 5 || line[5] != ':') { + flushHeaders() + inHeaders = true + continue + } + if inHeaders { + trimmed := strings.TrimRight(line, "\r\n") + if trimmed == "" { + flushHeaders() + inHeaders = false + continue + } + headerBuf.WriteString(trimmed) + headerBuf.WriteByte('\n') + // cap pathological headers + if headerBuf.Len() > 64<<10 { + flushHeaders() + inHeaders = false + } + } + } + if err == io.EOF { + flushHeaders() + break + } + if err != nil { + flushHeaders() + return nil, err + } + } + + var out []Entry + for em := range emails { + org, zone, role := classifyMailIdentity("", em) + out = append(out, Entry{ + ID: EntryID("person", em, ""), + Kind: "person", + Emails: []string{em}, + Org: org, + Sources: []string{path}, + Zone: zone, + Role: role, + Approve: false, + Status: "new", + Notes: "thunderbird_mbox_header", + }) + } + return out, nil +} diff --git a/catalog/mbox_test.go b/catalog/mbox_test.go new file mode 100644 index 0000000..67ad2f4 --- /dev/null +++ b/catalog/mbox_test.go @@ -0,0 +1,67 @@ +package catalog + +import ( + "os" + "path/filepath" + "testing" +) + +func TestParseMboxHeaderEmails(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "INBOX") + body := `From - Mon Jul 1 00:00:00 2016 +From: Axel Schaefer +To: Andriy Oblivantsev +Cc: noreply@example.com, client@stadt-example.de +Subject: test + +Body line ignored +From - Mon Jul 2 00:00:00 2016 +From: Someone +To: list@wheregroup.com + +more body +` + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + ents, err := parseMboxHeaderEmails(path) + if err != nil { + t.Fatal(err) + } + got := map[string]bool{} + for _, e := range ents { + if len(e.Emails) > 0 { + got[e.Emails[0]] = true + } + } + if !got["axel.schaefer@wheregroup.com"] || !got["andriy.oblivantsev@wheregroup.com"] { + t.Fatalf("%v", got) + } + if got["noreply@example.com"] { + t.Fatal("noreply should be filtered") + } + if !got["client@stadt-example.de"] { + t.Fatal("missing client") + } +} + +func TestScanThunderbirdRootOptsMbox(t *testing.T) { + root := t.TempDir() + imap := filepath.Join(root, "ImapMail", "mail.example.com") + if err := os.MkdirAll(imap, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(imap, "INBOX"), []byte( + "From - x\nFrom: a@wheregroup.com\nTo: b@wheregroup.com\n\nbody\n", + ), 0o644); err != nil { + t.Fatal(err) + } + doc, err := ScanThunderbirdRootOpts(root, ScanOptions{MboxHeaders: true}) + if err != nil { + t.Fatal(err) + } + if len(doc.Entries) < 2 { + t.Fatalf("%+v", doc.Entries) + } +} diff --git a/catalog/scan_projects.go b/catalog/scan_projects.go new file mode 100644 index 0000000..e8ae348 --- /dev/null +++ b/catalog/scan_projects.go @@ -0,0 +1,127 @@ +package catalog + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// ScanProjectsRoot finds git roots under root (max depth) and emits company rows. +func ScanProjectsRoot(root string, maxDepth int) (*Document, error) { + root = filepath.Clean(root) + if maxDepth <= 0 { + maxDepth = 4 + } + st, err := os.Stat(root) + if err != nil { + return nil, err + } + if !st.IsDir() { + return nil, fmt.Errorf("not a directory: %s", root) + } + + var entries []Entry + err = walkGitRoots(root, root, 0, maxDepth, &entries) + if err != nil { + return nil, err + } + + // Also top-level dirs as company stubs (even without git) — coverage 1B. + dents, _ := os.ReadDir(root) + for _, d := range dents { + if !d.IsDir() || strings.HasPrefix(d.Name(), ".") { + continue + } + name := d.Name() + skip := map[string]struct{}{ + "node_modules": {}, "vendor": {}, ".cache": {}, + } + if _, ok := skip[name]; ok { + continue + } + path := filepath.Join(root, name) + id := EntryID("company", "", name) + role, zone := classifyProjectName(name, "") + entries = append(entries, Entry{ + ID: id, + Kind: "company", + Name: name, + Sources: []string{path}, + Zone: zone, + Role: role, + Status: "new", + Notes: "top_level_dir", + }) + } + + return MergeDocs(&Document{Entries: entries}), nil +} + +func walkGitRoots(root, dir string, depth, maxDepth int, out *[]Entry) error { + if depth > maxDepth { + return nil + } + // is this a git root? + if st, err := os.Stat(filepath.Join(dir, ".git")); err == nil && (st.IsDir() || st.Mode().IsRegular()) { + name := filepath.Base(dir) + remote := gitRemoteOrigin(dir) + role, zone := classifyProjectName(name, remote) + *out = append(*out, Entry{ + ID: EntryID("company", "", name), + Kind: "company", + Name: name, + Sources: []string{dir}, + Remote: remote, + GitRoot: dir, + Zone: zone, + Role: role, + Status: "new", + }) + return nil // do not descend into nested repos from a git root + } + dents, err := os.ReadDir(dir) + if err != nil { + return nil + } + for _, d := range dents { + if !d.IsDir() { + continue + } + name := d.Name() + if name == ".git" || name == "node_modules" || name == "vendor" || name == ".venv" || name == "dist" { + continue + } + _ = walkGitRoots(root, filepath.Join(dir, name), depth+1, maxDepth, out) + } + return nil +} + +func gitRemoteOrigin(dir string) string { + cmd := exec.Command("git", "-C", dir, "remote", "get-url", "origin") + b, err := cmd.Output() + if err != nil { + return "" + } + return strings.TrimSpace(string(b)) +} + +func classifyProjectName(name, remote string) (role, zone string) { + lower := strings.ToLower(name) + remoteL := strings.ToLower(remote) + switch { + case strings.Contains(lower, "experiment") || strings.HasPrefix(lower, "test"): + return "experiment", "cold" + case lower == "mama" || lower == "personal" || strings.Contains(lower, "private"): + return "personal", "private" + case strings.Contains(remoteL, "git.produktor.io") || strings.Contains(remoteL, "github.com/eslider"): + return "work", "hot" + case strings.Contains(lower, "produktor") || strings.Contains(lower, "eslider") || + strings.Contains(lower, "asesoria") || strings.Contains(lower, "dyvenia") || + strings.Contains(lower, "onlyoffice"): + return "work", "warm" + default: + return "unknown", "warm" + } +} diff --git a/catalog/thunderbird.go b/catalog/thunderbird.go new file mode 100644 index 0000000..6b8265f --- /dev/null +++ b/catalog/thunderbird.go @@ -0,0 +1,191 @@ +package catalog + +import ( + "database/sql" + "os" + "regexp" + "strings" + + _ "modernc.org/sqlite" +) + +var emailRE = regexp.MustCompile(`(?i)[a-z0-9._%+\-]+@[a-z0-9.\-]+\.[a-z]{2,}`) + +// noisyEmail reports addresses that should not become CRM catalog persons. +func noisyEmail(email string) bool { + e := NormalizeEmail(email) + if e == "" || !strings.Contains(e, "@") { + return true + } + local, domain, ok := strings.Cut(e, "@") + if !ok { + return true + } + noiseLocal := []string{ + "noreply", "no-reply", "donotreply", "mailer-daemon", "postmaster", + "bounce", "notifications", "newsletter", "unsubscribe", + "root", "admin", "abuse", "webmaster", "hostmaster", + } + for _, p := range noiseLocal { + if local == p || strings.Contains(local, p) { + return true + } + } + // role / shared mailboxes (keep human dotted names) + roleExact := map[string]struct{}{ + "info": {}, "alle": {}, "all": {}, "office": {}, "office@": {}, + "wartung": {}, "starface": {}, "gitlab": {}, "pl": {}, "gf": {}, + "umsetzung": {}, "support": {}, "sales": {}, "billing": {}, + } + if _, ok := roleExact[local]; ok { + return true + } + if strings.HasSuffix(local, "-request") || strings.HasSuffix(local, "-owner") { + return true + } + noiseDomain := []string{ + "marketplace.amazon.", "reply.github.com", "users.noreply.github.com", + "groups.facebook.com", "googlegroups.com", "yahoogroups.", + "email.apple.com", "amazonses.com", "mailchimp.com", + "sendgrid.net", "mailgun.org", "postmarkapp.com", + "property.booking.com", "transvascular.com", + } + for _, p := range noiseDomain { + if strings.Contains(domain, p) { + return true + } + } + // random marketplace / tracking locals + if len(local) >= 16 && !strings.Contains(local, ".") && !strings.Contains(local, "-") { + if strings.Contains(domain, "amazon.") { + return true + } + } + return false +} + +func parseMABEmails(path string) ([]Entry, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + found := emailRE.FindAllString(string(b), -1) + var out []Entry + for _, raw := range found { + em := NormalizeEmail(raw) + if noisyEmail(em) { + continue + } + org, zone, role := classifyMailIdentity("", em) + out = append(out, Entry{ + ID: EntryID("person", em, ""), + Kind: "person", + Emails: []string{em}, + Org: org, + Sources: []string{path}, + Zone: zone, + Role: role, + Approve: false, + Status: "new", + Notes: "thunderbird_mab", + }) + } + return out, nil +} + +func parseGlodaContacts(dbPath string) ([]Entry, error) { + // read-only URI; immutable=1 helps when WAL/shm are missing + dsn := "file:" + dbPath + "?mode=ro&_pragma=query_only(1)" + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, err + } + defer db.Close() + + rows, err := db.Query(` + SELECT COALESCE(c.name, ''), i.value + FROM identities i + LEFT JOIN contacts c ON c.id = i.contactID + WHERE lower(i.kind) = 'email' AND i.value IS NOT NULL AND i.value != '' + `) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []Entry + for rows.Next() { + var name, value string + if err := rows.Scan(&name, &value); err != nil { + return out, err + } + em := NormalizeEmail(value) + // Gloda sometimes stores "Name " in value + if m := emailRE.FindString(em); m != "" { + em = NormalizeEmail(m) + } + if noisyEmail(em) { + continue + } + display := strings.TrimSpace(name) + // strip wrapping quotes / email leftovers + display = strings.Trim(display, `"'`) + if idx := strings.Index(display, "<"); idx > 0 { + display = strings.TrimSpace(display[:idx]) + } + first, last := "", "" + if display != "" { + first, last = SplitDisplayName(display) + } + org, zone, role := classifyMailIdentity(display, em) + out = append(out, Entry{ + ID: EntryID("person", em, display), + Kind: "person", + Name: display, + First: first, + Last: last, + Emails: []string{em}, + Org: org, + Sources: []string{dbPath}, + Zone: zone, + Role: role, + Approve: false, + Status: "new", + Notes: "thunderbird_gloda", + }) + } + return out, rows.Err() +} + +func classifyMailIdentity(name, email string) (org, zone, role string) { + em := NormalizeEmail(email) + _, domain, _ := strings.Cut(em, "@") + switch { + case domain == "wheregroup.com" || strings.Contains(strings.ToLower(name), "wheregroup"): + return "WhereGroup", "warm", "work" + case domain == "produktor.io" || domain == "eslider.de" || strings.HasSuffix(domain, ".produktor.io"): + return "produktor.io", "hot", "work" + case domain == "dyvenia.com": + return "Dyvenia", "warm", "work" + case domain == "immowelt.de" || domain == "immowelt.com": + return "Immowelt", "warm", "work" + case strings.HasSuffix(domain, ".de") && looksPublicSector(domain): + return domain, "warm", "work" + default: + return "", "private", "unknown" + } +} + +func looksPublicSector(domain string) bool { + d := strings.ToLower(domain) + hints := []string{ + "stadt-", "stadt.", "kreis-", "gemeinde", "landkreis", + "bund.", "bundes", "lvermgeo", "rvr.", "eba.", + } + for _, h := range hints { + if strings.Contains(d, h) { + return true + } + } + return false +} diff --git a/catalog/thunderbird_test.go b/catalog/thunderbird_test.go new file mode 100644 index 0000000..1b366f7 --- /dev/null +++ b/catalog/thunderbird_test.go @@ -0,0 +1,97 @@ +package catalog + +import ( + "database/sql" + "os" + "path/filepath" + "testing" + + _ "modernc.org/sqlite" +) + +func TestNoisyEmail(t *testing.T) { + if !noisyEmail("noreply@example.com") { + t.Fatal("expected noisy") + } + if !noisyEmail("x@marketplace.amazon.de") { + t.Fatal("amazon marketplace") + } + if noisyEmail("andriy.oblivantsev@wheregroup.com") { + t.Fatal("should keep wheregroup") + } +} + +func TestParseMABEmails(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "abook.mab") + body := `// mork junk + PrimaryEmail=andriy.oblivantsev@wheregroup.com + noreply@github.com + axel.schaefer@wheregroup.com +` + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + ents, err := parseMABEmails(path) + if err != nil { + t.Fatal(err) + } + if len(ents) != 2 { + t.Fatalf("got %d: %+v", len(ents), ents) + } + for _, e := range ents { + if e.Org != "WhereGroup" || e.Role != "work" { + t.Fatalf("%+v", e) + } + } +} + +func TestParseGlodaContacts(t *testing.T) { + dir := t.TempDir() + dbPath := filepath.Join(dir, "global-messages-db.sqlite") + db, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatal(err) + } + _, err = db.Exec(` + CREATE TABLE contacts (id INTEGER PRIMARY KEY, name TEXT); + CREATE TABLE identities (id INTEGER PRIMARY KEY, contactID INTEGER, kind TEXT, value TEXT); + INSERT INTO contacts VALUES (1, 'Axel Schaefer'); + INSERT INTO identities VALUES (1, 1, 'email', 'axel.schaefer@wheregroup.com'); + INSERT INTO contacts VALUES (2, 'Noise Bot'); + INSERT INTO identities VALUES (2, 2, 'email', 'noreply@example.com'); + `) + if err != nil { + t.Fatal(err) + } + _ = db.Close() + + ents, err := parseGlodaContacts(dbPath) + if err != nil { + t.Fatal(err) + } + if len(ents) != 1 { + t.Fatalf("got %d %+v", len(ents), ents) + } + if ents[0].First != "Axel" || ents[0].Emails[0] != "axel.schaefer@wheregroup.com" { + t.Fatalf("%+v", ents[0]) + } +} + +func TestScanThunderbirdRoot(t *testing.T) { + root := t.TempDir() + prof := filepath.Join(root, "profile") + if err := os.MkdirAll(prof, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(prof, "abook.mab"), []byte("mail=paul.schmidt@wheregroup.com\n"), 0o644); err != nil { + t.Fatal(err) + } + doc, err := ScanThunderbirdRoot(root) + if err != nil { + t.Fatal(err) + } + if len(doc.Entries) < 1 { + t.Fatal(doc.Entries) + } +} diff --git a/catalog/types.go b/catalog/types.go new file mode 100644 index 0000000..0a5a243 --- /dev/null +++ b/catalog/types.go @@ -0,0 +1,211 @@ +// Package catalog builds a YAML inventory of CRM persons/companies from +// filesystem contacts (VCF, folders) and project trees, then matches/applies +// against OnlyOffice CRM. +package catalog + +import ( + "fmt" + "os" + "strings" + "time" + + "gopkg.in/yaml.v3" +) + +// Entry is one catalog row (person or company). +type Entry struct { + ID string `yaml:"id"` + Kind string `yaml:"kind"` // person | company + Name string `yaml:"name,omitempty"` + First string `yaml:"first,omitempty"` + Last string `yaml:"last,omitempty"` + Emails []string `yaml:"emails,omitempty"` + Phones []string `yaml:"phones,omitempty"` + Org string `yaml:"org,omitempty"` + Sources []string `yaml:"sources,omitempty"` + Zone string `yaml:"zone"` + Role string `yaml:"role"` + OOID string `yaml:"oo_id,omitempty"` + Approve bool `yaml:"approve"` + Status string `yaml:"status,omitempty"` // new | exists | conflict | applied | skipped + Notes string `yaml:"notes,omitempty"` + Remote string `yaml:"remote,omitempty"` + GitRoot string `yaml:"git_root,omitempty"` +} + +// Document is the on-disk catalog file. +type Document struct { + GeneratedAt string `yaml:"generated_at"` + Entries []Entry `yaml:"entries"` +} + +// LoadYAML reads a catalog document from path. +func LoadYAML(path string) (*Document, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var doc Document + if err := yaml.Unmarshal(b, &doc); err != nil { + return nil, err + } + return &doc, nil +} + +// SaveYAML writes the catalog document. +func SaveYAML(path string, doc *Document) error { + doc.GeneratedAt = time.Now().UTC().Format(time.RFC3339) + b, err := yaml.Marshal(doc) + if err != nil { + return err + } + return os.WriteFile(path, b, 0o644) +} + +// NormalizeEmail lowercases and trims. +func NormalizeEmail(s string) string { + return strings.ToLower(strings.TrimSpace(s)) +} + +// NormalizeName collapses whitespace and lowercases for matching. +func NormalizeName(s string) string { + fields := strings.Fields(strings.ToLower(strings.TrimSpace(s))) + return strings.Join(fields, " ") +} + +// SplitDisplayName splits "First Last …" into first/last (last = remainder). +func SplitDisplayName(name string) (first, last string) { + parts := strings.Fields(strings.TrimSpace(name)) + if len(parts) == 0 { + return "", "" + } + if len(parts) == 1 { + return parts[0], "" + } + return parts[0], strings.Join(parts[1:], " ") +} + +// EntryID builds a stable-ish id from kind + email or name. +func EntryID(kind, email, name string) string { + if e := NormalizeEmail(email); e != "" { + return fmt.Sprintf("%s:%s", kind, e) + } + return fmt.Sprintf("%s:%s", kind, NormalizeName(name)) +} + +// MergeDocs unions entries by id; later sources append sources[] and fill empties. +func MergeDocs(docs ...*Document) *Document { + byID := map[string]*Entry{} + order := []string{} + for _, d := range docs { + if d == nil { + continue + } + for i := range d.Entries { + e := d.Entries[i] + id := e.ID + if id == "" { + primary := "" + if len(e.Emails) > 0 { + primary = e.Emails[0] + } + name := e.Name + if name == "" { + name = strings.TrimSpace(e.First + " " + e.Last) + } + id = EntryID(e.Kind, primary, name) + e.ID = id + } + if prev, ok := byID[id]; ok { + mergeEntry(prev, &e) + } else { + cp := e + byID[id] = &cp + order = append(order, id) + } + } + } + out := &Document{Entries: make([]Entry, 0, len(order))} + for _, id := range order { + out.Entries = append(out.Entries, *byID[id]) + } + return out +} + +func mergeEntry(dst, src *Entry) { + dst.Sources = uniqueStrings(append(dst.Sources, src.Sources...)) + dst.Emails = uniqueEmails(append(dst.Emails, src.Emails...)) + dst.Phones = uniqueStrings(append(dst.Phones, src.Phones...)) + if dst.First == "" { + dst.First = src.First + } + if dst.Last == "" { + dst.Last = src.Last + } + if dst.Name == "" { + dst.Name = src.Name + } + if dst.Org == "" { + dst.Org = src.Org + } + if dst.Remote == "" { + dst.Remote = src.Remote + } + if dst.GitRoot == "" { + dst.GitRoot = src.GitRoot + } + if dst.Notes == "" { + dst.Notes = src.Notes + } + // Prefer more specific zone/role from project scans over default private. + if dst.Zone == "private" && src.Zone != "" && src.Zone != "private" { + dst.Zone = src.Zone + } + if dst.Role == "" || (dst.Role == "unknown" && src.Role != "" && src.Role != "unknown") { + dst.Role = src.Role + } + // Preserve approve/oo_id/status from whichever already set. + if !dst.Approve && src.Approve { + dst.Approve = true + } + if dst.OOID == "" { + dst.OOID = src.OOID + } + if dst.Status == "" { + dst.Status = src.Status + } +} + +func uniqueEmails(in []string) []string { + seen := map[string]struct{}{} + var out []string + for _, e := range in { + e = NormalizeEmail(e) + if e == "" { + continue + } + if _, ok := seen[e]; ok { + continue + } + seen[e] = struct{}{} + out = append(out, e) + } + return out +} + +func uniqueStrings(in []string) []string { + seen := map[string]struct{}{} + var out []string + for _, s := range in { + s = strings.TrimSpace(s) + if s == "" { + continue + } + if _, ok := seen[s]; ok { + continue + } + seen[s] = struct{}{} + out = append(out, s) + } + return out +} diff --git a/catalog/vcf.go b/catalog/vcf.go new file mode 100644 index 0000000..45c6b5e --- /dev/null +++ b/catalog/vcf.go @@ -0,0 +1,218 @@ +package catalog + +import ( + "bufio" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/emersion/go-vcard" +) + +// ParseVCFFile extracts person candidates from a vCard file (may contain many cards). +func ParseVCFFile(path string) ([]Entry, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + dec := vcard.NewDecoder(f) + var out []Entry + for { + card, err := dec.Decode() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return out, fmt.Errorf("%s: %w", path, err) + } + e := entryFromCard(card, path) + if e.Name == "" && e.First == "" && len(e.Emails) == 0 { + continue + } + out = append(out, e) + } + return out, nil +} + +func entryFromCard(card vcard.Card, source string) Entry { + fn := strings.TrimSpace(card.PreferredValue(vcard.FieldFormattedName)) + n := card.Name() + first, last := "", "" + if n != nil { + first = strings.TrimSpace(n.GivenName) + last = strings.TrimSpace(n.FamilyName) + } + if fn == "" { + fn = strings.TrimSpace(first + " " + last) + } + if first == "" && last == "" && fn != "" { + first, last = SplitDisplayName(fn) + } + var emails, phones []string + for _, v := range card.Values(vcard.FieldEmail) { + if e := NormalizeEmail(v); e != "" { + emails = append(emails, e) + } + } + for _, v := range card.Values(vcard.FieldTelephone) { + p := strings.TrimSpace(v) + if p != "" { + phones = append(phones, p) + } + } + org := strings.TrimSpace(card.PreferredValue(vcard.FieldOrganization)) + primary := "" + if len(emails) > 0 { + primary = emails[0] + } + return Entry{ + ID: EntryID("person", primary, fn), + Kind: "person", + Name: fn, + First: first, + Last: last, + Emails: uniqueEmails(emails), + Phones: uniqueStrings(phones), + Org: org, + Sources: []string{source}, + Zone: "private", + Role: "unknown", + Approve: false, + Status: "new", + } +} + +// ScanContactsRoot walks a contacts directory: *.vcf anywhere, person-named +// subdirs, and *@*.txt email hint files. +func ScanContactsRoot(root string) (*Document, error) { + root = filepath.Clean(root) + st, err := os.Stat(root) + if err != nil { + return nil, err + } + if !st.IsDir() { + return nil, fmt.Errorf("not a directory: %s", root) + } + + var entries []Entry + seenVCF := map[string]struct{}{} + + err = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return nil // skip unreadable + } + if d.IsDir() { + base := d.Name() + if base == ".git" || base == "node_modules" { + return filepath.SkipDir + } + return nil + } + name := d.Name() + lower := strings.ToLower(name) + if strings.HasSuffix(lower, ".vcf") { + if _, ok := seenVCF[path]; ok { + return nil + } + seenVCF[path] = struct{}{} + parsed, perr := ParseVCFFile(path) + if perr != nil { + entries = append(entries, Entry{ + ID: EntryID("person", "", filepath.Base(path)), + Kind: "person", + Name: strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)), + Sources: []string{path}, + Zone: "private", + Role: "unknown", + Notes: "vcf_parse_error: " + perr.Error(), + Status: "new", + }) + return nil + } + entries = append(entries, parsed...) + return nil + } + // email hint files: name contains @ and ends with .txt + if strings.Contains(name, "@") && strings.HasSuffix(lower, ".txt") { + email := NormalizeEmail(strings.TrimSuffix(name, filepath.Ext(name))) + if !strings.Contains(email, "@") { + return nil + } + parent := filepath.Base(filepath.Dir(path)) + first, last := SplitDisplayName(parent) + entries = append(entries, Entry{ + ID: EntryID("person", email, parent), + Kind: "person", + Name: parent, + First: first, + Last: last, + Emails: []string{email}, + Sources: []string{path}, + Zone: "private", + Role: "unknown", + Status: "new", + }) + } + return nil + }) + if err != nil { + return nil, err + } + + // Top-level person folders without VCF still become stub persons. + dents, _ := os.ReadDir(root) + for _, d := range dents { + if !d.IsDir() { + continue + } + name := d.Name() + if name == "Contacts VCF's" || strings.HasPrefix(name, ".") { + continue + } + // skip obvious non-person dumps + lower := strings.ToLower(name) + if strings.Contains(lower, "vcf") { + continue + } + first, last := SplitDisplayName(name) + id := EntryID("person", "", name) + entries = append(entries, Entry{ + ID: id, + Kind: "person", + Name: name, + First: first, + Last: last, + Sources: []string{filepath.Join(root, name)}, + Zone: "private", + Role: "unknown", + Status: "new", + Notes: "folder_stub", + }) + } + + doc := MergeDocs(&Document{Entries: entries}) + return doc, nil +} + +// ReadEmailsFromTxt reads loose emails from a text file (one per line or free text). +func ReadEmailsFromTxt(path string) ([]string, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + var out []string + sc := bufio.NewScanner(f) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if strings.Contains(line, "@") && !strings.Contains(line, " ") { + out = append(out, NormalizeEmail(line)) + } + } + return uniqueEmails(out), sc.Err() +} diff --git a/cmd/oo/catalog.go b/cmd/oo/catalog.go new file mode 100644 index 0000000..c43e92f --- /dev/null +++ b/cmd/oo/catalog.go @@ -0,0 +1,298 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/eslider/go-onlyoffice/catalog" + "github.com/spf13/cobra" +) + +var catalogCmd = &cobra.Command{ + Use: "catalog", + Short: "Inventory clients/contacts from disk; match and apply to OO CRM", + Long: `Build a YAML catalog of persons/companies from contacts folders and project +trees, match against live OnlyOffice CRM, then create/update only rows with +approve: true. + +Typical flow: + oo catalog scan-contacts --root PATH -O /tmp/contacts.yaml + oo catalog scan-projects --root ~/projects -O /tmp/projects.yaml + oo catalog scan-thunderbird --root PATH -O /tmp/thunderbird.yaml + oo catalog merge -i /tmp/contacts.yaml -i /tmp/projects.yaml -i /tmp/thunderbird.yaml -O docs/catalog/clients-contacts.yaml + oo catalog match -i docs/catalog/clients-contacts.yaml + # edit approve: true on pilot rows + oo catalog apply --dry-run -i docs/catalog/clients-contacts.yaml + oo catalog apply --apply -i docs/catalog/clients-contacts.yaml +`} + +func init() { + rootCmd.AddCommand(catalogCmd) + catalogCmd.AddCommand(catalogScanContactsCmd()) + catalogCmd.AddCommand(catalogScanProjectsCmd()) + catalogCmd.AddCommand(catalogScanThunderbirdCmd()) + catalogCmd.AddCommand(catalogMergeCmd()) + catalogCmd.AddCommand(catalogMatchCmd()) + catalogCmd.AddCommand(catalogApplyCmd()) +} + +func catalogScanContactsCmd() *cobra.Command { + var outPath string + cmd := &cobra.Command{ + Use: "scan-contacts", + Short: "Parse VCF + folder stubs + email txt under a contacts root", + RunE: func(cmd *cobra.Command, args []string) error { + root, _ := cmd.Flags().GetString("root") + if root == "" { + return fmt.Errorf("--root is required") + } + doc, err := catalog.ScanContactsRoot(root) + if err != nil { + return err + } + return writeCatalog(cmd, doc, outPath) + }, + } + cmd.Flags().String("root", "", "contacts directory (local path)") + cmd.Flags().StringVarP(&outPath, "out", "O", "", "write YAML to this path (default: stdout summary)") + _ = cmd.MarkFlagRequired("root") + return cmd +} + +func catalogScanProjectsCmd() *cobra.Command { + var outPath string + var maxDepth int + cmd := &cobra.Command{ + Use: "scan-projects", + Short: "Git roots / remotes / top-level dirs → company rows", + RunE: func(cmd *cobra.Command, args []string) error { + root, _ := cmd.Flags().GetString("root") + if root == "" { + return fmt.Errorf("--root is required") + } + doc, err := catalog.ScanProjectsRoot(root, maxDepth) + if err != nil { + return err + } + return writeCatalog(cmd, doc, outPath) + }, + } + cmd.Flags().String("root", "", "projects directory (local path)") + cmd.Flags().IntVar(&maxDepth, "max-depth", 4, "max directory depth for git roots") + cmd.Flags().StringVarP(&outPath, "out", "O", "", "write YAML to this path") + _ = cmd.MarkFlagRequired("root") + return cmd +} + +func catalogScanThunderbirdCmd() *cobra.Command { + var outPath string + var mboxHeaders bool + cmd := &cobra.Command{ + Use: "scan-thunderbird", + Short: "Thunderbird profiles: abook/history.mab + Gloda SQLite contacts", + Long: `Walk a directory tree for Thunderbird profiles and extract person rows from: + - *.mab address books (email regex) + - global-messages-db.sqlite Gloda contacts/identities + - optional --mbox-headers: From/To/Cc/Reply-To from mbox folder files (no bodies) + +Noisy senders (noreply, Amazon marketplace, GitHub reply, …) are skipped. +Default zone is private; known work domains (e.g. wheregroup.com) get zone=warm role=work.`, + RunE: func(cmd *cobra.Command, args []string) error { + root, _ := cmd.Flags().GetString("root") + if root == "" { + return fmt.Errorf("--root is required") + } + doc, err := catalog.ScanThunderbirdRootOpts(root, catalog.ScanOptions{MboxHeaders: mboxHeaders}) + if err != nil { + return err + } + return writeCatalog(cmd, doc, outPath) + }, + } + cmd.Flags().String("root", "", "Thunderbird profile or parent directory (local path)") + cmd.Flags().BoolVar(&mboxHeaders, "mbox-headers", false, "also extract emails from mbox From/To/Cc headers") + cmd.Flags().StringVarP(&outPath, "out", "O", "", "write YAML to this path") + _ = cmd.MarkFlagRequired("root") + return cmd +} + +func catalogMergeCmd() *cobra.Command { + var inputs []string + var outPath string + cmd := &cobra.Command{ + Use: "merge", + Short: "Union + normalize catalog YAML files by entry id", + RunE: func(cmd *cobra.Command, args []string) error { + if len(inputs) == 0 { + return fmt.Errorf("at least one -i/--input required") + } + var docs []*catalog.Document + for _, p := range inputs { + d, err := catalog.LoadYAML(p) + if err != nil { + return err + } + docs = append(docs, d) + } + merged := catalog.MergeDocs(docs...) + if outPath == "" { + return fmt.Errorf("--out is required for merge") + } + if err := ensureParent(outPath); err != nil { + return err + } + if err := catalog.SaveYAML(outPath, merged); err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "merged %d entries → %s\n", len(merged.Entries), outPath) + return nil + }, + } + cmd.Flags().StringArrayVarP(&inputs, "input", "i", nil, "input catalog YAML (repeatable)") + cmd.Flags().StringVarP(&outPath, "out", "O", "", "output catalog YAML") + _ = cmd.MarkFlagRequired("out") + return cmd +} + +func catalogMatchCmd() *cobra.Command { + var inPath, outPath string + cmd := &cobra.Command{ + Use: "match", + Short: "Diff catalog vs live OO; set status new|exists|conflict and oo_id", + RunE: func(cmd *cobra.Command, args []string) error { + if inPath == "" { + return fmt.Errorf("--input is required") + } + doc, err := catalog.LoadYAML(inPath) + if err != nil { + return err + } + client, err := newOO(cmd) + if err != nil { + return err + } + if err := catalog.MatchAgainstOO(cmd.Context(), client, doc); err != nil { + return err + } + if outPath == "" { + outPath = inPath + } + if err := catalog.SaveYAML(outPath, doc); err != nil { + return err + } + counts := map[string]int{} + for _, e := range doc.Entries { + counts[e.Status]++ + } + fmt.Fprintf(cmd.OutOrStdout(), "matched → %s new=%d exists=%d conflict=%d\n", + outPath, counts["new"], counts["exists"], counts["conflict"]) + return nil + }, + } + cmd.Flags().StringVarP(&inPath, "input", "i", "", "catalog YAML") + cmd.Flags().StringVarP(&outPath, "out", "O", "", "output path (default: overwrite input)") + _ = cmd.MarkFlagRequired("input") + return cmd +} + +func catalogApplyCmd() *cobra.Command { + var inPath, outPath string + var dryRun, doApply bool + cmd := &cobra.Command{ + Use: "apply", + Short: "Create/update OO contacts for approve:true rows", + RunE: func(cmd *cobra.Command, args []string) error { + if !dryRun && !doApply { + return fmt.Errorf("pass --dry-run or --apply") + } + if dryRun && doApply { + return fmt.Errorf("use either --dry-run or --apply, not both") + } + if inPath == "" { + return fmt.Errorf("--input is required") + } + doc, err := catalog.LoadYAML(inPath) + if err != nil { + return err + } + client, err := newOO(cmd) + if err != nil { + return err + } + res, err := catalog.ApplyApproved(cmd.Context(), client, doc, dryRun) + if err != nil { + return err + } + if !dryRun { + if outPath == "" { + outPath = inPath + } + if err := catalog.SaveYAML(outPath, doc); err != nil { + return err + } + } + if outputFormat == "json" { + printJSON(res) + return nil + } + mode := "apply" + if dryRun { + mode = "dry-run" + } + fmt.Fprintf(cmd.OutOrStdout(), "%s: created=%d updated=%d skipped=%d errors=%d\n", + mode, res.Created, res.Updated, res.Skipped, len(res.Errors)) + for _, e := range res.Errors { + fmt.Fprintf(cmd.ErrOrStderr(), " error: %s\n", e) + } + return nil + }, + } + cmd.Flags().StringVarP(&inPath, "input", "i", "", "catalog YAML") + cmd.Flags().StringVarP(&outPath, "out", "O", "", "write updated catalog after apply (default: overwrite input)") + cmd.Flags().BoolVar(&dryRun, "dry-run", false, "count actions without writing to OO") + cmd.Flags().BoolVar(&doApply, "apply", false, "perform OO creates/updates") + _ = cmd.MarkFlagRequired("input") + return cmd +} + +func writeCatalog(cmd *cobra.Command, doc *catalog.Document, outPath string) error { + if outPath != "" { + if err := ensureParent(outPath); err != nil { + return err + } + if err := catalog.SaveYAML(outPath, doc); err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "wrote %d entries → %s\n", len(doc.Entries), outPath) + return nil + } + if outputFormat == "json" { + b, err := json.MarshalIndent(doc, "", " ") + if err != nil { + return err + } + fmt.Fprintln(cmd.OutOrStdout(), string(b)) + return nil + } + persons, companies := 0, 0 + for _, e := range doc.Entries { + if e.Kind == "company" { + companies++ + } else { + persons++ + } + } + fmt.Fprintf(cmd.OutOrStdout(), "entries=%d persons=%d companies=%d (pass -O PATH to write YAML)\n", + len(doc.Entries), persons, companies) + return nil +} + +func ensureParent(path string) error { + dir := filepath.Dir(path) + if dir == "" || dir == "." { + return nil + } + return os.MkdirAll(dir, 0o755) +} diff --git a/go.mod b/go.mod index 7bd630c..c4b1471 100644 --- a/go.mod +++ b/go.mod @@ -9,12 +9,15 @@ require ( github.com/charmbracelet/glamour v0.8.0 github.com/charmbracelet/lipgloss v0.12.1 github.com/charmbracelet/x/ansi v0.1.4 + github.com/emersion/go-vcard v0.0.0-20260618161152-d854b7e0e2d3 github.com/eslider/go-xls/v2 v2.1.0 github.com/google/go-querystring v1.2.0 github.com/joho/godotenv v1.5.1 github.com/mattn/go-runewidth v0.0.15 github.com/muesli/termenv v0.16.0 github.com/spf13/cobra v1.10.2 + gopkg.in/yaml.v3 v3.0.1 + modernc.org/sqlite v1.56.0 ) require ( @@ -25,22 +28,29 @@ require ( github.com/aymerick/douceur v0.2.0 // indirect github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect github.com/dlclark/regexp2 v1.11.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/lucasb-eyer/go-colorful v1.4.0 // indirect - github.com/mattn/go-isatty v0.0.22 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/microcosm-cc/bluemonday v1.0.27 // indirect github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/reflow v0.3.0 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/spf13/pflag v1.0.9 // indirect github.com/yuin/goldmark v1.8.2 // indirect github.com/yuin/goldmark-emoji v1.0.3 // indirect golang.org/x/net v0.55.0 // indirect golang.org/x/sync v0.21.0 // indirect - golang.org/x/sys v0.46.0 // indirect + golang.org/x/sys v0.47.0 // indirect golang.org/x/term v0.43.0 // indirect golang.org/x/text v0.37.0 // indirect + modernc.org/libc v1.74.4 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect ) diff --git a/go.sum b/go.sum index cbd1ee2..16adb49 100644 --- a/go.sum +++ b/go.sum @@ -33,14 +33,24 @@ github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81/go.mod h1:Yyn github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/emersion/go-vcard v0.0.0-20260618161152-d854b7e0e2d3 h1:B9YK+Tck5mTccyDhtxBzWyqGYcFxLyB6+noMNW4/VgI= +github.com/emersion/go-vcard v0.0.0-20260618161152-d854b7e0e2d3/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM= github.com/eslider/go-xls/v2 v2.1.0 h1:HszWKqYQbXxACmAXXWdMsfNl1NDBfGVBnJUPtyUHQ7A= github.com/eslider/go-xls/v2 v2.1.0/go.mod h1:xgxO6JrfuBr9jGUB+0z5l/yDmFFZ5diGk0ATGihxlMU= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.2.0 h1:yhqkPbu2/OH+V9BfpCVPZkNmUXhb2gBxJArfhIxNtP0= github.com/google/go-querystring v1.2.0/go.mod h1:8IFJqpSRITyJ8QhQ13bmbeMBDfmeEJZD5A0egEOmkqU= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -49,8 +59,8 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4= github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= -github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= -github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= @@ -66,8 +76,12 @@ github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= @@ -87,15 +101,50 @@ github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc github.com/yuin/goldmark-emoji v1.0.3 h1:aLRkLHOuBR2czCY4R8olwMjID+tENfhyFDMCRhbIQY4= github.com/yuin/goldmark-emoji v1.0.3/go.mod h1:tTkZEbwu5wkPmgTcitqddVxY9osFZiavD+r4AzQrh1U= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI= +modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= +modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= +modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= +modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.56.0 h1:/D8e2RfFqoy/Zc6PuC76U28zFwmI/sYx1Kjm4yEn9e0= +modernc.org/sqlite v1.56.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= From 8ced804a6fa9c421a5836d731fd7209e601d45c8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 19 Aug 2026 08:52:13 +0100 Subject: [PATCH 2/5] chore(main): release 0.11.0 (#24) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .release-please-manifest.json | 2 +- CHANGELOG.md | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 7d9b009..78e7f27 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.10.0" + ".": "0.11.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d5c1e7..9d489ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,13 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). * Document that invoice→deal must be set at create (`update --opportunity` often HTTP 400) +## [0.11.0](https://github.com/eSlider/go-onlyoffice/compare/v0.10.0...v0.11.0) (2026-08-18) + + +### Features + +* **catalog:** port oo catalog (scan/merge/match/apply) from legacy branch ([256864f](https://github.com/eSlider/go-onlyoffice/commit/256864f613f31570cf995dddc51255a9a91a38f8)) + ## [0.10.0](https://github.com/eSlider/go-onlyoffice/compare/v0.9.0...v0.10.0) (2026-08-13) From 10c1cfc6a7f00e0dc0e0642d631e077e010b94a7 Mon Sep 17 00:00:00 2001 From: Andrey Oblivantsev Date: Wed, 19 Aug 2026 11:12:58 +0100 Subject: [PATCH 3/5] ci: node16-safe actions + GitHub-only gates for Gitea mirror (#25) The Gitea act_runner runs action JS in node:16-bullseye; setup-go@v5 crashes with 'ReferenceError: ReadableStream is not defined'. Gitea discovers .github/workflows too, so: - setup-go@v5 -> @v4 (node16-safe, fine on GitHub) - gate GoReleaser (release.yml) + release-please jobs to github.server_url == https://github.com so the GitHub CD never runs on the Gitea mirror (GoReleaser/release-please are GitHub-release-centric) --- .github/workflows/release-please.yml | 1 + .github/workflows/release.yml | 1 + .github/workflows/test.yml | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 158c378..e5f8366 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -14,6 +14,7 @@ permissions: jobs: release-please: name: Release Please + if: github.server_url == 'https://github.com' runs-on: ubuntu-latest steps: - name: Run release-please diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f77ace4..4553864 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,6 +16,7 @@ permissions: jobs: goreleaser: name: GoReleaser + if: github.server_url == 'https://github.com' runs-on: ubuntu-latest steps: # Always clone default branch first. workflow_dispatch often races with diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6e8e58b..a5561c8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v4 with: go-version: ${{ matrix.go }} cache: true From ebbd5d5373abfeca5f160312f201cbd683f55e42 Mon Sep 17 00:00:00 2001 From: Andriy Oblivantsev Date: Thu, 20 Aug 2026 10:37:07 +0100 Subject: [PATCH 4/5] feat(files): add WebDAV-oriented Files operations Add value-oriented Files module operations for filesystem backends: ListDavFolder, ListDavSections (@root), CreateDavFolder, RenameDavFolder/ RenameDavFile, MoveDavItems, CopyDavItems, DeleteDavItems, UploadDavFile (streaming), DownloadDavFile. Dependency-free (net/http only). --- files_webdav.go | 421 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 421 insertions(+) create mode 100644 files_webdav.go diff --git a/files_webdav.go b/files_webdav.go new file mode 100644 index 0000000..f3fcafd --- /dev/null +++ b/files_webdav.go @@ -0,0 +1,421 @@ +package onlyoffice + +// WebDAV-oriented Files operations. These expose the Documents module through +// value types and cover everything needed to back a filesystem mapping: +// listing (including the virtual @root sections), folder/file CRUD, move/copy, +// and streaming upload/download. They are intentionally small and dependency +// free (only net/http), so callers are not forced to import heavier parts of +// the library. + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/url" + "strconv" + "strings" + "time" +) + +// DavFolder is a folder row from the Files module. +type DavFolder struct { + ID string + Title string + ParentID string + RootType int // 1=Common, 3=Trash, 5=My, 6=Share, 8=Projects, ... + FilesCount int + FoldersCount int + Access int + Shared bool + Updated string +} + +// DavFile is a file row from the Files module. +type DavFile struct { + ID string + Title string + Size int64 + Updated string + ViewURL string +} + +// DavListing is the contents of one folder. +type DavListing struct { + Current DavFolder + Files []DavFile + Folders []DavFolder +} + +// ListDavFolder returns the contents of a folder by id, which may be a +// symbolic root such as "@my". For "@root" use ListDavSections. +func (c *Client) ListDavFolder(ctx context.Context, id string) (*DavListing, error) { + raw, err := c.getJSON(ctx, "/api/2.0/files/"+url.PathEscape(id)) + if err != nil { + return nil, err + } + resp, err := responseField(raw, "response") + if err != nil { + return nil, err + } + // @root returns an array with a single blob; a normal folder returns an + // object. Normalize both. + if len(resp) > 0 && resp[0] == '[' { + var arr []*DavListing + if err := json.Unmarshal(resp, &arr); err != nil { + return nil, err + } + if len(arr) == 0 { + return &DavListing{}, nil + } + return arr[0], nil + } + var l DavListing + if err := json.Unmarshal(resp, &l); err != nil { + return nil, err + } + return &l, nil +} + +// ListDavSections returns the virtual top-level sections shown by @root +// ("In projects", "My documents", "Shared with me", "Common", "Favorites", +// "Recent", "Trash"). Each is the `current` folder of one @root element. +func (c *Client) ListDavSections(ctx context.Context) ([]DavFolder, error) { + raw, err := c.getJSON(ctx, "/api/2.0/files/@root") + if err != nil { + return nil, err + } + resp, err := responseField(raw, "response") + if err != nil { + return nil, err + } + var arr []struct { + Current DavFolder `json:"current"` + } + if err := json.Unmarshal(resp, &arr); err != nil { + // Tolerate a non-array (single listing) response. + var single DavListing + if err2 := json.Unmarshal(resp, &single); err2 != nil { + return nil, err + } + return []DavFolder{single.Current}, nil + } + sections := make([]DavFolder, 0, len(arr)) + for i := range arr { + sections = append(sections, arr[i].Current) + } + return sections, nil +} + +// CreateDavFolder creates a folder titled title inside parentID. +func (c *Client) CreateDavFolder(ctx context.Context, parentID, title string) (*DavFolder, error) { + raw, err := c.postJSON(ctx, "/api/2.0/files/folder/"+url.PathEscape(parentID), + map[string]string{"title": title}) + if err != nil { + return nil, err + } + var env struct { + Response *DavFolder `json:"response"` + } + if err := json.Unmarshal(raw, &env); err != nil { + return nil, err + } + if env.Response == nil { + return nil, fmt.Errorf("onlyoffice: empty create-folder response") + } + return env.Response, nil +} + +// RenameDavFolder renames a folder. +func (c *Client) RenameDavFolder(ctx context.Context, id, title string) error { + _, err := c.putJSON(ctx, "/api/2.0/files/folder/"+url.PathEscape(id), + map[string]string{"title": title}) + return err +} + +// RenameDavFile renames a file (title includes the extension). +func (c *Client) RenameDavFile(ctx context.Context, id, title string) error { + _, err := c.putJSON(ctx, "/api/2.0/files/file/"+url.PathEscape(id), + map[string]string{"title": title}) + return err +} + +// MoveDavItems moves the given folders and/or files into destFolderID. +func (c *Client) MoveDavItems(ctx context.Context, folderIDs, fileIDs []string, destFolderID string) error { + _, err := c.putJSON(ctx, "/api/2.0/files/fileops/move", map[string]any{ + "folderIds": nums(folderIDs), + "fileIds": nums(fileIDs), + "destFolderId": num(destFolderID), + "resolveType": "Skip", + "holdResult": true, + }) + return err +} + +// CopyDavItems copies the given folders and/or files into destFolderID. +func (c *Client) CopyDavItems(ctx context.Context, folderIDs, fileIDs []string, destFolderID string) error { + _, err := c.putJSON(ctx, "/api/2.0/files/fileops/copy", map[string]any{ + "folderIds": nums(folderIDs), + "fileIds": nums(fileIDs), + "destFolderId": num(destFolderID), + "conflictResolveType": "Skip", + "deleteAfter": true, + }) + return err +} + +// DeleteDavItems deletes the given folders and/or files. +func (c *Client) DeleteDavItems(ctx context.Context, folderIDs, fileIDs []string) error { + body := map[string]any{"DeleteAfter": true, "Immediately": false} + for _, id := range folderIDs { + if _, err := c.deleteJSON(ctx, "/api/2.0/files/folder/"+url.PathEscape(id), body); err != nil { + return err + } + } + for _, id := range fileIDs { + if _, err := c.deleteJSON(ctx, "/api/2.0/files/file/"+url.PathEscape(id), body); err != nil { + return err + } + } + return nil +} + +// UploadDavFile uploads src (fileName) into folderID, streaming from src. +func (c *Client) UploadDavFile(ctx context.Context, folderID, fileName string, src io.Reader) (*DavFile, error) { + raw, err := c.uploadReader(ctx, "/api/2.0/files/"+url.PathEscape(folderID)+"/upload", "file", fileName, src) + if err != nil { + return nil, err + } + var env struct { + Response *DavFile `json:"response"` + } + if err := json.Unmarshal(raw, &env); err != nil { + return nil, err + } + if env.Response == nil { + return nil, fmt.Errorf("onlyoffice: empty upload response") + } + return env.Response, nil +} + +// DownloadDavFile streams the file identified by id to w, returning bytes copied. +func (c *Client) DownloadDavFile(ctx context.Context, id string, w io.Writer) (int64, error) { + file, err := c.GetFile(ctx, id) + if err != nil { + return 0, err + } + if file.ViewURL == nil || *file.ViewURL == "" { + return 0, fmt.Errorf("onlyoffice: file %s has no viewUrl", id) + } + u := c.resolveAPIURL(*file.ViewURL) + auth, err := c.authHeader() + if err != nil { + return 0, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return 0, err + } + req.Header.Set("Authorization", auth) + resp, err := c.client.Do(req) + if err != nil { + return 0, err + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + return 0, fmt.Errorf("onlyoffice: download: %d", resp.StatusCode) + } + return io.Copy(w, resp.Body) +} + +// --- internal helpers ------------------------------------------------------- + +// deleteJSON performs an authenticated DELETE with an optional JSON body. +func (c *Client) deleteJSON(ctx context.Context, path string, body any) (json.RawMessage, error) { + var reader io.Reader + if body != nil { + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + reader = bytes.NewReader(buf) + } + auth, err := c.authHeader() + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, c.baseURL()+path, reader) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", auth) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + resp, err := c.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + raw, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + if resp.StatusCode >= 400 { + return nil, fmt.Errorf("DELETE %s: %d %s", path, resp.StatusCode, truncate(string(raw), 400)) + } + return raw, nil +} + +// uploadReader uploads a stream to path under the given form field name. +func (c *Client) uploadReader(ctx context.Context, path, fieldName, fileName string, src io.Reader) (json.RawMessage, error) { + var buf bytes.Buffer + mw := multipart.NewWriter(&buf) + part, err := mw.CreateFormFile(fieldName, fileName) + if err != nil { + return nil, err + } + if _, err := io.Copy(part, src); err != nil { + return nil, err + } + if err := mw.Close(); err != nil { + return nil, err + } + auth, err := c.authHeader() + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL()+path, &buf) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", auth) + req.Header.Set("Content-Type", mw.FormDataContentType()) + req.Header.Set("Accept", "application/json") + resp, err := c.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + raw, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + if resp.StatusCode >= 400 { + return nil, fmt.Errorf("upload %s: %d %s", path, resp.StatusCode, truncate(string(raw), 400)) + } + return raw, nil +} + +func nums(ids []string) []json.Number { + out := make([]json.Number, 0, len(ids)) + for _, id := range ids { + if _, err := strconv.Atoi(id); err == nil { + out = append(out, json.Number(id)) + } + } + return out +} + +func num(id string) any { + if _, err := strconv.Atoi(id); err == nil { + return json.Number(id) + } + return id +} + +// UnmarshalJSON decodes a folder from the portal envelope, including fields +// that the base FolderEntry omits (parentId, rootFolderType, access, ...). +func (f *DavFolder) UnmarshalJSON(b []byte) error { + var raw struct { + ID *json.Number `json:"id"` + Title *string `json:"title"` + ParentID *json.Number `json:"parentId"` + RootType *int `json:"rootFolderType"` + FilesCount *int `json:"filesCount"` + FoldersCount *int `json:"foldersCount"` + Access *int `json:"access"` + Shared *bool `json:"shared"` + Updated *string `json:"updated"` + } + if err := json.Unmarshal(b, &raw); err != nil { + return err + } + if raw.ID != nil { + f.ID = raw.ID.String() + } + if raw.Title != nil { + f.Title = *raw.Title + } + if raw.ParentID != nil { + f.ParentID = raw.ParentID.String() + } + if raw.RootType != nil { + f.RootType = *raw.RootType + } + if raw.FilesCount != nil { + f.FilesCount = *raw.FilesCount + } + if raw.FoldersCount != nil { + f.FoldersCount = *raw.FoldersCount + } + if raw.Access != nil { + f.Access = *raw.Access + } + if raw.Shared != nil { + f.Shared = *raw.Shared + } + if raw.Updated != nil { + f.Updated = *raw.Updated + } + return nil +} + +// UnmarshalJSON decodes a file row, capturing size and timestamps. +func (f *DavFile) UnmarshalJSON(b []byte) error { + var raw struct { + ID *json.Number `json:"id"` + Title *string `json:"title"` + PureSize *int64 `json:"pureContentLength"` + SizeStr *string `json:"contentLength"` + Updated *string `json:"updated"` + ViewURL *string `json:"viewUrl"` + } + if err := json.Unmarshal(b, &raw); err != nil { + return err + } + if raw.ID != nil { + f.ID = raw.ID.String() + } + if raw.Title != nil { + f.Title = *raw.Title + } + if raw.PureSize != nil { + f.Size = *raw.PureSize + } else if raw.SizeStr != nil { + if n, err := strconv.ParseInt(strings.Fields(*raw.SizeStr)[0], 10, 64); err == nil { + f.Size = n + } + } + if raw.Updated != nil { + f.Updated = *raw.Updated + } + if raw.ViewURL != nil { + f.ViewURL = *raw.ViewURL + } + return nil +} + +// ModTime parses the folder's updated timestamp. +func (f *DavFolder) ModTime() time.Time { + t, _ := time.Parse("2006-01-02T15:04:05.0000000-07:00", f.Updated) + return t +} + +// ModTime parses the file's updated timestamp. +func (f *DavFile) ModTime() time.Time { + t, _ := time.Parse("2006-01-02T15:04:05.0000000-07:00", f.Updated) + return t +} From ecf34ba51aae77f1626a60795f7329f25d9344e5 Mon Sep 17 00:00:00 2001 From: Andriy Oblivantsev Date: Thu, 20 Aug 2026 10:48:38 +0100 Subject: [PATCH 5/5] fix(files): rewrite viewUrl host to API base on download resolveAPIURL now rewrites absolute viewUrl hosts to the configured portal base so downloads stay on the internal network and keep the Authorization header (a cross-host HTTP->HTTPS redirect would otherwise strip it, failing large-file reads). --- files.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/files.go b/files.go index 794821c..c0d0cdd 100644 --- a/files.go +++ b/files.go @@ -319,8 +319,18 @@ func (c *Client) resolveAPIURL(ref string) string { if ref == "" { return ref } - if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") { - return ref + // Rewrite any host to the configured API base so downloads stay on the + // internal network and keep the Authorization header (no cross-host + // redirect that would strip it). Scheme-relative URLs are handled too. + if strings.HasPrefix(ref, "//") { + ref = "http:" + ref + } + if u, err := url.Parse(ref); err == nil && u.IsAbs() { + if base, err2 := url.Parse(c.baseURL()); err2 == nil { + u.Scheme = base.Scheme + u.Host = base.Host + return u.String() + } } base := c.baseURL() if strings.HasPrefix(ref, "/") {