From afb93feae56bc25df35798baf6fb2a45f7f7b804 Mon Sep 17 00:00:00 2001 From: Andriy Oblivantsev Date: Tue, 22 Sep 2026 22:03:36 +0100 Subject: [PATCH] chore: keep host/client specifics out of the tree (env & config) Showroom-safe: the tree no longer carries internal hosts, IPs, ports, personal names, client domains or client file names. Behaviour is unchanged and now supplied per deployment. - catalog: hardcoded mail-org / project classifiers become a YAML-driven Classifier (OO_CATALOG_CONFIG or --config); neutral default classifies nothing as work. New catalog/classify.go + example + tests. - storage_fallback: drop the baked-in MinIO endpoint IP; require MINIO_ENDPOINT (+ keys) from the env. - kontolink: build DocEditor links from $ONLYOFFICE_URL instead of a hardcoded portal host; kontoblatt: no client file id in the output name. - oo: load .env CLI-wide (bootstrap.LoadEnv in execute) so non-authenticating commands (catalog scans) also see config. - genericize comments/docs/fixtures (AGENTS, README, .env.example, crm-associations, catalog tests, ES/pdfattach tests, mails). --- .env.example | 11 ++- AGENTS.md | 13 ++- README.md | 2 +- catalog/catalog_test.go | 22 +++-- catalog/classify.example.yaml | 26 ++++++ catalog/classify.go | 127 +++++++++++++++++++++++++++++ catalog/classify_test.go | 64 +++++++++++++++ catalog/mbox.go | 16 ++-- catalog/mbox_test.go | 14 ++-- catalog/scan_projects.go | 38 ++++----- catalog/thunderbird.go | 27 +----- catalog/thunderbird_test.go | 34 +++++--- cmd/kontoblatt/main.go | 2 +- cmd/kontolink/main.go | 18 +++- cmd/oo/catalog.go | 28 ++++++- cmd/oo/common.go | 4 +- docs/crm-associations.md | 4 +- file_es_text_integration_test.go | 2 +- file_es_text_test.go | 4 +- files.go | 2 +- internal/docpipe/pdfattach_test.go | 4 +- mails.go | 4 +- mails_test.go | 2 +- storage_fallback.go | 19 +++-- 24 files changed, 377 insertions(+), 110 deletions(-) create mode 100644 catalog/classify.example.yaml create mode 100644 catalog/classify.go create mode 100644 catalog/classify_test.go diff --git a/.env.example b/.env.example index 5b462d6..10b549b 100644 --- a/.env.example +++ b/.env.example @@ -34,13 +34,18 @@ ONLYOFFICE_PROJECT_ID=33 # MinIO download fallback for the portal's stale AWS S3 consumer (older # Documents folders). When the portal redirects to amazonaws.com with access -# key "minio" (403 InvalidAccessKeyId), files are fetched from the local MinIO -# store instead. Without a key/secret the fallback is disabled. -# MINIO_ENDPOINT=http://192.168.188.10:9000 +# key "minio" (403 InvalidAccessKeyId), files are fetched from the configured +# MinIO store instead. Without endpoint + key/secret the fallback is disabled. +# MINIO_ENDPOINT=http://minio.example.com:9000 # MINIO_BUCKET=office # MINIO_ACCESS_KEY= # MINIO_SECRET_KEY= +# Catalog scan classification rules (client mail domains, project remotes/names). +# Copy catalog/classify.example.yaml and point this at it; no rules means +# nothing is classified as work. +# OO_CATALOG_CONFIG=~/.config/oo/catalog-classify.yaml + # oo search — direct Elasticsearch access for name + content search. ES lives # inside the OnlyOffice VM on localhost:9200; expose it with an SSH tunnel # (see docs/elasticsearch.md). ONLYOFFICE_ES_INDEX defaults to files_file. diff --git a/AGENTS.md b/AGENTS.md index b69277f..ec519e3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,7 +21,7 @@ Canonical Go client for OnlyOffice Workspace (Projects + Calendar + CRM) and the - **List table (`DataTable`)** — `cmd/office/ui/table*.go`. Column layout policies live in `cmd/office/model/table_layout.go` (`TableFlexLayoutFor`); cell rendering uses the bubbles/table inline pattern in `table_render.go` (`renderTableCell`, `padANSIWidth`). See `.cursor/skills/office-tui-table/SKILL.md` before changing center-pane tables. - **Shared bootstrap — `cmd/internal/bootstrap/`.** `LoadEnv()` + `NewClient(ctx)` extracted from `oo`; both binaries import it. - **Bulk Documents tools — `cmd/ooscan/`, `cmd/pdfamount/`, `cmd/kontoblatt/`, `cmd/kontolink/`.** Single-purpose binaries (folder index, PDF amounts, Kontoblatt summary/linking). Pace requests, route API calls through `DoRetry`; usage in README. -- **Personal ops tooling** (disk inventory, dossier→CRM sync, SearXNG) lives in private [`eSlider/oo-workspace`](https://git.produktor.io/eSlider/oo-workspace) (`oow`), not in this public tree. +- **Personal ops tooling** (disk inventory, dossier→CRM sync, SearXNG) lives in a private companion repo `eSlider/oo-workspace` (the `oow` CLI), not in this public tree. ## Rules @@ -33,6 +33,11 @@ Canonical Go client for OnlyOffice Workspace (Projects + Calendar + CRM) and the - **Documents for agents:** prefer Markdown in git; OnlyOffice UI is weak for `.md`/`.txt`. Use `oo docs put-md` (md→docx) and `oo docs put-txt` (txt→docx, preserves line breaks). All upload paths default to **upsert** by `stem|ext` (`--replace`, default true); `--no-replace` fails on conflict; `--allow-duplicate` opts into raw OO append. `oo projects files dedupe PROJECT_ID` reports/removes duplicate stem|ext copies (`--apply`, `--cross`; includes project root folder). - Every table output goes through `printTable(headers, rows)`; every single-object through `printObject(v)`. Do not `fmt.Println` rows ad-hoc or the `--output json` flag breaks for that command. - No secrets in the repo; use `.env` (gitignored). Commit `.env.example` only. +- **No host/client specifics in the tree.** Endpoints, IPs/ports, client mail + domains, project remotes/names and personal names stay out of source and + fixtures — they come from env/config (`MINIO_*`, `OO_CATALOG_CONFIG`, see + [`catalog/classify.example.yaml`](catalog/classify.example.yaml)). This repo is + mirrored to GitHub as a public showroom, so the tree must stay project-generic. - Follow SemVer on tags; this repo is tagged at GitHub under `git@github.com:eSlider/go-onlyoffice.git`. ### Testing policy (2026-04-24) @@ -57,6 +62,6 @@ write `mux.HandleFunc("/api/2.0/...")` to emulate OnlyOffice, we write an ## Related - [`docs/README.md`](docs/README.md) — reference index (file client, ES, SQL, rclone). -- [`eSlider/inventar`](https://git.produktor.io/eSlider/inventar) — ASR/ADR (see ASR-0008 Go library module conventions). -- [`eSlider/inventar-sync`](https://git.produktor.io/eSlider/inventar-sync) — OnlyOffice → Gitea issue sync, consumes this library. -- [`produktor.io/vidarr`](https://git.produktor.io/produktor.io/vidarr) — legacy consumer being migrated from `pkg/onlyoffice` to this module. +- `eSlider/inventar` — ASR/ADR (see ASR-0008 Go library module conventions). +- `eSlider/inventar-sync` — OnlyOffice → Gitea issue sync, consumes this library. +- `vidarr` — legacy consumer being migrated from `pkg/onlyoffice` to this module. diff --git a/README.md b/README.md index 42badd6..7c7384e 100644 --- a/README.md +++ b/README.md @@ -743,7 +743,7 @@ opens a shared cooldown gate and `Retry-After` is honoured (see `DoRetry` and ooscan 659 # recursive index → TSV: file_id, folder_id, path, title ooscan 659 666 > oo-index.tsv # several roots into one index pdfamount 671 # "Zu zahlender Betrag" per PDF → TSV: file_id, title, amount -kontoblatt 3906 ./kontoblatt.xlsx # summary (Gegenkonto/Monat) uploaded next to source +kontoblatt 1234 ./kontoblatt.xlsx # summary (Gegenkonto/Monat) uploaded next to source kontolink IN.xlsx oo-index.tsv OUT.xlsx [FILE_ID] [AMOUNTS_TSV] # kontolink writes DocEditor links into the Link column: Beleg → supplier+month # → amount+date (5th arg = pdfamount output); with FILE_ID it updates the diff --git a/catalog/catalog_test.go b/catalog/catalog_test.go index 47ed1e6..a9d9058 100644 --- a/catalog/catalog_test.go +++ b/catalog/catalog_test.go @@ -100,26 +100,38 @@ END:VCARD func TestScanProjectsRoot(t *testing.T) { root := t.TempDir() - repo := filepath.Join(root, "produktor-demo") + repo := filepath.Join(root, "acme-demo") if err := os.MkdirAll(filepath.Join(repo, ".git"), 0o755); err != nil { t.Fatal(err) } - doc, err := ScanProjectsRoot(root, 3) + cl := &Classifier{WorkNames: []string{"acme"}} + doc, err := ScanProjectsRootOpts(root, 3, ScanOptions{Classifier: cl}) if err != nil { t.Fatal(err) } found := false for _, e := range doc.Entries { - if e.Kind == "company" && e.Name == "produktor-demo" { + if e.Kind == "company" && e.Name == "acme-demo" { found = true - if e.Role != "work" { - t.Fatalf("role=%q", e.Role) + if e.Role != "work" || e.Zone != "warm" { + t.Fatalf("role=%q zone=%q", e.Role, e.Zone) } } } if !found { t.Fatalf("missing company: %+v", doc.Entries) } + + // Neutral default leaves it unclassified. + doc, err = ScanProjectsRoot(root, 3) + if err != nil { + t.Fatal(err) + } + for _, e := range doc.Entries { + if e.Kind == "company" && e.Name == "acme-demo" && e.Role != "unknown" { + t.Fatalf("neutral role=%q", e.Role) + } + } } func TestEntryID(t *testing.T) { diff --git a/catalog/classify.example.yaml b/catalog/classify.example.yaml new file mode 100644 index 0000000..5d4b10b --- /dev/null +++ b/catalog/classify.example.yaml @@ -0,0 +1,26 @@ +# Deployment classification rules for `oo catalog scan-projects` / +# `oo catalog scan-thunderbird`. Point OO_CATALOG_CONFIG (or --config) at a copy +# of this file. Keep your real rules out of the repository — they name your +# clients and hosts. The library defaults to no rules (nothing is "work"). +# +# work_remotes: a git remote containing any of these substrings → work/hot. +work_remotes: + - git.internal.example + - github.com/acme +# +# work_names: a project directory name containing any of these substrings → work/warm. +work_names: + - acme +# +# mail_orgs: ordered rules for Thunderbird/mbox identities; the first match wins. +# Match by exact `domain`, `suffix` (e.g. ".example.com") and/or display `name`. +# `zone` defaults to hot, `role` to work. +mail_orgs: + - domain: acme.example + org: Acme GmbH + zone: hot + role: work + - suffix: .gov.example + org: Public Sector + zone: warm + role: work diff --git a/catalog/classify.go b/catalog/classify.go new file mode 100644 index 0000000..31d83ec --- /dev/null +++ b/catalog/classify.go @@ -0,0 +1,127 @@ +package catalog + +import ( + "fmt" + "os" + "strings" + + "gopkg.in/yaml.v3" +) + +// Classifier maps project trees and mail identities to catalog org/zone/role. +// +// The library ships with neutral defaults: nothing is classified as work unless +// the deployment supplies rules. Those rules are deployment-specific, so they +// live in a YAML config file (path from OO_CATALOG_CONFIG or the --config flag), +// not in the code. See catalog/classify.example.yaml. +type Classifier struct { + // WorkRemotes: a git remote containing any of these substrings → work/hot. + WorkRemotes []string `yaml:"work_remotes,omitempty"` + // WorkNames: a project name containing any of these substrings → work/warm. + WorkNames []string `yaml:"work_names,omitempty"` + // MailOrgs: ordered mail-identity rules; the first match wins. + MailOrgs []MailRule `yaml:"mail_orgs,omitempty"` +} + +// MailRule maps an email domain and/or a display-name substring to an org with +// a zone/role. At least one of Domain, Suffix or Name must be set. +type MailRule struct { + Domain string `yaml:"domain,omitempty"` // exact domain, case-insensitive + Suffix string `yaml:"suffix,omitempty"` // domain suffix, e.g. ".example.com" + Name string `yaml:"name,omitempty"` // substring of the display name + Org string `yaml:"org"` + Zone string `yaml:"zone,omitempty"` // default "hot" + Role string `yaml:"role,omitempty"` // default "work" +} + +// DefaultClassifier returns the neutral classifier (no deployment rules). +func DefaultClassifier() *Classifier { return &Classifier{} } + +// LoadClassifier reads a classifier config from a YAML file. +func LoadClassifier(path string) (*Classifier, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var c Classifier + if err := yaml.Unmarshal(b, &c); err != nil { + return nil, fmt.Errorf("parse classifier config %s: %w", path, err) + } + return &c, nil +} + +// LoadClassifierFromEnv loads the classifier named by OO_CATALOG_CONFIG. An +// empty variable yields the neutral classifier. +func LoadClassifierFromEnv() (*Classifier, error) { + path := strings.TrimSpace(os.Getenv("OO_CATALOG_CONFIG")) + if path == "" { + return DefaultClassifier(), nil + } + return LoadClassifier(path) +} + +// ClassifyProject returns (role, zone) for a project name and git remote. +// Generic name heuristics come first; deployment rules supply the work cases. +func (c *Classifier) ClassifyProject(name, remote string) (role, zone string) { + lower := strings.ToLower(name) + remoteL := strings.ToLower(remote) + switch { + case strings.Contains(lower, "experiment") || strings.HasPrefix(lower, "test"): + return "experiment", "cold" + case lower == "mama" || lower == "personal" || strings.Contains(lower, "private"): + return "personal", "private" + } + if c != nil { + for _, r := range c.WorkRemotes { + if r != "" && strings.Contains(remoteL, strings.ToLower(r)) { + return "work", "hot" + } + } + for _, n := range c.WorkNames { + if n != "" && strings.Contains(lower, strings.ToLower(n)) { + return "work", "warm" + } + } + } + return "unknown", "warm" +} + +// ClassifyMail returns (org, zone, role) for a mail identity. name is the +// display name (may be empty); email is the address. +func (c *Classifier) ClassifyMail(name, email string) (org, zone, role string) { + em := NormalizeEmail(email) + _, domain, _ := strings.Cut(em, "@") + nameL := strings.ToLower(strings.TrimSpace(name)) + if c != nil { + for _, r := range c.MailOrgs { + if !mailRuleMatches(r, domain, nameL) { + continue + } + z, ro := r.Zone, r.Role + if z == "" { + z = "hot" + } + if ro == "" { + ro = "work" + } + return r.Org, z, ro + } + } + if strings.HasSuffix(domain, ".de") && looksPublicSector(domain) { + return domain, "warm", "work" + } + return "", "private", "unknown" +} + +func mailRuleMatches(r MailRule, domain, nameL string) bool { + if r.Domain != "" && domain == strings.ToLower(strings.TrimSpace(r.Domain)) { + return true + } + if r.Suffix != "" && strings.HasSuffix(domain, strings.ToLower(strings.TrimSpace(r.Suffix))) { + return true + } + if r.Name != "" && nameL != "" && strings.Contains(nameL, strings.ToLower(strings.TrimSpace(r.Name))) { + return true + } + return false +} diff --git a/catalog/classify_test.go b/catalog/classify_test.go new file mode 100644 index 0000000..130767f --- /dev/null +++ b/catalog/classify_test.go @@ -0,0 +1,64 @@ +package catalog + +import ( + "os" + "path/filepath" + "testing" +) + +func TestClassifierRules(t *testing.T) { + cl := &Classifier{ + WorkRemotes: []string{"git.internal.example"}, + WorkNames: []string{"acme"}, + MailOrgs: []MailRule{ + {Domain: "acme.example", Org: "Acme", Zone: "hot", Role: "work"}, + {Suffix: ".gov.example", Org: "Public", Zone: "warm", Role: "work"}, + }, + } + if role, zone := cl.ClassifyProject("acme-app", "git@git.internal.example:team/acme-app.git"); role != "work" || zone != "hot" { + t.Fatalf("remote: %s/%s", role, zone) + } + if role, zone := cl.ClassifyProject("acme-demo", ""); role != "work" || zone != "warm" { + t.Fatalf("name: %s/%s", role, zone) + } + if role, zone := cl.ClassifyProject("experiment-x", ""); role != "experiment" || zone != "cold" { + t.Fatalf("experiment: %s/%s", role, zone) + } + if org, zone, role := cl.ClassifyMail("", "bob@acme.example"); org != "Acme" || zone != "hot" || role != "work" { + t.Fatalf("mail: %s/%s/%s", org, zone, role) + } + if org, _, _ := cl.ClassifyMail("X", "x@team.gov.example"); org != "Public" { + t.Fatalf("suffix: %s", org) + } + if org, zone, role := cl.ClassifyMail("", "someone@unknown.example"); org != "" || zone != "private" || role != "unknown" { + t.Fatalf("neutral: %s/%s/%s", org, zone, role) + } +} + +func TestLoadClassifier(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "classify.yaml") + if err := os.WriteFile(path, []byte("work_names:\n - acme\nmail_orgs:\n - domain: acme.example\n org: Acme\n"), 0o644); err != nil { + t.Fatal(err) + } + cl, err := LoadClassifier(path) + if err != nil { + t.Fatal(err) + } + if role, _ := cl.ClassifyProject("acme-app", ""); role != "work" { + t.Fatalf("role=%s", role) + } + if org, zone, _ := cl.ClassifyMail("", "a@acme.example"); org != "Acme" || zone != "hot" { + t.Fatalf("org=%s zone=%s", org, zone) + } + + t.Setenv("OO_CATALOG_CONFIG", path) + if envCl, err := LoadClassifierFromEnv(); err != nil || envCl == nil || len(envCl.WorkNames) == 0 { + t.Fatalf("env: %+v %v", envCl, err) + } + t.Setenv("OO_CATALOG_CONFIG", "") + neutral, err := LoadClassifierFromEnv() + if err != nil || len(neutral.WorkNames) != 0 || len(neutral.MailOrgs) != 0 { + t.Fatalf("neutral: %+v %v", neutral, err) + } +} diff --git a/catalog/mbox.go b/catalog/mbox.go index e4810aa..086b184 100644 --- a/catalog/mbox.go +++ b/catalog/mbox.go @@ -16,6 +16,8 @@ type ScanOptions struct { MboxHeaders bool // MboxMaxBytes skips individual mbox files larger than this (0 = 256 MiB default). MboxMaxBytes int64 + // Classifier supplies deployment classification rules; nil → neutral default. + Classifier *Classifier } // ScanThunderbirdRoot finds Thunderbird profiles under root and emits person rows @@ -37,6 +39,10 @@ func ScanThunderbirdRootOpts(root string, opts ScanOptions) (*Document, error) { if opts.MboxMaxBytes <= 0 { opts.MboxMaxBytes = 256 << 20 } + cl := opts.Classifier + if cl == nil { + cl = DefaultClassifier() + } var entries []Entry seenDB := map[string]struct{}{} @@ -64,7 +70,7 @@ func ScanThunderbirdRootOpts(root string, opts ScanOptions) (*Document, error) { return nil } seenDB[path] = struct{}{} - parsed, perr := parseGlodaContacts(path) + parsed, perr := parseGlodaContacts(path, cl) if perr != nil { entries = append(entries, Entry{ ID: EntryID("person", "", filepath.Base(path)), @@ -84,7 +90,7 @@ func ScanThunderbirdRootOpts(root string, opts ScanOptions) (*Document, error) { return nil } seenMAB[path] = struct{}{} - parsed, perr := parseMABEmails(path) + parsed, perr := parseMABEmails(path, cl) if perr != nil { return nil } @@ -101,7 +107,7 @@ func ScanThunderbirdRootOpts(root string, opts ScanOptions) (*Document, error) { if info.Size() > opts.MboxMaxBytes { return nil } - parsed, perr := parseMboxHeaderEmails(path) + parsed, perr := parseMboxHeaderEmails(path, cl) if perr != nil { return nil } @@ -150,7 +156,7 @@ func isLikelyMboxFile(name, path string) bool { } // parseMboxHeaderEmails extracts addresses from From/To/Cc/Reply-To headers only. -func parseMboxHeaderEmails(path string) ([]Entry, error) { +func parseMboxHeaderEmails(path string, cl *Classifier) ([]Entry, error) { f, err := os.Open(path) if err != nil { return nil, err @@ -220,7 +226,7 @@ func parseMboxHeaderEmails(path string) ([]Entry, error) { var out []Entry for em := range emails { - org, zone, role := classifyMailIdentity("", em) + org, zone, role := cl.ClassifyMail("", em) out = append(out, Entry{ ID: EntryID("person", em, ""), Kind: "person", diff --git a/catalog/mbox_test.go b/catalog/mbox_test.go index 67ad2f4..4c20474 100644 --- a/catalog/mbox_test.go +++ b/catalog/mbox_test.go @@ -10,22 +10,22 @@ func TestParseMboxHeaderEmails(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "INBOX") body := `From - Mon Jul 1 00:00:00 2016 -From: Axel Schaefer -To: Andriy Oblivantsev +From: Alice Smith +To: Bob Jones Cc: noreply@example.com, client@stadt-example.de Subject: test Body line ignored From - Mon Jul 2 00:00:00 2016 -From: Someone -To: list@wheregroup.com +From: Someone +To: list@acme.example more body ` if err := os.WriteFile(path, []byte(body), 0o644); err != nil { t.Fatal(err) } - ents, err := parseMboxHeaderEmails(path) + ents, err := parseMboxHeaderEmails(path, DefaultClassifier()) if err != nil { t.Fatal(err) } @@ -35,7 +35,7 @@ more body got[e.Emails[0]] = true } } - if !got["axel.schaefer@wheregroup.com"] || !got["andriy.oblivantsev@wheregroup.com"] { + if !got["alice.smith@acme.example"] || !got["bob.jones@acme.example"] { t.Fatalf("%v", got) } if got["noreply@example.com"] { @@ -53,7 +53,7 @@ func TestScanThunderbirdRootOptsMbox(t *testing.T) { t.Fatal(err) } if err := os.WriteFile(filepath.Join(imap, "INBOX"), []byte( - "From - x\nFrom: a@wheregroup.com\nTo: b@wheregroup.com\n\nbody\n", + "From - x\nFrom: a@acme.example\nTo: b@acme.example\n\nbody\n", ), 0o644); err != nil { t.Fatal(err) } diff --git a/catalog/scan_projects.go b/catalog/scan_projects.go index e8ae348..00a15f5 100644 --- a/catalog/scan_projects.go +++ b/catalog/scan_projects.go @@ -10,10 +10,19 @@ import ( // ScanProjectsRoot finds git roots under root (max depth) and emits company rows. func ScanProjectsRoot(root string, maxDepth int) (*Document, error) { + return ScanProjectsRootOpts(root, maxDepth, ScanOptions{}) +} + +// ScanProjectsRootOpts is ScanProjectsRoot with deployment classification rules. +func ScanProjectsRootOpts(root string, maxDepth int, opts ScanOptions) (*Document, error) { root = filepath.Clean(root) if maxDepth <= 0 { maxDepth = 4 } + cl := opts.Classifier + if cl == nil { + cl = DefaultClassifier() + } st, err := os.Stat(root) if err != nil { return nil, err @@ -23,7 +32,7 @@ func ScanProjectsRoot(root string, maxDepth int) (*Document, error) { } var entries []Entry - err = walkGitRoots(root, root, 0, maxDepth, &entries) + err = walkGitRoots(root, root, 0, maxDepth, cl, &entries) if err != nil { return nil, err } @@ -43,7 +52,7 @@ func ScanProjectsRoot(root string, maxDepth int) (*Document, error) { } path := filepath.Join(root, name) id := EntryID("company", "", name) - role, zone := classifyProjectName(name, "") + role, zone := cl.ClassifyProject(name, "") entries = append(entries, Entry{ ID: id, Kind: "company", @@ -59,7 +68,7 @@ func ScanProjectsRoot(root string, maxDepth int) (*Document, error) { return MergeDocs(&Document{Entries: entries}), nil } -func walkGitRoots(root, dir string, depth, maxDepth int, out *[]Entry) error { +func walkGitRoots(root, dir string, depth, maxDepth int, cl *Classifier, out *[]Entry) error { if depth > maxDepth { return nil } @@ -67,7 +76,7 @@ func walkGitRoots(root, dir string, depth, maxDepth int, out *[]Entry) error { if st, err := os.Stat(filepath.Join(dir, ".git")); err == nil && (st.IsDir() || st.Mode().IsRegular()) { name := filepath.Base(dir) remote := gitRemoteOrigin(dir) - role, zone := classifyProjectName(name, remote) + role, zone := cl.ClassifyProject(name, remote) *out = append(*out, Entry{ ID: EntryID("company", "", name), Kind: "company", @@ -93,7 +102,7 @@ func walkGitRoots(root, dir string, depth, maxDepth int, out *[]Entry) error { if name == ".git" || name == "node_modules" || name == "vendor" || name == ".venv" || name == "dist" { continue } - _ = walkGitRoots(root, filepath.Join(dir, name), depth+1, maxDepth, out) + _ = walkGitRoots(root, filepath.Join(dir, name), depth+1, maxDepth, cl, out) } return nil } @@ -106,22 +115,3 @@ func gitRemoteOrigin(dir string) string { } return strings.TrimSpace(string(b)) } - -func classifyProjectName(name, remote string) (role, zone string) { - lower := strings.ToLower(name) - remoteL := strings.ToLower(remote) - switch { - case strings.Contains(lower, "experiment") || strings.HasPrefix(lower, "test"): - return "experiment", "cold" - case lower == "mama" || lower == "personal" || strings.Contains(lower, "private"): - return "personal", "private" - case strings.Contains(remoteL, "git.produktor.io") || strings.Contains(remoteL, "github.com/eslider"): - return "work", "hot" - case strings.Contains(lower, "produktor") || strings.Contains(lower, "eslider") || - strings.Contains(lower, "asesoria") || strings.Contains(lower, "dyvenia") || - strings.Contains(lower, "onlyoffice"): - return "work", "warm" - default: - return "unknown", "warm" - } -} diff --git a/catalog/thunderbird.go b/catalog/thunderbird.go index 6b8265f..1fceea5 100644 --- a/catalog/thunderbird.go +++ b/catalog/thunderbird.go @@ -64,7 +64,7 @@ func noisyEmail(email string) bool { return false } -func parseMABEmails(path string) ([]Entry, error) { +func parseMABEmails(path string, cl *Classifier) ([]Entry, error) { b, err := os.ReadFile(path) if err != nil { return nil, err @@ -76,7 +76,7 @@ func parseMABEmails(path string) ([]Entry, error) { if noisyEmail(em) { continue } - org, zone, role := classifyMailIdentity("", em) + org, zone, role := cl.ClassifyMail("", em) out = append(out, Entry{ ID: EntryID("person", em, ""), Kind: "person", @@ -93,7 +93,7 @@ func parseMABEmails(path string) ([]Entry, error) { return out, nil } -func parseGlodaContacts(dbPath string) ([]Entry, error) { +func parseGlodaContacts(dbPath string, cl *Classifier) ([]Entry, error) { // read-only URI; immutable=1 helps when WAL/shm are missing dsn := "file:" + dbPath + "?mode=ro&_pragma=query_only(1)" db, err := sql.Open("sqlite", dsn) @@ -137,7 +137,7 @@ func parseGlodaContacts(dbPath string) ([]Entry, error) { if display != "" { first, last = SplitDisplayName(display) } - org, zone, role := classifyMailIdentity(display, em) + org, zone, role := cl.ClassifyMail(display, em) out = append(out, Entry{ ID: EntryID("person", em, display), Kind: "person", @@ -157,25 +157,6 @@ func parseGlodaContacts(dbPath string) ([]Entry, error) { return out, rows.Err() } -func classifyMailIdentity(name, email string) (org, zone, role string) { - em := NormalizeEmail(email) - _, domain, _ := strings.Cut(em, "@") - switch { - case domain == "wheregroup.com" || strings.Contains(strings.ToLower(name), "wheregroup"): - return "WhereGroup", "warm", "work" - case domain == "produktor.io" || domain == "eslider.de" || strings.HasSuffix(domain, ".produktor.io"): - return "produktor.io", "hot", "work" - case domain == "dyvenia.com": - return "Dyvenia", "warm", "work" - case domain == "immowelt.de" || domain == "immowelt.com": - return "Immowelt", "warm", "work" - case strings.HasSuffix(domain, ".de") && looksPublicSector(domain): - return domain, "warm", "work" - default: - return "", "private", "unknown" - } -} - func looksPublicSector(domain string) bool { d := strings.ToLower(domain) hints := []string{ diff --git a/catalog/thunderbird_test.go b/catalog/thunderbird_test.go index 1b366f7..769faec 100644 --- a/catalog/thunderbird_test.go +++ b/catalog/thunderbird_test.go @@ -16,8 +16,8 @@ func TestNoisyEmail(t *testing.T) { if !noisyEmail("x@marketplace.amazon.de") { t.Fatal("amazon marketplace") } - if noisyEmail("andriy.oblivantsev@wheregroup.com") { - t.Fatal("should keep wheregroup") + if noisyEmail("alice.smith@acme.example") { + t.Fatal("should keep a human work address") } } @@ -25,14 +25,15 @@ func TestParseMABEmails(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "abook.mab") body := `// mork junk - PrimaryEmail=andriy.oblivantsev@wheregroup.com + PrimaryEmail=alice.smith@acme.example noreply@github.com - axel.schaefer@wheregroup.com + bob.jones@acme.example ` if err := os.WriteFile(path, []byte(body), 0o644); err != nil { t.Fatal(err) } - ents, err := parseMABEmails(path) + // Neutral default: no deployment rules → unclassified. + ents, err := parseMABEmails(path, DefaultClassifier()) if err != nil { t.Fatal(err) } @@ -40,7 +41,18 @@ func TestParseMABEmails(t *testing.T) { t.Fatalf("got %d: %+v", len(ents), ents) } for _, e := range ents { - if e.Org != "WhereGroup" || e.Role != "work" { + if e.Org != "" || e.Role != "unknown" { + t.Fatalf("%+v", e) + } + } + // A deployment rule classifies the domain as work. + cl := &Classifier{MailOrgs: []MailRule{{Domain: "acme.example", Org: "Acme", Zone: "warm", Role: "work"}}} + ents, err = parseMABEmails(path, cl) + if err != nil { + t.Fatal(err) + } + for _, e := range ents { + if e.Org != "Acme" || e.Role != "work" || e.Zone != "warm" { t.Fatalf("%+v", e) } } @@ -56,8 +68,8 @@ func TestParseGlodaContacts(t *testing.T) { _, err = db.Exec(` CREATE TABLE contacts (id INTEGER PRIMARY KEY, name TEXT); CREATE TABLE identities (id INTEGER PRIMARY KEY, contactID INTEGER, kind TEXT, value TEXT); - INSERT INTO contacts VALUES (1, 'Axel Schaefer'); - INSERT INTO identities VALUES (1, 1, 'email', 'axel.schaefer@wheregroup.com'); + INSERT INTO contacts VALUES (1, 'Alice Smith'); + INSERT INTO identities VALUES (1, 1, 'email', 'alice.smith@acme.example'); INSERT INTO contacts VALUES (2, 'Noise Bot'); INSERT INTO identities VALUES (2, 2, 'email', 'noreply@example.com'); `) @@ -66,14 +78,14 @@ func TestParseGlodaContacts(t *testing.T) { } _ = db.Close() - ents, err := parseGlodaContacts(dbPath) + ents, err := parseGlodaContacts(dbPath, DefaultClassifier()) if err != nil { t.Fatal(err) } if len(ents) != 1 { t.Fatalf("got %d %+v", len(ents), ents) } - if ents[0].First != "Axel" || ents[0].Emails[0] != "axel.schaefer@wheregroup.com" { + if ents[0].First != "Alice" || ents[0].Emails[0] != "alice.smith@acme.example" { t.Fatalf("%+v", ents[0]) } } @@ -84,7 +96,7 @@ func TestScanThunderbirdRoot(t *testing.T) { if err := os.MkdirAll(prof, 0o755); err != nil { t.Fatal(err) } - if err := os.WriteFile(filepath.Join(prof, "abook.mab"), []byte("mail=paul.schmidt@wheregroup.com\n"), 0o644); err != nil { + if err := os.WriteFile(filepath.Join(prof, "abook.mab"), []byte("mail=paul.schmidt@acme.example\n"), 0o644); err != nil { t.Fatal(err) } doc, err := ScanThunderbirdRoot(root) diff --git a/cmd/kontoblatt/main.go b/cmd/kontoblatt/main.go index 37a8a0d..48698ee 100644 --- a/cmd/kontoblatt/main.go +++ b/cmd/kontoblatt/main.go @@ -162,7 +162,7 @@ func main() { } fmt.Printf("source: id=%s title=%q folder=%s\n", fileID, title, folder) - name := "Kontoblatt-1591-2025-Zusammenfassung.xlsx" + name := "Kontoblatt-Zusammenfassung.xlsx" tmp := "/tmp/opencode/" + name data, _ := os.ReadFile(outPath) if err := os.WriteFile(tmp, data, 0o600); err != nil { diff --git a/cmd/kontolink/main.go b/cmd/kontolink/main.go index a1b4dba..f3bc994 100644 --- a/cmd/kontolink/main.go +++ b/cmd/kontolink/main.go @@ -76,7 +76,15 @@ func nearest(cands []entry, rd time.Time) (entry, bool) { return cands[best], true } -const linkPrefix = "https://office.pro-dukt.de/Products/Files/DocEditor.aspx?fileid=" +// portalBaseFromEnv resolves the portal base URL used to build document links. +func portalBaseFromEnv() string { + for _, k := range []string{"ONLYOFFICE_URL", "ONLYOFFICE_HOST", "OO_URL"} { + if v := strings.TrimSpace(os.Getenv(k)); v != "" { + return strings.TrimRight(v, "/") + } + } + return "" +} type entry struct { id, path, title, norm string @@ -91,6 +99,12 @@ func main() { } in, idxPath, out := os.Args[1], os.Args[2], os.Args[3] + portal := portalBaseFromEnv() + if portal == "" { + fmt.Fprintln(os.Stderr, "set ONLYOFFICE_URL (or ONLYOFFICE_HOST/OO_URL) to build document links") + os.Exit(2) + } + idxRaw, err := os.ReadFile(idxPath) if err != nil { panic(err) @@ -153,7 +167,7 @@ func main() { continue } ref, _ := excelize.CoordinatesToCellName(8, i+1) - if err := f.SetCellValue(sheet, ref, linkPrefix+e.id); err != nil { + if err := f.SetCellValue(sheet, ref, portal+"/Products/Files/DocEditor.aspx?fileid="+e.id); err != nil { panic(err) } used[e.id] = true diff --git a/cmd/oo/catalog.go b/cmd/oo/catalog.go index c43e92f..0566bc6 100644 --- a/cmd/oo/catalog.go +++ b/cmd/oo/catalog.go @@ -64,6 +64,7 @@ func catalogScanContactsCmd() *cobra.Command { func catalogScanProjectsCmd() *cobra.Command { var outPath string var maxDepth int + var configPath string cmd := &cobra.Command{ Use: "scan-projects", Short: "Git roots / remotes / top-level dirs → company rows", @@ -72,7 +73,11 @@ func catalogScanProjectsCmd() *cobra.Command { if root == "" { return fmt.Errorf("--root is required") } - doc, err := catalog.ScanProjectsRoot(root, maxDepth) + cl, err := catalogClassifier(configPath) + if err != nil { + return err + } + doc, err := catalog.ScanProjectsRootOpts(root, maxDepth, catalog.ScanOptions{Classifier: cl}) if err != nil { return err } @@ -81,6 +86,7 @@ func catalogScanProjectsCmd() *cobra.Command { } cmd.Flags().String("root", "", "projects directory (local path)") cmd.Flags().IntVar(&maxDepth, "max-depth", 4, "max directory depth for git roots") + cmd.Flags().StringVar(&configPath, "config", "", "classification rules YAML (default $OO_CATALOG_CONFIG)") cmd.Flags().StringVarP(&outPath, "out", "O", "", "write YAML to this path") _ = cmd.MarkFlagRequired("root") return cmd @@ -89,6 +95,7 @@ func catalogScanProjectsCmd() *cobra.Command { func catalogScanThunderbirdCmd() *cobra.Command { var outPath string var mboxHeaders bool + var configPath string cmd := &cobra.Command{ Use: "scan-thunderbird", Short: "Thunderbird profiles: abook/history.mab + Gloda SQLite contacts", @@ -98,13 +105,18 @@ func catalogScanThunderbirdCmd() *cobra.Command { - optional --mbox-headers: From/To/Cc/Reply-To from mbox folder files (no bodies) Noisy senders (noreply, Amazon marketplace, GitHub reply, …) are skipped. -Default zone is private; known work domains (e.g. wheregroup.com) get zone=warm role=work.`, +Default zone is private; work domains/names are classified from the rules in +$OO_CATALOG_CONFIG (or --config) — none are hardcoded.`, RunE: func(cmd *cobra.Command, args []string) error { root, _ := cmd.Flags().GetString("root") if root == "" { return fmt.Errorf("--root is required") } - doc, err := catalog.ScanThunderbirdRootOpts(root, catalog.ScanOptions{MboxHeaders: mboxHeaders}) + cl, err := catalogClassifier(configPath) + if err != nil { + return err + } + doc, err := catalog.ScanThunderbirdRootOpts(root, catalog.ScanOptions{MboxHeaders: mboxHeaders, Classifier: cl}) if err != nil { return err } @@ -113,11 +125,21 @@ Default zone is private; known work domains (e.g. wheregroup.com) get zone=warm } cmd.Flags().String("root", "", "Thunderbird profile or parent directory (local path)") cmd.Flags().BoolVar(&mboxHeaders, "mbox-headers", false, "also extract emails from mbox From/To/Cc headers") + cmd.Flags().StringVar(&configPath, "config", "", "classification rules YAML (default $OO_CATALOG_CONFIG)") cmd.Flags().StringVarP(&outPath, "out", "O", "", "write YAML to this path") _ = cmd.MarkFlagRequired("root") return cmd } +// catalogClassifier loads classification rules from --config, else +// $OO_CATALOG_CONFIG, else the neutral default (no rules). +func catalogClassifier(configPath string) (*catalog.Classifier, error) { + if configPath == "" { + return catalog.LoadClassifierFromEnv() + } + return catalog.LoadClassifier(configPath) +} + func catalogMergeCmd() *cobra.Command { var inputs []string var outPath string diff --git a/cmd/oo/common.go b/cmd/oo/common.go index 687f801..b651c63 100644 --- a/cmd/oo/common.go +++ b/cmd/oo/common.go @@ -31,7 +31,9 @@ func init() { } // execute runs the root command. Exported only to main.go in the same package. -func execute() error { return rootCmd.Execute() } +// .env is loaded CLI-wide so non-authenticating commands (e.g. catalog scans) +// also see configuration such as OO_CATALOG_CONFIG. +func execute() error { bootstrap.LoadEnv(); return rootCmd.Execute() } // newOO loads env (only .env in CWD) and returns an authenticated client. // godotenv is a CLI-only concern; the library itself never loads dotfiles. diff --git a/docs/crm-associations.md b/docs/crm-associations.md index dfe744b..b8645e0 100644 --- a/docs/crm-associations.md +++ b/docs/crm-associations.md @@ -120,5 +120,5 @@ oo mails draft-invoice --invoice INVOICE_ID --to billing@example.com ## Related - README § invoices / mail / CRM cleanup -- Personal workspace tooling (disk inventory, dossier sync): private - `git.produktor.io/eSlider/oo-workspace` (`oow` CLI) +- Personal workspace tooling (disk inventory, dossier sync) lives in a private + companion repo (`oo-workspace`, the `oow` CLI). diff --git a/file_es_text_integration_test.go b/file_es_text_integration_test.go index 30d295e..975f2ef 100644 --- a/file_es_text_integration_test.go +++ b/file_es_text_integration_test.go @@ -52,7 +52,7 @@ func TestIntegrationESTextIndex(t *testing.T) { token := "gotes" + stamp doc := TextDoc{ ID: "3578", - Title: "2026-07-28-S1021-Edelweiss-rechnung.pdf", + Title: "2026-07-28-S1021-acme-rechnung.pdf", FolderID: "634", Ext: "pdf", Content: "Begleitzettel SGB XI — Rechnung " + token, diff --git a/file_es_text_test.go b/file_es_text_test.go index aaeeca3..67b51fb 100644 --- a/file_es_text_test.go +++ b/file_es_text_test.go @@ -78,7 +78,7 @@ func TestParseESTextResponse(t *testing.T) { { "_id": "3578", "_score": 3.21, - "_source": {"id": "3578", "title": "2026-07-28-S1021-Edelweiss-rechnung.pdf", "folder": "634", "ext": "pdf"}, + "_source": {"id": "3578", "title": "2026-07-28-S1021-acme-rechnung.pdf", "folder": "634", "ext": "pdf"}, "highlight": {"content": ["Begleitzettel … S1021 …"]} } ] @@ -92,7 +92,7 @@ func TestParseESTextResponse(t *testing.T) { t.Fatalf("hits = %d, want 1", len(hits)) } h := hits[0] - if h.ID != "3578" || h.Title != "2026-07-28-S1021-Edelweiss-rechnung.pdf" || h.Kind != File { + if h.ID != "3578" || h.Title != "2026-07-28-S1021-acme-rechnung.pdf" || h.Kind != File { t.Errorf("entry = %+v", h.Entry) } if h.ParentID != "634" || !reflect.DeepEqual(h.Path, []string{"634"}) { diff --git a/files.go b/files.go index 690e510..737612b 100644 --- a/files.go +++ b/files.go @@ -278,7 +278,7 @@ func (c *Client) RenameFile(ctx context.Context, fileID, newTitle string) (*File // DeleteFiles permanently deletes files by numeric id (Documents module). // Uses per-file DELETE (DeleteDavItems); fileops/delete returns 200 on some -// portals (e.g. produktor.io) without removing the file. +// portals without actually removing the file. // // Deprecated: use FileStore.Delete via Client.Files()/Client.FileStore. func (c *Client) DeleteFiles(ctx context.Context, fileIDs []int) error { diff --git a/internal/docpipe/pdfattach_test.go b/internal/docpipe/pdfattach_test.go index d2a3665..640c605 100644 --- a/internal/docpipe/pdfattach_test.go +++ b/internal/docpipe/pdfattach_test.go @@ -105,10 +105,10 @@ func TestXMLToText(t *testing.T) { raw := []byte(` S1063 -Edelweiss & Co42.00 +Acme & Co42.00 `) got := xmlToText(raw) - for _, want := range []string{"S1063", "Edelweiss & Co", "42.00"} { + for _, want := range []string{"S1063", "Acme & Co", "42.00"} { if !strings.Contains(got, want) { t.Errorf("xmlToText missing %q:\n%s", want, got) } diff --git a/mails.go b/mails.go index cc17ec9..d29be6b 100644 --- a/mails.go +++ b/mails.go @@ -161,7 +161,7 @@ func (c *Client) RemoveMailMessages(ctx context.Context, ids ...int) (map[string // Id 0 creates a new draft. Body is HTML; the API field name is "body" (not htmlBody). type SaveMailDraftParams struct { ID int64 // 0 = create - From string // mailbox address, e.g. eslider@gmail.com + From string // mailbox address, e.g. user@example.com To string // comma-separated or single address Cc string Bcc string @@ -372,7 +372,7 @@ func mailMessagesPath(f MailMessagesFilter, page, count int) string { // ParseMailAddress splits a RFC 5322 mailbox string into display name and email. // Examples: // - `"LinkedIn" ` → name LinkedIn, address a@b.com -// - `eslider@gmail.com` → address only +// - `user@example.com` → address only func ParseMailAddress(raw string) (name, address string) { raw = strings.TrimSpace(raw) if raw == "" { diff --git a/mails_test.go b/mails_test.go index bf9a35c..f86c628 100644 --- a/mails_test.go +++ b/mails_test.go @@ -56,7 +56,7 @@ func TestParseMailAddress(t *testing.T) { }{ {`"LinkedIn Jobbenachrichtigungen" `, "LinkedIn Jobbenachrichtigungen", "jobalerts-noreply@linkedin.com"}, {`"Bitfinex" `, "Bitfinex", "no-reply@bitfinex.com"}, - {"eslider@gmail.com", "", "eslider@gmail.com"}, + {"user@example.com", "", "user@example.com"}, {`"Glassdoor-Jobs" `, "Glassdoor-Jobs", "noreply@glassdoor.com"}, {"", "", ""}, } diff --git a/storage_fallback.go b/storage_fallback.go index be829af..e6d59ed 100644 --- a/storage_fallback.go +++ b/storage_fallback.go @@ -2,11 +2,13 @@ package onlyoffice // MinIO download fallback for the portal's stale AWS S3 consumer. // -// On the Fibu EDL portal some older Documents files live in S3/MinIO, but the -// portal's storage consumer still points at s3.us-east-1.amazonaws.com with -// access key "minio". Downloads of those files answer 403 InvalidAccessKeyId. -// The bytes are present in the local MinIO store under a deterministic object -// key, so the client retries the GET there. +// On some portals older Documents files live in S3/MinIO, but the portal's +// storage consumer still points at s3.us-east-1.amazonaws.com with access key +// "minio". Downloads of those files answer 403 InvalidAccessKeyId. The bytes are +// present in the local MinIO store under a deterministic object key, so the +// client retries the GET there. Endpoint/bucket/keys come from the environment +// (MINIO_ENDPOINT, MINIO_BUCKET, MINIO_ACCESS_KEY, MINIO_SECRET_KEY); without +// them the fallback is disabled. import ( "context" @@ -25,9 +27,8 @@ import ( ) const ( - defaultMinioEndpoint = "http://192.168.188.10:9000" - defaultMinioBucket = "office" - minioRegion = "us-east-1" + defaultMinioBucket = "office" + minioRegion = "us-east-1" ) // minioObjectKey is the fallback object key layout the portal's S3 consumer @@ -96,7 +97,7 @@ type minioConfig struct { // Secrets are never defaulted; without access/secret keys the fallback is off. func loadMinioConfig() minioConfig { return minioConfig{ - Endpoint: strings.TrimRight(firstNonEmpty(os.Getenv("MINIO_ENDPOINT"), defaultMinioEndpoint), "/"), + Endpoint: strings.TrimRight(strings.TrimSpace(os.Getenv("MINIO_ENDPOINT")), "/"), Bucket: firstNonEmpty(os.Getenv("MINIO_BUCKET"), defaultMinioBucket), AccessKey: os.Getenv("MINIO_ACCESS_KEY"), SecretKey: os.Getenv("MINIO_SECRET_KEY"),