diff --git a/AGENTS.md b/AGENTS.md index e7ab45d..d57865b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,7 +37,8 @@ docs/ published docs skills/ in-project agent skills (vendored, no external links) bin/ self-describing tools bin/{subject}/{method}.go (shebang) bin/brain/ search.go, serve.go; libs in internal/brain and internal/httpapi -internal/ shared Go (brain/rank is cgo-free) +bin/chats/ sync.go import.go facts.go apply.go; libs in internal/chats +internal/ shared Go (brain/rank is cgo-free; chats parsers too) bin/watch/ corpus watcher (internal via bin/brain/watch later) bin/mail/ mail pipeline: sync (Go), import (md), index_mail (rebuild) bin/tools/ vendored python libs behind bin/* (kblib, yamlout, websearch) diff --git a/PLAN.md b/PLAN.md index d086911..d30673d 100644 --- a/PLAN.md +++ b/PLAN.md @@ -57,6 +57,8 @@ detective method: **a fact needs ≥2 independent sources or it is brain/search.go deduction: facts → info → web-search kb/get kb/stats kb/eval brain/serve.go HTTP API (internal/httpapi) + chats/sync.go import.go facts.go apply.go + (libs in internal/chats; no chats index) md/import md/select md/tables md/gaps (mistune) brain/extract brain/audit brain/deduce (thinking wrapper) web/search (vendored) diff --git a/bin/chats/apply.go b/bin/chats/apply.go new file mode 100755 index 0000000..b03bd88 --- /dev/null +++ b/bin/chats/apply.go @@ -0,0 +1,19 @@ +//usr/bin/env go run -tags=chats_apply "$0" "$@"; exit +//go:build chats_apply +// +// bin/chats/apply.go - push extracted chat facts to OnlyOffice CRM. +// +// ./bin/chats/apply.go [--dry-run] +// +// NOTE: never run `gofmt -w` on this file — it breaks the shebang. +package main + +import ( + "os" + + "github.com/eSlider/2dph/internal/chats" +) + +func main() { + os.Exit(chats.RunApply(os.Args[1:])) +} diff --git a/bin/chats/doc.go b/bin/chats/doc.go new file mode 100644 index 0000000..4a8cb8d --- /dev/null +++ b/bin/chats/doc.go @@ -0,0 +1,4 @@ +// Commands in this directory are shebang mains (sync.go, import.go, facts.go, +// apply.go), each behind an exclusive build tag so `go build ./bin/chats` +// does not see two mains. Shared code lives in internal/chats. +package main diff --git a/bin/chats/facts.go b/bin/chats/facts.go new file mode 100755 index 0000000..3b50580 --- /dev/null +++ b/bin/chats/facts.go @@ -0,0 +1,20 @@ +//usr/bin/env go run -tags=chats_facts "$0" "$@"; exit +//go:build chats_facts +// +// bin/chats/facts.go - extract phone/email/linkedin facts from JSONL. +// +// ./bin/chats/facts.go +// +// Writes var/chats/facts/. Does not index the brain. +// NOTE: never run `gofmt -w` on this file — it breaks the shebang. +package main + +import ( + "os" + + "github.com/eSlider/2dph/internal/chats" +) + +func main() { + os.Exit(chats.RunFacts(os.Args[1:])) +} diff --git a/bin/chats/go.sum b/bin/chats/go.sum deleted file mode 100644 index e69de29..0000000 diff --git a/bin/chats/import.go b/bin/chats/import.go new file mode 100755 index 0000000..de020ef --- /dev/null +++ b/bin/chats/import.go @@ -0,0 +1,20 @@ +//usr/bin/env go run -tags=chats_import "$0" "$@"; exit +//go:build chats_import +// +// bin/chats/import.go - JSONL → markdown under var/chats/md/. +// +// ./bin/chats/import.go +// +// Conversion only. Brain ingest is bin/brain/index.go, not this command. +// NOTE: never run `gofmt -w` on this file — it breaks the shebang. +package main + +import ( + "os" + + "github.com/eSlider/2dph/internal/chats" +) + +func main() { + os.Exit(chats.RunImport(os.Args[1:])) +} diff --git a/bin/chats/index_cmd.go b/bin/chats/index_cmd.go deleted file mode 100644 index dcc0ada..0000000 --- a/bin/chats/index_cmd.go +++ /dev/null @@ -1,56 +0,0 @@ -package main - -import ( - "bytes" - "flag" - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" -) - -func runIndex(args []string) int { - fs := flag.NewFlagSet("chats index", flag.ContinueOnError) - help := fs.Bool("help", false, "") - fs.SetOutput(os.Stderr) - if err := fs.Parse(args); err != nil { - return 2 - } - if *help { - fmt.Fprintln(os.Stderr, "usage: chats index") - return 0 - } - - root := repoRoot() - mdDir := filepath.Join(chatsDir(), "md") - - _, err := os.Stat(mdDir) - if os.IsNotExist(err) { - fmt.Fprintf(os.Stderr, "chats index: no chat markdown at %s; run 'chats import' first\n", mdDir) - return 1 - } - - indexScript := filepath.Join(root, "bin", "kb", "index") - if _, err := os.Stat(indexScript); os.IsNotExist(err) { - fmt.Fprintf(os.Stderr, "chats index: %s not found\n", indexScript) - return 1 - } - - cmd := exec.Command(indexScript, "--corpus", mdDir) - var outBuf, errBuf bytes.Buffer - cmd.Stdout = &outBuf - cmd.Stderr = &errBuf - cmd.Dir = root - - if err := cmd.Run(); err != nil { - fmt.Fprintf(os.Stderr, "chats index: %v\n%s", err, errBuf.String()) - return 1 - } - result := strings.TrimSpace(outBuf.String()) - if result == "" { - result = strings.TrimSpace(errBuf.String()) - } - fmt.Printf("chats index: %s\n", result) - return 0 -} diff --git a/bin/chats/main.go b/bin/chats/main.go deleted file mode 100644 index 0545209..0000000 --- a/bin/chats/main.go +++ /dev/null @@ -1,115 +0,0 @@ -// bin/chats - sync, import, index, extract facts, and apply chat data -// from Telegram, WhatsApp, LinkedIn into the brain and OnlyOffice CRM. -// -// Usage: -// -// chats sync telegram [--limit N] [--since DATE] [--phone PHONE] -// chats sync whatsapp [--qr] [--limit N] -// chats sync linkedin [--limit N] -// chats import # JSONL → MD (all sources) -// chats index # rebuild var/kb.lbug with chats -// chats facts # extract + cross-check -// chats apply [--dry-run] # push to OnlyOffice CRM -package main - -import ( - "fmt" - "os" - "strings" -) - -func main() { - if len(os.Args) < 2 { - usage() - os.Exit(2) - } - cmd := os.Args[1] - args := os.Args[2:] - switch cmd { - case "sync": - if len(args) < 1 { - usage() - os.Exit(2) - } - platform := args[0] - platformArgs := args[1:] - switch platform { - case "telegram": - os.Exit(runSyncTelegram(platformArgs)) - case "whatsapp": - fmt.Fprintf(os.Stderr, "chats: WhatsApp not implemented yet\n") - os.Exit(1) - case "linkedin": - os.Exit(runSyncLinkedIn(platformArgs)) - default: - fmt.Fprintf(os.Stderr, "chats: unknown platform %q\n", platform) - os.Exit(2) - } - case "import": - os.Exit(runImport(args)) - case "index": - os.Exit(runIndex(args)) - case "facts": - os.Exit(runFacts(args)) - case "apply": - os.Exit(runApply(args)) - case "help", "-h", "--help": - usage() - return - default: - fmt.Fprintf(os.Stderr, "chats: unknown command %q\n", cmd) - usage() - os.Exit(2) - } -} - -func usage() { - w := os.Stderr - fmt.Fprintln(w, `Usage: chats [args] - -Commands: - sync telegram [--limit N] [--since DATE] [--phone PHONE] - sync whatsapp [--qr] [--limit N] - sync linkedin [--limit N] - import JSONL → MD (all sources) - index rebuild var/kb.lbug with chats - facts extract + cross-check facts - apply [--dry-run] push to OnlyOffice CRM - -Output layout: - var/chats///messages.jsonl - var/chats/md///messages.md`) -} - -// repoRoot locates the 2dph project root by walking up from the binary. -func repoRoot() string { - if v := os.Getenv("KB_ROOT"); v != "" { - return v - } - wd, err := os.Getwd() - if err != nil { - return "." - } - for i := 0; i < 10; i++ { - if _, err := os.Stat(wd + "/var"); err == nil { - return wd - } - if _, err := os.Stat(wd + "/.git"); err == nil { - return wd - } - parent := wd - if idx := strings.LastIndex(wd, "/"); idx >= 0 { - parent = wd[:idx] - } - if parent == wd { - break - } - wd = parent - } - return "." -} - -// chatsDir returns var/chats under the repo root. -func chatsDir() string { - return repoRoot() + "/var/chats" -} diff --git a/bin/chats/sync.go b/bin/chats/sync.go new file mode 100755 index 0000000..14eaac0 --- /dev/null +++ b/bin/chats/sync.go @@ -0,0 +1,41 @@ +//usr/bin/env go run -tags=chats_sync "$0" "$@"; exit +//go:build chats_sync +// +// bin/chats/sync.go - download chat messages to var/chats//. +// +// ./bin/chats/sync.go telegram [--limit N] [--phone PHONE] +// ./bin/chats/sync.go linkedin [--limit N] [--refresh] +// +// NOTE: never run `gofmt -w` on this file — it breaks the shebang. +package main + +import ( + "fmt" + "os" + + "github.com/eSlider/2dph/internal/chats" +) + +func main() { + if len(os.Args) < 2 { + fmt.Fprintln(os.Stderr, `usage: bin/chats/sync.go telegram|linkedin [flags]`) + os.Exit(2) + } + platform := os.Args[1] + args := os.Args[2:] + switch platform { + case "telegram": + os.Exit(chats.RunSyncTelegram(args)) + case "linkedin": + os.Exit(chats.RunSyncLinkedIn(args)) + case "whatsapp": + fmt.Fprintln(os.Stderr, "chats: WhatsApp not implemented yet") + os.Exit(1) + case "help", "-h", "--help": + fmt.Fprintln(os.Stderr, `usage: bin/chats/sync.go telegram|linkedin [flags]`) + return + default: + fmt.Fprintf(os.Stderr, "chats: unknown platform %q\n", platform) + os.Exit(2) + } +} diff --git a/bin/tools/test_bin_layout.py b/bin/tools/test_bin_layout.py index bf7e9e7..203aa9a 100644 --- a/bin/tools/test_bin_layout.py +++ b/bin/tools/test_bin_layout.py @@ -34,3 +34,32 @@ class BinLayoutTest(unittest.TestCase): def test_no_main_go_under_bin_brain(self) -> None: main = ROOT / "bin" / "brain" / "main.go" self.assertFalse(main.exists(), "bin/brain/main.go is not a method") + + def test_chats_methods_are_shebangs_not_main(self) -> None: + chats = ROOT / "bin" / "chats" + self.assertFalse( + (chats / "main.go").exists(), + "bin/chats/main.go is a dispatcher, not a method", + ) + self.assertFalse( + (chats / "index_cmd.go").exists(), + "chats index is a brain write hiding under the wrong subject", + ) + for method in ("sync.go", "import.go", "facts.go", "apply.go"): + p = chats / method + self.assertTrue(p.is_file(), f"missing bin/chats/{method}") + first = p.read_text().splitlines()[0] + self.assertTrue( + first.startswith("//usr/bin/env go run"), + f"{method} shebang, got {first!r}", + ) + + def test_chats_lib_lives_in_internal(self) -> None: + self.assertTrue( + (ROOT / "internal" / "chats" / "linkedin.go").is_file(), + "LinkedIn parser must live in internal/chats", + ) + self.assertFalse( + (ROOT / "bin" / "chats" / "linkedin.go").exists(), + "parser must not stay under bin/chats as a second main", + ) diff --git a/docs/chat-import-plan.md b/docs/chat-import-plan.md index 2072ebc..91875e0 100644 --- a/docs/chat-import-plan.md +++ b/docs/chat-import-plan.md @@ -15,9 +15,10 @@ OO_CLI (default: $HOME/go/bin/oo) ## Quick reference ``` -./bin/chat sync telegram --limit 100 -./bin/chat import -./bin/chat index -./bin/chat facts -./bin/chat apply --dry-run +./bin/chats/sync.go telegram --limit 100 +./bin/chats/import.go +./bin/chats/facts.go +./bin/chats/apply.go --dry-run ``` + +JSONL → markdown only. Brain ingest is `bin/brain/index.go` (not a `chats index`). diff --git a/bin/chats/apply_cmd.go b/internal/chats/apply.go similarity index 98% rename from bin/chats/apply_cmd.go rename to internal/chats/apply.go index 1b17127..3d200b1 100644 --- a/bin/chats/apply_cmd.go +++ b/internal/chats/apply.go @@ -1,4 +1,4 @@ -package main +package chats import ( "bytes" @@ -24,7 +24,7 @@ type ooContact struct { } `json:"commonData"` } -func runApply(args []string) int { +func RunApply(args []string) int { fs := flag.NewFlagSet("chats apply", flag.ContinueOnError) dryRun := fs.Bool("dry-run", false, "show what would be done without writing") help := fs.Bool("help", false, "") @@ -176,7 +176,7 @@ func runApply(args []string) int { } func loadFacts() ([]ExtractedFact, error) { - factsPath := filepath.Join(chatsDir(), "facts", "chat-facts.json") + factsPath := filepath.Join(Dir(), "facts", "chat-facts.json") data, err := os.ReadFile(factsPath) if err != nil { if os.IsNotExist(err) { diff --git a/bin/chats/chats_test.go b/internal/chats/chats_test.go similarity index 97% rename from bin/chats/chats_test.go rename to internal/chats/chats_test.go index 31b21ed..e9c150a 100644 --- a/bin/chats/chats_test.go +++ b/internal/chats/chats_test.go @@ -3,7 +3,7 @@ // These are integration tests using real data and real Telegram API (when // credentials are available). They follow the TDD workflow pattern: // sync → import → facts → verify. -package main +package chats import ( "encoding/json" @@ -48,7 +48,7 @@ func TestChatsImport(t *testing.T) { t.Cleanup(func() { os.Chdir(cwd) }) t.Setenv("KB_ROOT", dir) - exitCode := runImport([]string{}) + exitCode := RunImport([]string{}) if exitCode != 0 { t.Fatalf("import exit code %d", exitCode) } @@ -140,7 +140,7 @@ func TestChatsImportEmpty(t *testing.T) { t.Cleanup(func() { os.Chdir(cwd) }) t.Setenv("KB_ROOT", dir) - exitCode := runImport([]string{}) + exitCode := RunImport([]string{}) if exitCode == 0 { t.Fatal("expected non-zero exit for empty data dir") } @@ -171,7 +171,7 @@ func TestChatsRoundTrip(t *testing.T) { t.Cleanup(func() { os.Chdir(cwd) }) t.Setenv("KB_ROOT", dir) - if code := runImport([]string{}); code != 0 { + if code := RunImport([]string{}); code != 0 { t.Fatalf("import exit %d", code) } diff --git a/bin/chats/facts_cmd.go b/internal/chats/facts.go similarity index 89% rename from bin/chats/facts_cmd.go rename to internal/chats/facts.go index 0f0786d..8213be0 100644 --- a/bin/chats/facts_cmd.go +++ b/internal/chats/facts.go @@ -1,13 +1,11 @@ -package main +package chats import ( "bufio" - "bytes" "encoding/json" "flag" "fmt" "os" - "os/exec" "path/filepath" "regexp" "strings" @@ -77,7 +75,7 @@ type ExtractedFact struct { MessageID string `json:"message_id"` } -func runFacts(args []string) int { +func RunFacts(args []string) int { fs := flag.NewFlagSet("chats facts", flag.ContinueOnError) help := fs.Bool("help", false, "") fs.SetOutput(os.Stderr) @@ -89,7 +87,7 @@ func runFacts(args []string) int { return 0 } - root := chatsDir() + root := Dir() telegramDir := filepath.Join(root, "telegram") entries, err := os.ReadDir(telegramDir) @@ -149,7 +147,7 @@ func runFacts(args []string) int { } fmt.Printf("chats facts: saved to %s\n", factsPath) - writeFactsToBrain(root, allFacts) + writeFactsMarkdown(allFacts) return 0 } @@ -273,14 +271,10 @@ func filterFacts(facts []ExtractedFact, factType string) []ExtractedFact { return result } -func writeFactsToBrain(root string, facts []ExtractedFact) { - indexScript := filepath.Join(root, "bin", "kb", "index") - if _, err := os.Stat(indexScript); os.IsNotExist(err) { - fmt.Fprintf(os.Stderr, "chats facts: kb/index not found, skipping brain write\n") - return - } - - mdDir := filepath.Join(chatsDir(), "facts") +// writeFactsMarkdown stores a sidecar for humans. Brain ingest is +// bin/brain/index.go (not this subject). +func writeFactsMarkdown(facts []ExtractedFact) { + mdDir := filepath.Join(Dir(), "facts") if err := os.MkdirAll(mdDir, 0755); err != nil { fmt.Fprintf(os.Stderr, "chats facts: mkdir %s: %v\n", mdDir, err) return @@ -288,7 +282,7 @@ func writeFactsToBrain(root string, facts []ExtractedFact) { var sb strings.Builder sb.WriteString("---\n") - sb.WriteString("root: facts\n") + sb.WriteString("root: info\n") sb.WriteString("---\n\n") sb.WriteString("# Chat-Derived Facts\n\n") for _, f := range facts { @@ -302,15 +296,5 @@ func writeFactsToBrain(root string, facts []ExtractedFact) { fmt.Fprintf(os.Stderr, "chats facts: write %s: %v\n", factsMD, err) return } - - cmd := exec.Command(indexScript, "--corpus", mdDir, "--skip-indexes") - var outBuf, errBuf bytes.Buffer - cmd.Stdout = &outBuf - cmd.Stderr = &errBuf - cmd.Dir = root - if err := cmd.Run(); err != nil { - fmt.Fprintf(os.Stderr, "chats facts: brain index: %v\n%s", err, errBuf.String()) - return - } - fmt.Printf("chats facts: written to brain (%s)\n", strings.TrimSpace(outBuf.String())) + fmt.Printf("chats facts: markdown %s (index via brain, not chats)\n", factsMD) } diff --git a/bin/chats/import_cmd.go b/internal/chats/import.go similarity index 98% rename from bin/chats/import_cmd.go rename to internal/chats/import.go index 6a0ac6e..2154e33 100644 --- a/bin/chats/import_cmd.go +++ b/internal/chats/import.go @@ -1,4 +1,4 @@ -package main +package chats import ( "bufio" @@ -13,7 +13,7 @@ import ( "strings" ) -func runImport(args []string) int { +func RunImport(args []string) int { fs := flag.NewFlagSet("chats import", flag.ContinueOnError) help := fs.Bool("help", false, "") fs.SetOutput(os.Stderr) @@ -25,7 +25,7 @@ func runImport(args []string) int { return 0 } - root := chatsDir() + root := Dir() mdRoot := filepath.Join(root, "md") glob := filepath.Join(root, "telegram", "*", "messages.jsonl") diff --git a/bin/chats/linkedin.go b/internal/chats/linkedin.go similarity index 99% rename from bin/chats/linkedin.go rename to internal/chats/linkedin.go index 78f9377..33bf500 100644 --- a/bin/chats/linkedin.go +++ b/internal/chats/linkedin.go @@ -1,4 +1,4 @@ -package main +package chats import ( "bufio" diff --git a/bin/chats/linkedin_test.go b/internal/chats/linkedin_test.go similarity index 99% rename from bin/chats/linkedin_test.go rename to internal/chats/linkedin_test.go index aeb96dd..cdf4a94 100644 --- a/bin/chats/linkedin_test.go +++ b/internal/chats/linkedin_test.go @@ -1,4 +1,4 @@ -package main +package chats import ( "errors" diff --git a/bin/chats/mcpclient.go b/internal/chats/mcpclient.go similarity index 99% rename from bin/chats/mcpclient.go rename to internal/chats/mcpclient.go index 44264d7..306de76 100644 --- a/bin/chats/mcpclient.go +++ b/internal/chats/mcpclient.go @@ -1,4 +1,4 @@ -package main +package chats import ( "bufio" diff --git a/internal/chats/paths.go b/internal/chats/paths.go new file mode 100644 index 0000000..a74fcab --- /dev/null +++ b/internal/chats/paths.go @@ -0,0 +1,39 @@ +package chats + +import ( + "os" + "strings" +) + +// Root locates the 2dph project root (KB_ROOT, or walk up for var/ or .git). +func Root() string { + if v := os.Getenv("KB_ROOT"); v != "" { + return v + } + wd, err := os.Getwd() + if err != nil { + return "." + } + for i := 0; i < 10; i++ { + if _, err := os.Stat(wd + "/var"); err == nil { + return wd + } + if _, err := os.Stat(wd + "/.git"); err == nil { + return wd + } + parent := wd + if idx := strings.LastIndex(wd, "/"); idx >= 0 { + parent = wd[:idx] + } + if parent == wd { + break + } + wd = parent + } + return "." +} + +// Dir is var/chats under the project root. +func Dir() string { + return Root() + "/var/chats" +} diff --git a/bin/chats/source.go b/internal/chats/source.go similarity index 98% rename from bin/chats/source.go rename to internal/chats/source.go index 9223d10..677a0ce 100644 --- a/bin/chats/source.go +++ b/internal/chats/source.go @@ -1,4 +1,4 @@ -package main +package chats import ( "context" diff --git a/bin/chats/sync_linkedin_cmd.go b/internal/chats/sync_linkedin.go similarity index 96% rename from bin/chats/sync_linkedin_cmd.go rename to internal/chats/sync_linkedin.go index 13a0d81..0b02f1b 100644 --- a/bin/chats/sync_linkedin_cmd.go +++ b/internal/chats/sync_linkedin.go @@ -1,4 +1,4 @@ -package main +package chats import ( "context" @@ -28,7 +28,7 @@ func checkLinkedInSession(userDataDir string) (bool, error) { return false, nil } -func runSyncLinkedIn(args []string) int { +func RunSyncLinkedIn(args []string) int { fs := flag.NewFlagSet("chats sync linkedin", flag.ContinueOnError) limit := fs.Int("limit", 0, "max messages per conversation (0 = all)") refresh := fs.Bool("refresh", false, "refresh session from live webtop browser before sync") @@ -72,7 +72,7 @@ func runSyncLinkedIn(args []string) int { defer cancel() start := time.Now() - if err := src.Sync(ctx, chatsDir(), *limit); err != nil { + if err := src.Sync(ctx, Dir(), *limit); err != nil { fmt.Fprintf(os.Stderr, "chats sync linkedin: %v\n", err) return 1 } diff --git a/bin/chats/sync_cmd.go b/internal/chats/sync_telegram.go similarity index 95% rename from bin/chats/sync_cmd.go rename to internal/chats/sync_telegram.go index 50823d5..306ef4d 100644 --- a/bin/chats/sync_cmd.go +++ b/internal/chats/sync_telegram.go @@ -1,4 +1,4 @@ -package main +package chats import ( "context" @@ -11,7 +11,7 @@ import ( "time" ) -func runSyncTelegram(args []string) int { +func RunSyncTelegram(args []string) int { fs := flag.NewFlagSet("chats sync telegram", flag.ContinueOnError) limit := fs.Int("limit", 0, "max messages per chat (0 = all)") phone := fs.String("phone", "", "phone number (default env TELEGRAM_PHONE)") @@ -78,7 +78,7 @@ func runSyncTelegram(args []string) int { defer cancel() start := time.Now() - if err := src.Sync(ctx, chatsDir(), *limit); err != nil { + if err := src.Sync(ctx, Dir(), *limit); err != nil { fmt.Fprintf(os.Stderr, "chats sync telegram: %v\n", err) return 1 } diff --git a/bin/chats/testdata/linkedin_conversation.json b/internal/chats/testdata/linkedin_conversation.json similarity index 100% rename from bin/chats/testdata/linkedin_conversation.json rename to internal/chats/testdata/linkedin_conversation.json diff --git a/bin/chats/testdata/linkedin_inbox.json b/internal/chats/testdata/linkedin_inbox.json similarity index 100% rename from bin/chats/testdata/linkedin_inbox.json rename to internal/chats/testdata/linkedin_inbox.json