diff --git a/bin/chat b/bin/chat new file mode 100755 index 0000000..4260cea --- /dev/null +++ b/bin/chat @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# bin/chats - sync, import, index, facts, apply for Telegram/WhatsApp/LinkedIn. +# Builds the chats binary on first run / when source changes, then execs it. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +BIN="$ROOT/var/bin/chats" +SRC="$ROOT/bin/chats" + +mkdir -p "$ROOT/var/bin" + +need_build=0 +if [ ! -x "$BIN" ]; then + need_build=1 +else + while IFS= read -r -d '' f; do + if [ "$f" -nt "$BIN" ]; then + need_build=1 + break + fi + done < <(find "$SRC" -name '*.go' -print0 2>/dev/null) +fi + +if [ "$need_build" -eq 1 ]; then + echo "Building chats..." >&2 + (cd "$SRC" && go build -o "$BIN" .) || exit 1 +fi + +exec "$BIN" "$@" diff --git a/bin/chats/apply_cmd.go b/bin/chats/apply_cmd.go new file mode 100644 index 0000000..a846cf2 --- /dev/null +++ b/bin/chats/apply_cmd.go @@ -0,0 +1,318 @@ +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +type ooContact struct { + ID int `json:"id"` + DisplayName string `json:"displayName"` + FirstName string `json:"firstName"` + LastName string `json:"lastName"` + About string `json:"about"` + CommonData []struct { + InfoType int `json:"infoType"` + Data string `json:"data"` + Category string `json:"categoryName"` + } `json:"commonData"` +} + +func runApply(args []string) int { + fs := flag.NewFlagSet("chats apply", flag.ContinueOnError) + dryRun := fs.Bool("dry-run", false, "show what would be done without writing") + help := fs.Bool("help", false, "") + fs.SetOutput(os.Stderr) + if err := fs.Parse(args); err != nil { + return 2 + } + if *help { + fmt.Fprintln(os.Stderr, "usage: chats apply [--dry-run]") + return 0 + } + + ooCLI := findOO() + if ooCLI == "" { + fmt.Fprintln(os.Stderr, "chats apply: oo CLI not found; set OO_CLI or install go-onlyoffice") + return 1 + } + + facts, err := loadFacts() + if err != nil { + fmt.Fprintf(os.Stderr, "chats apply: %v\n", err) + return 1 + } + if len(facts) == 0 { + fmt.Println("chats apply: no facts to process") + return 0 + } + + phoneFacts := filterFacts(facts, "phone") + emailFacts := filterFacts(facts, "email") + + phoneFacts = dedupeFacts(phoneFacts) + emailFacts = dedupeFacts(emailFacts) + + type resolvedFact struct { + Fact ExtractedFact + OoID int + OoName string + Action string // "info-add" or "persons-create" + } + + var resolved []resolvedFact + + for _, f := range phoneFacts { + contact, err := searchContact(ooCLI, f.ChatName) + if err != nil || contact == nil { + fmt.Printf(" ✗ %s: phone %s — not found in CRM\n", f.ChatName, f.Value) + resolved = append(resolved, resolvedFact{Fact: f, Action: "persons-create"}) + continue + } + hasPhone := false + for _, d := range contact.CommonData { + if d.InfoType == 2 { + hasPhone = true + break + } + } + if hasPhone { + fmt.Printf(" ✓ %s (ID %d): phone %s — already has phone, skip\n", contact.DisplayName, contact.ID, f.Value) + continue + } + fmt.Printf(" → %s (ID %d): add phone %s\n", contact.DisplayName, contact.ID, f.Value) + resolved = append(resolved, resolvedFact{ + Fact: f, OoID: contact.ID, OoName: contact.DisplayName, Action: "info-add", + }) + } + + for _, f := range emailFacts { + if strings.EqualFold(f.Value, envVar("ONLYOFFICE_USER", "")) || + strings.EqualFold(f.Value, envVar("OO_USER", "")) || + strings.EqualFold(f.Value, os.Getenv("EMAIL")) { + continue + } + contact, err := searchContact(ooCLI, f.ChatName) + if err != nil || contact == nil { + fmt.Printf(" ✗ %s: email %s — not found in CRM\n", f.ChatName, f.Value) + resolved = append(resolved, resolvedFact{Fact: f, Action: "persons-create"}) + continue + } + hasEmail := false + for _, d := range contact.CommonData { + if d.InfoType == 1 && d.Data == f.Value { + hasEmail = true + break + } + } + if hasEmail { + fmt.Printf(" ✓ %s (ID %d): email %s — already exists\n", contact.DisplayName, contact.ID, f.Value) + continue + } + fmt.Printf(" → %s (ID %d): add email %s\n", contact.DisplayName, contact.ID, f.Value) + resolved = append(resolved, resolvedFact{ + Fact: f, OoID: contact.ID, OoName: contact.DisplayName, Action: "info-add", + }) + } + + if len(resolved) == 0 { + fmt.Println("chats apply: nothing to apply") + return 0 + } + + fmt.Printf("\nchats apply: %d actions to apply\n", len(resolved)) + + if *dryRun { + for _, r := range resolved { + switch r.Action { + case "info-add": + infoType := "Phone" + if r.Fact.FactType == "email" { + infoType = "Email" + } + fmt.Printf(" [dry-run] oo contacts info-add %d --type %s --value %s\n", + r.OoID, infoType, r.Fact.Value) + case "persons-create": + fmt.Printf(" [dry-run] oo persons create --first %q --about %q\n", + r.Fact.ChatName, "Contact from Telegram chat") + } + } + return 0 + } + + success := 0 + failed := 0 + for _, r := range resolved { + switch r.Action { + case "info-add": + infoType := "Phone" + if r.Fact.FactType == "email" { + infoType = "Email" + } + if err := ooInfoAdd(ooCLI, r.OoID, infoType, r.Fact.Value); err != nil { + fmt.Fprintf(os.Stderr, " ✗ info-add %s: %v\n", r.Fact.Value, err) + failed++ + } else { + fmt.Printf(" ✓ %s → %s (ID %d)\n", r.Fact.Value, r.OoName, r.OoID) + success++ + } + case "persons-create": + fmt.Printf(" - create %s (skipped — needs review)\n", r.Fact.ChatName) + success++ + } + } + + fmt.Printf("\nchats apply: %d succeeded, %d failed\n", success, failed) + if failed > 0 { + return 1 + } + return 0 +} + +func loadFacts() ([]ExtractedFact, error) { + factsPath := filepath.Join(chatsDir(), "facts", "chat-facts.json") + data, err := os.ReadFile(factsPath) + if err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("no facts at %s; run 'chats facts' first", factsPath) + } + return nil, fmt.Errorf("read facts: %w", err) + } + var facts []ExtractedFact + if err := json.Unmarshal(data, &facts); err != nil { + return nil, fmt.Errorf("parse facts: %w", err) + } + return facts, nil +} + +func dedupeFacts(facts []ExtractedFact) []ExtractedFact { + seen := make(map[string]bool) + var result []ExtractedFact + for _, f := range facts { + norm := normalizePhone(f.Value) + key := f.ChatName + ":" + factTypeKey(f.FactType) + ":" + norm + if seen[key] { + continue + } + seen[key] = true + f.Value = norm + result = append(result, f) + } + return result +} + +func normalizePhone(s string) string { + var digits []rune + for _, r := range s { + if r >= '0' && r <= '9' { + digits = append(digits, r) + } + } + if len(digits) > 0 { + return string(digits) + } + return s +} + +func factTypeKey(t string) string { + switch t { + case "phone": + return "p" + case "email": + return "e" + default: + return t + } +} + +func findOO() string { + if v := os.Getenv("OO_CLI"); v != "" { + if _, err := os.Stat(v); err == nil { + return v + } + } + candidates := []string{ + filepath.Join(os.Getenv("HOME"), "go", "bin", "oo"), + "/home/ano/go/bin/oo", + } + for _, c := range candidates { + if _, err := os.Stat(c); err == nil { + return c + } + } + return "" +} + +func searchContact(ooCLI, name string) (*ooContact, error) { + query := name + // Try full name first + if c, _ := searchByQuery(ooCLI, query); c != nil { + return c, nil + } + // Try first word + firstWord := strings.Fields(name)[0] + if firstWord != name { + if c, _ := searchByQuery(ooCLI, firstWord); c != nil { + return c, nil + } + } + return nil, nil +} + +func searchByQuery(ooCLI, query string) (*ooContact, error) { + cmd := exec.Command(ooCLI, "persons", "list", "--search", query, "-o", "json") + var outBuf, errBuf bytes.Buffer + cmd.Stdout = &outBuf + cmd.Stderr = &errBuf + cmd.Env = os.Environ() + + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("oo persons list: %w\n%s", err, errBuf.String()) + } + var contacts []ooContact + if err := json.Unmarshal(outBuf.Bytes(), &contacts); err != nil { + return nil, nil + } + for _, c := range contacts { + lower := strings.ToLower(c.DisplayName) + lowerQuery := strings.ToLower(query) + if strings.EqualFold(c.DisplayName, query) || + strings.Contains(lower, lowerQuery) || + strings.Contains(lowerQuery, strings.ToLower(c.FirstName)) { + return &c, nil + } + for _, d := range c.CommonData { + if d.InfoType == 1 && strings.Contains(strings.ToLower(d.Data), lowerQuery) { + return &c, nil + } + } + } + if len(contacts) > 0 { + return &contacts[0], nil + } + return nil, nil +} + +func ooInfoAdd(ooCLI string, contactID int, infoType, value string) error { + cmd := exec.Command(ooCLI, "contacts", "info-add", + fmt.Sprintf("%d", contactID), + "--type", infoType, + "--value", value, + "--category", "Work", + "-o", "json", + ) + var outBuf, errBuf bytes.Buffer + cmd.Stdout = &outBuf + cmd.Stderr = &errBuf + cmd.Env = os.Environ() + if err := cmd.Run(); err != nil { + return fmt.Errorf("info-add: %w\n%s", err, errBuf.String()) + } + return nil +} diff --git a/bin/chats/chats_test.go b/bin/chats/chats_test.go new file mode 100644 index 0000000..31b21ed --- /dev/null +++ b/bin/chats/chats_test.go @@ -0,0 +1,202 @@ +// System tests for bin/chats. +// +// These are integration tests using real data and real Telegram API (when +// credentials are available). They follow the TDD workflow pattern: +// sync → import → facts → verify. +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// TestChatsImport validates JSONL → MD conversion with a synthetic fixture. +func TestChatsImport(t *testing.T) { + dir := t.TempDir() + root := filepath.Join(dir, "var", "chats") + chatDir := filepath.Join(root, "telegram", "test_user_123") + if err := os.MkdirAll(chatDir, 0755); err != nil { + t.Fatal(err) + } + + jsonlPath := filepath.Join(chatDir, "messages.jsonl") + f, err := os.Create(jsonlPath) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + enc := json.NewEncoder(f) + messages := []Message{ + {ID: "tg_1", Timestamp: "2026-01-15T10:30:00Z", From: "Alice", Text: "Hello!", Platform: "telegram"}, + {ID: "tg_2", Timestamp: "2026-01-15T10:31:00Z", From: "Bob", Text: "Hi Alice, my phone is +34 612 345 678", Platform: "telegram"}, + {ID: "tg_3", Timestamp: "2026-01-15T10:32:00Z", From: "Alice", Text: "Check my LinkedIn: https://linkedin.com/in/alice-test", Platform: "telegram"}, + {ID: "tg_4", Timestamp: "2026-01-15T10:33:00Z", From: "Bob", Text: "My email is bob@example.com, working on Project X", Platform: "telegram"}, + } + for _, m := range messages { + if err := enc.Encode(m); err != nil { + t.Fatal(err) + } + } + f.Close() + + cwd, _ := os.Getwd() + os.Chdir(dir) + t.Cleanup(func() { os.Chdir(cwd) }) + t.Setenv("KB_ROOT", dir) + + exitCode := runImport([]string{}) + if exitCode != 0 { + t.Fatalf("import exit code %d", exitCode) + } + + mdGlob := filepath.Join(root, "md", "telegram", "*", "messages.md") + matches, err := filepath.Glob(mdGlob) + if err != nil { + t.Fatal(err) + } + if len(matches) == 0 { + t.Fatal("no markdown files created by import") + } + + mdData, err := os.ReadFile(matches[0]) + if err != nil { + t.Fatal(err) + } + content := string(mdData) + + if !strings.Contains(content, "Alice") { + t.Error("markdown missing sender name 'Alice'") + } + if !strings.Contains(content, "2026-01-15") { + t.Error("markdown missing date") + } + if !strings.Contains(content, "---") { + t.Error("markdown missing YAML frontmatter") + } +} + +// TestChatsFacts validates fact extraction from JSONL fixture. +func TestChatsFacts(t *testing.T) { + dir := t.TempDir() + root := filepath.Join(dir, "var", "chats") + chatDir := filepath.Join(root, "telegram", "test_user_facts") + if err := os.MkdirAll(chatDir, 0755); err != nil { + t.Fatal(err) + } + + jsonlPath := filepath.Join(chatDir, "messages.jsonl") + f, err := os.Create(jsonlPath) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + enc := json.NewEncoder(f) + messages := []Message{ + {ID: "tg_10", Timestamp: "2026-06-01T12:00:00Z", From: "Charlie", Text: "Call me at +1 555 123 4567", Platform: "telegram"}, + {ID: "tg_11", Timestamp: "2026-06-01T12:01:00Z", From: "Charlie", Text: "My LinkedIn is linkedin.com/in/charlie-dev", Platform: "telegram"}, + {ID: "tg_12", Timestamp: "2026-06-01T12:02:00Z", From: "Charlie", Text: "Email: charlie@dev.com", Platform: "telegram"}, + {ID: "tg_13", Timestamp: "2026-06-01T12:03:00Z", From: "Charlie", Text: "I work at Acme Corp on Project Mercury", Platform: "telegram"}, + } + for _, m := range messages { + if err := enc.Encode(m); err != nil { + t.Fatal(err) + } + } + f.Close() + + facts, _ := extractFacts(jsonlPath, "test_user_facts") + if len(facts) == 0 { + t.Fatal("expected facts, got none") + } + + types := make(map[string]int) + for _, f := range facts { + types[f.FactType]++ + } + if types["phone"] < 1 { + t.Errorf("expected >=1 phone fact, got %d", types["phone"]) + } + if types["email"] < 1 { + t.Errorf("expected >=1 email fact, got %d", types["email"]) + } + if types["linkedin"] < 1 { + t.Errorf("expected >=1 linkedin fact, got %d", types["linkedin"]) + } + if types["skill"] < 1 { + t.Errorf("expected >=1 skill fact, got %d", types["skill"]) + } +} + +// TestChatsImportEmptyDir tests that import handles no JSONL gracefully. +func TestChatsImportEmpty(t *testing.T) { + dir := t.TempDir() + cwd, _ := os.Getwd() + os.Chdir(dir) + t.Cleanup(func() { os.Chdir(cwd) }) + t.Setenv("KB_ROOT", dir) + + exitCode := runImport([]string{}) + if exitCode == 0 { + t.Fatal("expected non-zero exit for empty data dir") + } +} + +// TestChatsRoundTrip creates a synthetic JSONL, imports it, then verifies +// the markdown structure is parseable and contains YAML frontmatter. +func TestChatsRoundTrip(t *testing.T) { + dir := t.TempDir() + root := filepath.Join(dir, "var", "chats") + chatDir := filepath.Join(root, "telegram", "rt_user") + if err := os.MkdirAll(chatDir, 0755); err != nil { + t.Fatal(err) + } + + jsonlPath := filepath.Join(chatDir, "messages.jsonl") + f, err := os.Create(jsonlPath) + if err != nil { + t.Fatal(err) + } + enc := json.NewEncoder(f) + enc.Encode(Message{ID: "tg_100", Timestamp: "2026-07-01T08:00:00Z", From: "Diana", Text: "Hey", Platform: "telegram"}) + enc.Encode(Message{ID: "tg_101", Timestamp: "2026-07-01T08:01:00Z", From: "Diana", Text: "How are you?", Platform: "telegram"}) + f.Close() + + cwd, _ := os.Getwd() + os.Chdir(dir) + t.Cleanup(func() { os.Chdir(cwd) }) + t.Setenv("KB_ROOT", dir) + + if code := runImport([]string{}); code != 0 { + t.Fatalf("import exit %d", code) + } + + mdGlob := filepath.Join(root, "md", "telegram", "*", "messages.md") + matches, _ := filepath.Glob(mdGlob) + if len(matches) == 0 { + t.Fatal("no markdown produced") + } + + data, err := os.ReadFile(matches[0]) + if err != nil { + t.Fatal(err) + } + content := string(data) + + if !strings.HasPrefix(content, "---") { + t.Error("markdown should start with YAML frontmatter delimiter") + } + if !strings.Contains(content, "platform: telegram") { + t.Error("markdown should contain platform field") + } + if !strings.Contains(content, "message_count: 2") { + t.Error("markdown should contain correct message count") + } + if !strings.Contains(content, "Diana") { + t.Error("markdown should contain participants") + } +} diff --git a/bin/chats/facts_cmd.go b/bin/chats/facts_cmd.go new file mode 100644 index 0000000..0f0786d --- /dev/null +++ b/bin/chats/facts_cmd.go @@ -0,0 +1,316 @@ +package main + +import ( + "bufio" + "bytes" + "encoding/json" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" +) + +var ( + phoneRegex = regexp.MustCompile(`[+\d][\d\s\-()]{6,25}\d`) + dateRegex = regexp.MustCompile(`^\d{2,4}[-/]\d{1,2}[-/]\d{2,4}$`) + rangeRegex = regexp.MustCompile(`^\d+\s*[-–]\s*\d+$`) + linkedinRegex = regexp.MustCompile(`linkedin\.com/in/[\w-]+`) + emailRegex = regexp.MustCompile(`[\w.+-]+@[\w-]+\.[\w.-]+`) + projectRegex = regexp.MustCompile(`(?i)project\s*[:/]\s*(.+)`) + dealRegex = regexp.MustCompile(`(?i)(deal|opportunity)\s*[:/]\s*(.+)`) + skillRegex = regexp.MustCompile(`(?i)(works?|worked|working)\s+(at|on|with)\s+([A-Z][\w\s]+)`) +) + +func isValidPhone(s string) bool { + s = strings.TrimSpace(s) + s = strings.Trim(s, "+()-\t ") + if len(s) < 6 || len(s) > 25 { + return false + } + if dateRegex.MatchString(s) || rangeRegex.MatchString(s) { + return false + } + if strings.ContainsAny(s, "/abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") { + return false + } + if strings.Contains(s, "000") || strings.Contains(s, "500 ") || strings.Contains(s, "000 ") { + return false + } + digits := 0 + for _, r := range s { + if r >= '0' && r <= '9' { + digits++ + } + } + if digits < 7 || digits > 15 { + return false + } + // Card number pattern: 16 digits with possible spaces + if digits == 16 { + return false + } + // Date-like: 8 digits starting with 20xx or 19xx + if len(s) <= 8 && digits == 8 && (strings.HasPrefix(s, "20") || strings.HasPrefix(s, "19")) { + return false + } + // 11+ digits starting with 2 - unlikely phone + if digits >= 11 && strings.HasPrefix(s, "2") && !strings.HasPrefix(s, "+") { + return false + } + // Must start with + or be at least 7 digits + if !strings.HasPrefix(s, "+") && digits < 7 { + return false + } + return true +} + +type ExtractedFact struct { + ChatID string `json:"chat_id"` + ChatName string `json:"chat_name"` + Platform string `json:"platform"` + FactType string `json:"fact_type"` + Value string `json:"value"` + Source string `json:"source"` + MessageID string `json:"message_id"` +} + +func runFacts(args []string) int { + fs := flag.NewFlagSet("chats facts", flag.ContinueOnError) + help := fs.Bool("help", false, "") + fs.SetOutput(os.Stderr) + if err := fs.Parse(args); err != nil { + return 2 + } + if *help { + fmt.Fprintln(os.Stderr, "usage: chats facts") + return 0 + } + + root := chatsDir() + telegramDir := filepath.Join(root, "telegram") + + entries, err := os.ReadDir(telegramDir) + if err != nil { + fmt.Fprintf(os.Stderr, "chats facts: read %s: %v\n", telegramDir, err) + return 1 + } + + var allFacts []ExtractedFact + for _, entry := range entries { + if !entry.IsDir() { + continue + } + chatID := entry.Name() + jsonlPath := filepath.Join(telegramDir, chatID, "messages.jsonl") + info, err := os.Stat(jsonlPath) + if err != nil { + continue + } + if info.Size() == 0 { + continue + } + + facts, chatName := extractFacts(jsonlPath, chatID) + allFacts = append(allFacts, facts...) + _ = chatName + } + + if len(allFacts) == 0 { + fmt.Println("chats facts: no facts extracted") + return 0 + } + + phoneFacts := filterFacts(allFacts, "phone") + emailFacts := filterFacts(allFacts, "email") + linkedinFacts := filterFacts(allFacts, "linkedin") + projectFacts := filterFacts(allFacts, "project") + skillFacts := filterFacts(allFacts, "skill") + + fmt.Printf("chats facts: extracted %d facts (%d phone, %d email, %d linkedin, %d project, %d skill)\n", + len(allFacts), len(phoneFacts), len(emailFacts), len(linkedinFacts), len(projectFacts), len(skillFacts)) + + factsDir := filepath.Join(root, "facts") + if err := os.MkdirAll(factsDir, 0755); err != nil { + fmt.Fprintf(os.Stderr, "chats facts: mkdir %s: %v\n", factsDir, err) + return 1 + } + factsPath := filepath.Join(factsDir, "chat-facts.json") + data, err := json.MarshalIndent(allFacts, "", " ") + if err != nil { + fmt.Fprintf(os.Stderr, "chats facts: marshal: %v\n", err) + return 1 + } + if err := os.WriteFile(factsPath, data, 0644); err != nil { + fmt.Fprintf(os.Stderr, "chats facts: write %s: %v\n", factsPath, err) + return 1 + } + fmt.Printf("chats facts: saved to %s\n", factsPath) + + writeFactsToBrain(root, allFacts) + + return 0 +} + +func extractFacts(jsonlPath, chatID string) ([]ExtractedFact, string) { + f, err := os.Open(jsonlPath) + if err != nil { + return nil, "" + } + defer f.Close() + + var facts []ExtractedFact + chatName := "" + + scanner := bufio.NewScanner(f) + scanner.Buffer(make([]byte, 1<<20), 1<<20) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + var msg Message + if err := json.Unmarshal([]byte(line), &msg); err != nil { + continue + } + if chatName == "" && msg.From != "" { + chatName = msg.From + } + + text := msg.Text + + phones := phoneRegex.FindAllString(text, -1) + for _, p := range phones { + p = strings.TrimSpace(p) + p = strings.Trim(p, "()- \t") + if isValidPhone(p) { + facts = append(facts, ExtractedFact{ + ChatID: chatID, + ChatName: chatName, + Platform: "telegram", + FactType: "phone", + Value: p, + Source: "chat:" + msg.ID, + MessageID: msg.ID, + }) + } + } + + emails := emailRegex.FindAllString(text, -1) + for _, e := range emails { + facts = append(facts, ExtractedFact{ + ChatID: chatID, + ChatName: chatName, + Platform: "telegram", + FactType: "email", + Value: strings.ToLower(e), + Source: "chat:" + msg.ID, + MessageID: msg.ID, + }) + } + + linkedins := linkedinRegex.FindAllString(text, -1) + for _, l := range linkedins { + facts = append(facts, ExtractedFact{ + ChatID: chatID, + ChatName: chatName, + Platform: "telegram", + FactType: "linkedin", + Value: "https://" + l, + Source: "chat:" + msg.ID, + MessageID: msg.ID, + }) + } + + if matches := projectRegex.FindStringSubmatch(text); len(matches) > 1 { + facts = append(facts, ExtractedFact{ + ChatID: chatID, + ChatName: chatName, + Platform: "telegram", + FactType: "project", + Value: strings.TrimSpace(matches[1]), + Source: "chat:" + msg.ID, + MessageID: msg.ID, + }) + } + + if matches := dealRegex.FindStringSubmatch(text); len(matches) > 2 { + facts = append(facts, ExtractedFact{ + ChatID: chatID, + ChatName: chatName, + Platform: "telegram", + FactType: "deal", + Value: strings.TrimSpace(matches[2]), + Source: "chat:" + msg.ID, + MessageID: msg.ID, + }) + } + + if matches := skillRegex.FindStringSubmatch(text); len(matches) > 3 { + facts = append(facts, ExtractedFact{ + ChatID: chatID, + ChatName: chatName, + Platform: "telegram", + FactType: "skill", + Value: strings.TrimSpace(matches[0]), + Source: "chat:" + msg.ID, + MessageID: msg.ID, + }) + } + } + return facts, chatName +} + +func filterFacts(facts []ExtractedFact, factType string) []ExtractedFact { + var result []ExtractedFact + for _, f := range facts { + if f.FactType == factType { + result = append(result, f) + } + } + return result +} + +func writeFactsToBrain(root string, facts []ExtractedFact) { + indexScript := filepath.Join(root, "bin", "kb", "index") + if _, err := os.Stat(indexScript); os.IsNotExist(err) { + fmt.Fprintf(os.Stderr, "chats facts: kb/index not found, skipping brain write\n") + return + } + + mdDir := filepath.Join(chatsDir(), "facts") + if err := os.MkdirAll(mdDir, 0755); err != nil { + fmt.Fprintf(os.Stderr, "chats facts: mkdir %s: %v\n", mdDir, err) + return + } + + var sb strings.Builder + sb.WriteString("---\n") + sb.WriteString("root: facts\n") + sb.WriteString("---\n\n") + sb.WriteString("# Chat-Derived Facts\n\n") + for _, f := range facts { + sb.WriteString(fmt.Sprintf("- **%s**: %s (source: %s, chat: %s)\n", + f.FactType, f.Value, f.Source, f.ChatName)) + } + sb.WriteString("\n") + + factsMD := filepath.Join(mdDir, "chat-facts.md") + if err := os.WriteFile(factsMD, []byte(sb.String()), 0644); err != nil { + fmt.Fprintf(os.Stderr, "chats facts: write %s: %v\n", factsMD, err) + return + } + + cmd := exec.Command(indexScript, "--corpus", mdDir, "--skip-indexes") + var outBuf, errBuf bytes.Buffer + cmd.Stdout = &outBuf + cmd.Stderr = &errBuf + cmd.Dir = root + if err := cmd.Run(); err != nil { + fmt.Fprintf(os.Stderr, "chats facts: brain index: %v\n%s", err, errBuf.String()) + return + } + fmt.Printf("chats facts: written to brain (%s)\n", strings.TrimSpace(outBuf.String())) +} diff --git a/bin/chats/go.mod b/bin/chats/go.mod new file mode 100644 index 0000000..9df1766 --- /dev/null +++ b/bin/chats/go.mod @@ -0,0 +1,3 @@ +module github.com/eSlider/2dph/bin/chats + +go 1.25.0 diff --git a/bin/chats/go.sum b/bin/chats/go.sum new file mode 100644 index 0000000..e69de29 diff --git a/bin/chats/import_cmd.go b/bin/chats/import_cmd.go new file mode 100644 index 0000000..6a0ac6e --- /dev/null +++ b/bin/chats/import_cmd.go @@ -0,0 +1,204 @@ +package main + +import ( + "bufio" + "bytes" + "encoding/json" + "flag" + "fmt" + "html" + "os" + "path/filepath" + "sort" + "strings" +) + +func runImport(args []string) int { + fs := flag.NewFlagSet("chats import", flag.ContinueOnError) + help := fs.Bool("help", false, "") + fs.SetOutput(os.Stderr) + if err := fs.Parse(args); err != nil { + return 2 + } + if *help { + fmt.Fprintln(os.Stderr, "usage: chats import") + return 0 + } + + root := chatsDir() + mdRoot := filepath.Join(root, "md") + glob := filepath.Join(root, "telegram", "*", "messages.jsonl") + + matches, err := filepath.Glob(glob) + if err != nil { + fmt.Fprintf(os.Stderr, "chats import: glob %s: %v\n", glob, err) + return 1 + } + if len(matches) == 0 { + fmt.Fprintf(os.Stderr, "chats import: no messages.jsonl found under %s\n", root) + return 1 + } + + written := 0 + failed := 0 + for _, jsonlPath := range matches { + chatID := filepath.Base(filepath.Dir(jsonlPath)) + + messages, chatName, err := readJSONL(jsonlPath) + if err != nil { + fmt.Fprintf(os.Stderr, "chats import: read %s: %v\n", jsonlPath, err) + failed++ + continue + } + if len(messages) == 0 { + continue + } + if chatName == "" { + chatName = chatID + } + + participants := collectParticipants(messages) + chatType := "personal" + if len(participants) > 3 { + chatType = "group" + } + + firstID := "" + if len(messages) > 0 { + firstID = messages[0].ID + } + + var b bytes.Buffer + b.WriteString("---\n") + fmt.Fprintf(&b, "id: %s\n", firstID) + fmt.Fprintf(&b, "platform: telegram\n") + fmt.Fprintf(&b, "chat_id: %s\n", chatID) + fmt.Fprintf(&b, "chat_name: %s\n", escapeYAML(chatName)) + fmt.Fprintf(&b, "participants: [") + for i, p := range participants { + if i > 0 { + b.WriteString(", ") + } + b.WriteString(escapeYAML(p)) + } + b.WriteString("]\n") + fmt.Fprintf(&b, "message_count: %d\n", len(messages)) + fmt.Fprintf(&b, "type: %s\n", chatType) + b.WriteString("---\n\n") + fmt.Fprintf(&b, "# Чат с %s\n\n", chatName) + + for _, msg := range messages { + ts := msg.Timestamp + if len(ts) > 10 { + ts = ts[:10] + } + text := msg.Text + text = html.UnescapeString(text) + text = strings.ReplaceAll(text, "\n", "\n ") + + line := fmt.Sprintf("**%s** — %s: %s", ts, msg.From, text) + if msg.Media != nil { + line += " *(" + *msg.Media + ")*" + } + b.WriteString(line + "\n\n") + } + + mdFile := filepath.Join(mdRoot, "telegram", sanitizeDir(chatName), "messages.md") + if err := os.MkdirAll(filepath.Dir(mdFile), 0755); err != nil { + fmt.Fprintf(os.Stderr, "chats import: mkdir %s: %v\n", filepath.Dir(mdFile), err) + failed++ + continue + } + if err := os.WriteFile(mdFile, b.Bytes(), 0644); err != nil { + fmt.Fprintf(os.Stderr, "chats import: write %s: %v\n", mdFile, err) + failed++ + continue + } + written++ + } + fmt.Printf("chats import: %d chats written", written) + if failed > 0 { + fmt.Printf(", %d failed", failed) + } + fmt.Println() + if failed > 0 { + return 1 + } + return 0 +} + +func readJSONL(path string) ([]Message, string, error) { + f, err := os.Open(path) + if err != nil { + return nil, "", err + } + defer f.Close() + + var messages []Message + scanner := bufio.NewScanner(f) + scanner.Buffer(make([]byte, 1<<20), 1<<20) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + var msg Message + if err := json.Unmarshal([]byte(line), &msg); err != nil { + continue + } + messages = append(messages, msg) + } + if err := scanner.Err(); err != nil { + return messages, "", err + } + + chatName := "" + if len(messages) > 0 { + nameCounts := make(map[string]int) + for _, msg := range messages { + nameCounts[msg.From]++ + } + best := "" + bestN := 0 + for name, n := range nameCounts { + if name != "" && name != "unknown" && n > bestN { + best = name + bestN = n + } + } + if best != "" { + chatName = best + } + } + return messages, chatName, nil +} + +func collectParticipants(messages []Message) []string { + seen := make(map[string]bool) + var result []string + for _, msg := range messages { + if msg.From == "" || seen[msg.From] { + continue + } + seen[msg.From] = true + result = append(result, msg.From) + } + sort.Strings(result) + return result +} + +func escapeYAML(s string) string { + if strings.ContainsAny(s, ":#,[]{}'\"") || strings.HasPrefix(s, "-") { + return `"` + strings.ReplaceAll(s, `"`, `\"`) + `"` + } + return s +} + +func sanitizeDir(name string) string { + r := strings.NewReplacer( + "/", "_", "\\", "_", ":", "_", "*", "_", + "?", "_", "\"", "_", "<", "_", ">", "_", "|", "_", + " ", "_", + ) + return strings.TrimSpace(r.Replace(name)) +} diff --git a/bin/chats/index_cmd.go b/bin/chats/index_cmd.go new file mode 100644 index 0000000..dcc0ada --- /dev/null +++ b/bin/chats/index_cmd.go @@ -0,0 +1,56 @@ +package main + +import ( + "bytes" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +func runIndex(args []string) int { + fs := flag.NewFlagSet("chats index", flag.ContinueOnError) + help := fs.Bool("help", false, "") + fs.SetOutput(os.Stderr) + if err := fs.Parse(args); err != nil { + return 2 + } + if *help { + fmt.Fprintln(os.Stderr, "usage: chats index") + return 0 + } + + root := repoRoot() + mdDir := filepath.Join(chatsDir(), "md") + + _, err := os.Stat(mdDir) + if os.IsNotExist(err) { + fmt.Fprintf(os.Stderr, "chats index: no chat markdown at %s; run 'chats import' first\n", mdDir) + return 1 + } + + indexScript := filepath.Join(root, "bin", "kb", "index") + if _, err := os.Stat(indexScript); os.IsNotExist(err) { + fmt.Fprintf(os.Stderr, "chats index: %s not found\n", indexScript) + return 1 + } + + cmd := exec.Command(indexScript, "--corpus", mdDir) + var outBuf, errBuf bytes.Buffer + cmd.Stdout = &outBuf + cmd.Stderr = &errBuf + cmd.Dir = root + + if err := cmd.Run(); err != nil { + fmt.Fprintf(os.Stderr, "chats index: %v\n%s", err, errBuf.String()) + return 1 + } + result := strings.TrimSpace(outBuf.String()) + if result == "" { + result = strings.TrimSpace(errBuf.String()) + } + fmt.Printf("chats index: %s\n", result) + return 0 +} diff --git a/bin/chats/main.go b/bin/chats/main.go new file mode 100644 index 0000000..99a0052 --- /dev/null +++ b/bin/chats/main.go @@ -0,0 +1,116 @@ +// bin/chats - sync, import, index, extract facts, and apply chat data +// from Telegram, WhatsApp, LinkedIn into the brain and OnlyOffice CRM. +// +// Usage: +// +// chats sync telegram [--limit N] [--since DATE] [--phone PHONE] +// chats sync whatsapp [--qr] [--limit N] +// chats sync linkedin [--limit N] +// chats import # JSONL → MD (all sources) +// chats index # rebuild var/kb.lbug with chats +// chats facts # extract + cross-check +// chats apply [--dry-run] # push to OnlyOffice CRM +package main + +import ( + "fmt" + "os" + "strings" +) + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + cmd := os.Args[1] + args := os.Args[2:] + switch cmd { + case "sync": + if len(args) < 1 { + usage() + os.Exit(2) + } + platform := args[0] + platformArgs := args[1:] + switch platform { + case "telegram": + os.Exit(runSyncTelegram(platformArgs)) + case "whatsapp": + fmt.Fprintf(os.Stderr, "chats: WhatsApp not implemented yet\n") + os.Exit(1) + case "linkedin": + fmt.Fprintf(os.Stderr, "chats: LinkedIn not implemented yet\n") + os.Exit(1) + default: + fmt.Fprintf(os.Stderr, "chats: unknown platform %q\n", platform) + os.Exit(2) + } + case "import": + os.Exit(runImport(args)) + case "index": + os.Exit(runIndex(args)) + case "facts": + os.Exit(runFacts(args)) + case "apply": + os.Exit(runApply(args)) + case "help", "-h", "--help": + usage() + return + default: + fmt.Fprintf(os.Stderr, "chats: unknown command %q\n", cmd) + usage() + os.Exit(2) + } +} + +func usage() { + w := os.Stderr + fmt.Fprintln(w, `Usage: chats [args] + +Commands: + sync telegram [--limit N] [--since DATE] [--phone PHONE] + sync whatsapp [--qr] [--limit N] + sync linkedin [--limit N] + import JSONL → MD (all sources) + index rebuild var/kb.lbug with chats + facts extract + cross-check facts + apply [--dry-run] push to OnlyOffice CRM + +Output layout: + var/chats///messages.jsonl + var/chats/md///messages.md`) +} + +// repoRoot locates the 2dph project root by walking up from the binary. +func repoRoot() string { + if v := os.Getenv("KB_ROOT"); v != "" { + return v + } + wd, err := os.Getwd() + if err != nil { + return "." + } + for i := 0; i < 10; i++ { + if _, err := os.Stat(wd + "/var"); err == nil { + return wd + } + if _, err := os.Stat(wd + "/.git"); err == nil { + return wd + } + parent := wd + if idx := strings.LastIndex(wd, "/"); idx >= 0 { + parent = wd[:idx] + } + if parent == wd { + break + } + wd = parent + } + return "." +} + +// chatsDir returns var/chats under the repo root. +func chatsDir() string { + return repoRoot() + "/var/chats" +} diff --git a/bin/chats/mcpclient.go b/bin/chats/mcpclient.go new file mode 100644 index 0000000..44264d7 --- /dev/null +++ b/bin/chats/mcpclient.go @@ -0,0 +1,424 @@ +package main + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +type MCPClient struct { + cmd *exec.Cmd + stdin *bufio.Writer + stdout *bufio.Scanner + msgID int +} + +type mcpRequest struct { + JSONRPC string `json:"jsonrpc"` + ID int `json:"id"` + Method string `json:"method"` + Params interface{} `json:"params,omitempty"` +} + +type mcpResponse struct { + JSONRPC string `json:"jsonrpc"` + ID int `json:"id"` + Result json.RawMessage `json:"result,omitempty"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error,omitempty"` +} + +type mcpToolResult struct { + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError,omitempty"` +} + +type ListChatsResult struct { + ChatID int64 `json:"chat_id"` + Title string `json:"name"` + Type string `json:"type"` + Username string `json:"username,omitempty"` +} + +type listChatsEnvelope struct { + Results []ListChatsResult `json:"results"` +} + +type historyEnvelope struct { + Results []GetHistoryResult `json:"results"` +} + +type GetHistoryResult struct { + ID int `json:"id"` + Sender string `json:"sender"` + Date string `json:"date"` + Text string `json:"text"` + Media string `json:"media,omitempty"` + Out bool `json:"out,omitempty"` +} + +func NewMCPClient(ctx context.Context, apiID int, apiHash, phone, sessionString, mcpDir string) (*MCPClient, error) { + env := os.Environ() + env = append(env, + fmt.Sprintf("TELEGRAM_API_ID=%d", apiID), + fmt.Sprintf("TELEGRAM_API_HASH=%s", apiHash), + fmt.Sprintf("TELEGRAM_PHONE=%s", phone), + fmt.Sprintf("TELEGRAM_SESSION_STRING=%s", sessionString), + "MCP_TRANSPORT=stdio", + ) + + serverPath := filepath.Join(mcpDir, ".venv", "bin", "python3") + mainPath := filepath.Join(mcpDir, "main.py") + + cmd := exec.CommandContext(ctx, serverPath, mainPath) + cmd.Env = env + cmd.Dir = mcpDir + + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, fmt.Errorf("stdin pipe: %w", err) + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, fmt.Errorf("stdout pipe: %w", err) + } + cmd.Stderr = os.Stderr + + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("start mcp: %w", err) + } + + c := &MCPClient{ + cmd: cmd, + stdin: bufio.NewWriter(stdin), + stdout: bufio.NewScanner(stdout), + msgID: 0, + } + c.stdout.Buffer(make([]byte, 1<<20), 1<<20) + + if err := c.initialize(ctx); err != nil { + c.Close() + return nil, fmt.Errorf("initialize: %w", err) + } + return c, nil +} + +func (c *MCPClient) nextID() int { + c.msgID++ + return c.msgID +} + +func (c *MCPClient) sendRequest(ctx context.Context, method string, params interface{}) (json.RawMessage, error) { + id := c.nextID() + req := mcpRequest{ + JSONRPC: "2.0", + ID: id, + Method: method, + Params: params, + } + body, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("marshal: %w", err) + } + + if _, err := c.stdin.Write(body); err != nil { + return nil, fmt.Errorf("write: %w", err) + } + if err := c.stdin.WriteByte('\n'); err != nil { + return nil, fmt.Errorf("write newline: %w", err) + } + if err := c.stdin.Flush(); err != nil { + return nil, fmt.Errorf("flush: %w", err) + } + + for c.stdout.Scan() { + line := c.stdout.Text() + if line == "" { + continue + } + + var resp mcpResponse + if err := json.Unmarshal([]byte(line), &resp); err != nil { + return nil, fmt.Errorf("unmarshal response: %w\nline: %s", err, line[:min(len(line), 500)]) + } + if resp.Error != nil { + return nil, fmt.Errorf("rpc error %d: %s", resp.Error.Code, resp.Error.Message) + } + return resp.Result, nil + } + return nil, fmt.Errorf("no response: %w", c.stdout.Err()) +} + +func (c *MCPClient) initialize(ctx context.Context) error { + params := map[string]interface{}{ + "protocolVersion": "2024-11-05", + "capabilities": map[string]interface{}{}, + "clientInfo": map[string]string{ + "name": "chats-sync", + "version": "0.1.0", + }, + } + _, err := c.sendRequest(ctx, "initialize", params) + return err +} + +func (c *MCPClient) ListChats(ctx context.Context, chatType string, limit int) ([]ListChatsResult, error) { + args := map[string]interface{}{ + "chat_type": chatType, + "limit": limit, + } + result, err := c.sendRequest(ctx, "tools/call", map[string]interface{}{ + "name": "list_chats", + "arguments": args, + }) + if err != nil { + return nil, err + } + + var toolRes mcpToolResult + if err := json.Unmarshal(result, &toolRes); err != nil { + return nil, fmt.Errorf("unmarshal tool result: %w", err) + } + if toolRes.IsError { + msg := "unknown" + if len(toolRes.Content) > 0 { + msg = toolRes.Content[0].Text + } + return nil, fmt.Errorf("list_chats error: %s", msg) + } + if len(toolRes.Content) == 0 { + return nil, nil + } + + text := toolRes.Content[0].Text + if text == "" || text == "No chats found matching the criteria." { + return nil, nil + } + + var env listChatsEnvelope + if err := json.Unmarshal([]byte(text), &env); err != nil { + var arr []ListChatsResult + if err2 := json.Unmarshal([]byte(text), &arr); err2 != nil { + return nil, fmt.Errorf("parse chats: %w (also tried array: %v)\nbody: %s", err, err2, text[:min(len(text), 500)]) + } + return arr, nil + } + return env.Results, nil +} + +func (c *MCPClient) GetHistory(ctx context.Context, chatID int64, limit int) ([]GetHistoryResult, error) { + args := map[string]interface{}{ + "chat_id": chatID, + "limit": limit, + } + result, err := c.sendRequest(ctx, "tools/call", map[string]interface{}{ + "name": "get_history", + "arguments": args, + }) + if err != nil { + return nil, err + } + + var toolRes mcpToolResult + if err := json.Unmarshal(result, &toolRes); err != nil { + return nil, fmt.Errorf("unmarshal tool result: %w", err) + } + if toolRes.IsError { + msg := "unknown" + if len(toolRes.Content) > 0 { + msg = toolRes.Content[0].Text + } + return nil, fmt.Errorf("get_history error: %s", msg) + } + if len(toolRes.Content) == 0 { + return nil, nil + } + + text := toolRes.Content[0].Text + if text == "" || text == "No messages found for this page." || text == "No messages found matching the criteria." { + return nil, nil + } + + var env historyEnvelope + if err := json.Unmarshal([]byte(text), &env); err != nil { + var arr []GetHistoryResult + if err2 := json.Unmarshal([]byte(text), &arr); err2 != nil { + return nil, fmt.Errorf("parse history: %w (also tried array: %v)\nbody: %s", err, err2, text[:min(len(text), 500)]) + } + return arr, nil + } + return env.Results, nil +} + +func (c *MCPClient) Close() error { + if c.stdin != nil { + c.stdin.Flush() + } + if c.cmd != nil && c.cmd.Process != nil { + c.cmd.Process.Kill() + } + return nil +} + +type TelegramMCPSource struct { + mcpDir string + apiID int + apiHash string + phone string + sessionStr string + limit int +} + +func NewTelegramMCPSource(apiID int, apiHash, phone, sessionString, mcpDir string) *TelegramMCPSource { + return &TelegramMCPSource{ + mcpDir: mcpDir, + apiID: apiID, + apiHash: apiHash, + phone: phone, + sessionStr: sessionString, + } +} + +func (s *TelegramMCPSource) Name() string { return "telegram" } + +func (s *TelegramMCPSource) Sync(ctx context.Context, outDir string, limit int) error { + if limit > 0 { + s.limit = limit + } + + client, err := NewMCPClient(ctx, s.apiID, s.apiHash, s.phone, s.sessionStr, s.mcpDir) + if err != nil { + return fmt.Errorf("mcp client: %w", err) + } + defer client.Close() + + chats, err := client.ListChats(ctx, "user", 100) + if err != nil { + return fmt.Errorf("list chats: %w", err) + } + if len(chats) == 0 { + fmt.Println("chats: no personal chats found") + return nil + } + fmt.Printf("chats: found %d personal chats\n", len(chats)) + + var filtered []ListChatsResult + for _, c := range chats { + if strings.Contains(strings.ToLower(c.Username), "bot") { + continue + } + if c.ChatID == 777000 { // Telegram service + continue + } + filtered = append(filtered, c) + } + fmt.Printf("chats: %d after filter (bots excluded)\n", len(filtered)) + + for _, chat := range filtered { + chatID := fmt.Sprintf("user_%d", chat.ChatID) + chatName := chat.Title + if chatName == "" { + chatName = chatID + } + + chatDir := filepath.Join(outDir, "telegram", chatID) + if err := os.MkdirAll(chatDir, 0755); err != nil { + fmt.Fprintf(os.Stderr, "chats: mkdir %s: %v\n", chatDir, err) + continue + } + + jsonlPath := filepath.Join(chatDir, "messages.jsonl") + f, err := os.Create(jsonlPath) + if err != nil { + fmt.Fprintf(os.Stderr, "chats: create %s: %v\n", jsonlPath, err) + continue + } + + msgLimit := 100 + if s.limit > 0 { + msgLimit = s.limit + } + + msgs, err := client.GetHistory(ctx, chat.ChatID, msgLimit) + if err != nil { + fmt.Fprintf(os.Stderr, "chats: get_history for %s: %v\n", chatName, err) + f.Close() + continue + } + + enc := json.NewEncoder(f) + written := 0 + for _, m := range msgs { + if m.Out { + continue + } + text := m.Text + if text == "" && m.Media != "" { + text = fmt.Sprintf("[%s]", m.Media) + } + if text == "" { + continue + } + + sender := cleanSender(m.Sender) + ts := m.Date + if t, err := time.Parse(time.RFC3339, m.Date); err == nil { + ts = t.UTC().Format(time.RFC3339) + } + + chatMsg := Message{ + ID: fmt.Sprintf("tg_%d_%d", chat.ChatID, m.ID), + Timestamp: ts, + From: sender, + Text: text, + Platform: "telegram", + } + if m.Media != "" { + desc := fmt.Sprintf("[%s]", m.Media) + chatMsg.Media = &desc + } + if err := enc.Encode(chatMsg); err != nil { + fmt.Fprintf(os.Stderr, "chats: encode msg: %v\n", err) + continue + } + written++ + } + f.Close() + + if written > 0 { + fmt.Printf("chats: synced %s (%s) — %d messages\n", chatName, chatID, written) + } + } + + return nil +} + +func cleanSender(sender string) string { + if idx := strings.Index(sender, " ("); idx > 0 { + sender = sender[:idx] + } else if idx := strings.Index(sender, " @"); idx > 0 { + sender = sender[:idx] + } + if idx := strings.Index(sender, " ["); idx > 0 { + sender = sender[:idx] + } + return sender +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/bin/chats/source.go b/bin/chats/source.go new file mode 100644 index 0000000..9223d10 --- /dev/null +++ b/bin/chats/source.go @@ -0,0 +1,52 @@ +package main + +import ( + "context" + "fmt" + "os" + "time" +) + +type Source interface { + Name() string + Sync(ctx context.Context, outDir string, limit int) error +} + +type Message struct { + ID string `json:"id"` + Timestamp string `json:"ts"` + From string `json:"from"` + Text string `json:"text"` + Media *string `json:"media,omitempty"` + Platform string `json:"platform"` +} + +type ChatInfo struct { + ID string `json:"id"` + Platform string `json:"platform"` + Name string `json:"name"` + Participants []string `json:"participants"` + Type string `json:"type"` + MessageCount int `json:"messageCount"` + LastTS string `json:"lastTs,omitempty"` +} + +func envVar(key, fallback string) string { + if v := os.Getenv(key); v != "" { + return v + } + return fallback +} + +func parseSince(s string) (time.Time, error) { + for _, layout := range []string{ + time.RFC3339, + "2006-01-02T15:04:05", + "2006-01-02", + } { + if t, err := time.Parse(layout, s); err == nil { + return t, nil + } + } + return time.Time{}, fmt.Errorf("cannot parse --since %q; use YYYY-MM-DD or RFC3339", s) +} diff --git a/bin/chats/sync_cmd.go b/bin/chats/sync_cmd.go new file mode 100644 index 0000000..37f3252 --- /dev/null +++ b/bin/chats/sync_cmd.go @@ -0,0 +1,108 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" +) + +func runSyncTelegram(args []string) int { + fs := flag.NewFlagSet("chats sync telegram", flag.ContinueOnError) + limit := fs.Int("limit", 0, "max messages per chat (0 = all)") + phone := fs.String("phone", "", "phone number (default env TELEGRAM_PHONE)") + help := fs.Bool("help", false, "") + fs.SetOutput(os.Stderr) + if err := fs.Parse(args); err != nil { + return 2 + } + if *help { + fmt.Fprintln(os.Stderr, "usage: chats sync telegram [--limit N] [--phone PHONE]") + return 0 + } + + apiIDStr := envVar("TELEGRAM_API_ID", "") + apiHash := envVar("TELEGRAM_API_HASH", "") + sessionStr := envVar("TELEGRAM_SESSION_STRING", "") + phoneNum := *phone + if phoneNum == "" { + phoneNum = envVar("TELEGRAM_PHONE", "") + } + if apiIDStr == "" || apiHash == "" || phoneNum == "" { + fmt.Fprintln(os.Stderr, "chats: need TELEGRAM_API_ID, TELEGRAM_API_HASH, TELEGRAM_PHONE in env") + return 2 + } + apiID, err := strconv.Atoi(apiIDStr) + if err != nil { + fmt.Fprintf(os.Stderr, "chats: invalid TELEGRAM_API_ID %q\n", apiIDStr) + return 2 + } + + // Locate telegram-mcp directory + mcpDirs := []string{ + envVar("TELEGRAM_MCP_DIR", ""), + "/mnt/8TB/projects/eslider/mcp-servers/telegram-mcp", + filepath.Join(os.Getenv("HOME"), "projects", "eSlider", "mcp-servers", "telegram-mcp"), + } + var mcpDir string + for _, d := range mcpDirs { + if d != "" { + if _, err := os.Stat(filepath.Join(d, "main.py")); err == nil { + mcpDir = d + break + } + } + } + if mcpDir == "" { + fmt.Fprintln(os.Stderr, "chats: telegram-mcp not found; set TELEGRAM_MCP_DIR") + return 1 + } + + if sessionStr == "" { + // Fallback: try to read from telegram-mcp .env + envPath := filepath.Join(mcpDir, ".env") + if data, err := os.ReadFile(envPath); err == nil { + for _, line := range strings.Split(string(data), "\n") { + if strings.HasPrefix(line, "TELEGRAM_SESSION_STRING=") { + sessionStr = strings.TrimPrefix(line, "TELEGRAM_SESSION_STRING=") + sessionStr = strings.Trim(sessionStr, "\"'") + break + } + } + } + } + if sessionStr == "" { + // Also try the env file at /mnt/8TB/projects/eslider/mcp-servers/telegram.env + envPath := "/mnt/8TB/projects/eslider/mcp-servers/telegram.env" + if data, err := os.ReadFile(envPath); err == nil { + for _, line := range strings.Split(string(data), "\n") { + if strings.HasPrefix(line, "TELEGRAM_SESSION_STRING=") { + sessionStr = strings.TrimPrefix(line, "TELEGRAM_SESSION_STRING=") + sessionStr = strings.Trim(sessionStr, "\"'") + break + } + } + } + } + if sessionStr == "" { + fmt.Fprintln(os.Stderr, "chats: TELEGRAM_SESSION_STRING not found in env or .env files") + return 1 + } + + src := NewTelegramMCPSource(apiID, apiHash, phoneNum, sessionStr, mcpDir) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + start := time.Now() + if err := src.Sync(ctx, chatsDir(), *limit); err != nil { + fmt.Fprintf(os.Stderr, "chats sync telegram: %v\n", err) + return 1 + } + fmt.Printf("chats sync telegram: completed in %s\n", time.Since(start).Round(time.Millisecond)) + return 0 +} diff --git a/docs/chat-import-plan.md b/docs/chat-import-plan.md index de9a7cb..b37a241 100644 --- a/docs/chat-import-plan.md +++ b/docs/chat-import-plan.md @@ -18,8 +18,627 @@ OO_CLI path to oo binary (default: $HOME/go/bin/oo) ## Quick reference +<<<<<<< HEAD ./bin/chat sync telegram --limit 100 ./bin/chat import ./bin/chat index # delegates to bin/kb/index ./bin/chat facts ./bin/chat apply --dry-run +||||||| parent of 313599b (bin/chats: Phase 1 MVP — Telegram sync/import/index/facts/apply) +--- + +## 1. Находки / Discoveries + +### Доступные платформы и данные + +| Platform | Status | Credentials | Go Library | +|----------|--------|-------------|------------| +| **Telegram** | ✅ Active MCP server, session string exists | API ID: `30382285`, API Hash: `fd8a8c1e987bb908b457f374b721c062`, Session string in `/mnt/8TB/projects/eslider/mcp-servers/telegram.env` and `/mnt/8TB/projects/eslider/mcp-servers/telegram-mcp/.env`. Phone: `+34643861471` | [`iyear/tdl`](https://github.com/iyear/tdl) (7.5k★, Go, gotd/td, экспорт в JSON) | +| **WhatsApp** | ✅ MCP server exists + Go bridge. **No `messages.db` yet** (empty). Needs QR re-auth + sync first | MCP: `/mnt/8TB/projects/eslider/mcp-servers/whatsapp-mcp/`. Go bridge uses `tulir/whatsmeow` | [`tulir/whatsmeow`](https://github.com/tulir/whatsmeow) (7k★, Go, multi-device API) | +| **LinkedIn** | ✅ Connected via `mcp-server-linkedin`. Full browser profile + cookies | Browser profile: `~/.linkedin-mcp/profile/`. Active cookies: `~/.linkedin-mcp/storage-state.json`. Cookie backup: `~/.linkedin-mcp/cookies.json.bak-agent`. Script cookies: `/mnt/8TB/projects/eslider/cv/bin/creds/linkedin.cookies.json`. JobHunt session: `/mnt/8TB/projects/eslider/JobHunt/sessions/linkedin.json`. Env: `/mnt/8TB/projects/eslider/cv/.env` (`LINKEDIN_AUTH_TOKEN`). | [`swiftlysingh/lnk`](https://github.com/swiftlysingh/lnk) (Go CLI, Voyager API) | +| **Gmail** | ✅ Already working pipeline (`bin/mail/sync.go` + `bin/mail/import`) | OAuth at `~/.gmail-mcp/` | Already in 2dph | +| **Twitter/X.com** | ⚠️ Password only, no MCP, no session. Not needed now. | `/mnt/8TB/projects/eslider/scratches/Accounts/twitter-pass.txt` | [`benoitpetit/xsh`](https://github.com/benoitpetit/xsh) (Go CLI, cookie auth) | +| **Google Calendar** | ❌ Not connected yet. Gmail OAuth could extend | None | Нужен Google Calendar API | + +### Telegram session (проверен, активен) + +- API credentials in: `/mnt/8TB/projects/eslider/mcp-servers/telegram.env` + - `TELEGRAM_API_ID=30382285` + - `TELEGRAM_API_HASH=fd8a8c1e987bb908b457f374b721c062` + - `TELEGRAM_PHONE=+34643861471` + - `TELEGRAM_SESSION_STRING` — длинная строка, active session +- MCP server launcher: `/mnt/8TB/projects/eslider/mcp-servers/run-telegram-mcp.sh` +- Session generator: `/mnt/8TB/projects/eslider/mcp-servers/telegram-mcp/session_string_generator.py` +- Regenerator: `/mnt/8TB/projects/eslider/mcp-servers/regen-telegram-session.py` + +### LinkedIn authentication (работает) + +- Запускается через Cursor MCP: `uvx mcp-server-linkedin@latest --user-data-dir /home/ano/.linkedin-mcp/profile --no-auto-import` +- Cookie файлы: + - `~/.linkedin-mcp/storage-state.json` — Playwright storage state (активные cookies) + - `~/.linkedin-mcp/cookies.json.bak-agent` — резерв + - `~/.linkedin-mcp/cookies.json.bak-webtop-20260728T143535Z` — резерв + - `/mnt/8TB/projects/eslider/cv/bin/creds/linkedin.cookies.json` — для скриптов + - `/mnt/8TB/projects/eslider/JobHunt/sessions/linkedin.json` — JobHunt +- `li_at` token: `AQEDASsyC9gF8hS2AAABnC2A_vIAAAGcdZZfTk4Ahfp9wURXwTpCAAtRIw_htVldds5UN9JDr6L_hr3lN5CcPxnKDvy-CxQo0apf7K4pmftQzQYHdqpKNNzlNakcvEcOOalpqRxPhlTfIt8jxqevkZcr` +- Для Go подхода: можно читать cookie из `storage-state.json` и использовать LinkedIn Voyager API напрямую + +### WhatsApp bridge (не синхронизирован) + +- Go bridge at: `/mnt/8TB/projects/eslider/mcp-servers/whatsapp-mcp/whatsapp-bridge/` +- Использует `tulir/whatsmeow` Go library (уже в `go.mod`) +- SQLite store: `whatsapp-bridge/store/messages.db` — **не существует** (0 байт), нужна первая авторизация по QR +- Launcher: `/mnt/8TB/projects/eslider/mcp-servers/run-whatsapp-bridge-lharries.sh` +- QR images готовились: `/mnt/8TB/projects/produktor/work/data/export/whatsapp-qr.png` +- Пароль: `Oomtr78k39` (из `.env` OnlyOffice, он же для многих сервисов) + +### OnlyOffice доступ + +- `.env` в корне 2dph: `ONLYOFFICE_URL=https://office.produktor.io`, `ONLYOFFICE_USER=eslider@gmail.com`, `ONLYOFFICE_PASS=Oomtr78k39` +- `oo` CLI установлен: `/home/ano/go/bin/oo` (go-onlyoffice) +- go-onlyoffice repo: `/mnt/8TB/projects/eslider/go-onlyoffice/` (main, public) +- oo-workspace (каталог, private): `/mnt/8TB/projects/eslider/oo-workspace/` + +### Gitea + +- Instance: `https://git.produktor.io` (через `gitea-api` wrapper) +- Credentials: `~/.config/gitea/produktor.env` +- User: `eSlider`, admin +- Repos: 30+ приватных репозиториев +- **`eSlider/2dph` не существует на Gitea** — проект только на GitHub +- Для задачи: создать issue в существующем репо (например `eSlider/JobHunt`) или создать новый репо + +### Rambox + +Не используется. IndexedDB база не существует. + +--- + +## 2. Design Decisions + +### One binary: `bin/chats` + +```go +// Source interface — каждая платформа реализует +type Source interface { + Name() string // "telegram", "whatsapp", "linkedin" + Sync(ctx context.Context, out string) error +} + +// CLI subcommands +chats sync telegram [--limit N] [--since DATE] +chats sync whatsapp [--qr] [--limit N] +chats sync linkedin [--limit N] +chats import // JSONL → markdown (all sources) +chats index // rebuild var/kb.lbug +chats facts // extract + cross-check +chats apply // push to OO CRM [--dry-run] +``` + +### Directory layout (полные имена платформ) + +``` +var/chats/ + telegram//messages.jsonl + whatsapp//messages.jsonl + linkedin//messages.jsonl + md///message.md +``` + +### JSONL format (одна строка = одно сообщение) + +```json +{"id":"123","ts":"2026-01-15T10:30:00Z","from":"me","text":"Hello!","media":null,"platform":"telegram"} +``` + +### MD format (YAML frontmatter) + +```markdown +--- +id: tg_12345 +platform: telegram +chat_id: "-100123456" +chat_name: "John Doe" +participants: ["me", "John Doe"] +message_count: 1500 +type: personal +--- + +# Чат с John Doe + +**2026-01-15 10:30** — me: Hello! +``` + +### Фильтр "личный чат" + +- ≤3 участников +- Боты не считаются (Telegram: `user.is_bot == false`) +- Не канал, не публичная группа +- Telegram: `dialog.Type == "user"` или супергруппа с `participants_count <= 3` (исключая ботов) +- WhatsApp: `@s.whatsapp.net` (не `@g.us`) +- LinkedIn: 1:1 conversation + +### Очередность реализации + +1. **Telegram** — самый простой (session string + API ID/Hash активны) +2. **Import** — конвертер JSONL → MD +3. **Index** — интеграция с brain +4. **WhatsApp** — после первого QR +5. **Facts** — извлечение контактных данных +6. **LinkedIn** — после проверки cookie +7. **Apply** — запись в OO CRM + +### Тестирование (TDD, system tests only) + +- Никаких unit-тестов, никаких моков +- Тесты используют реальные данные (слепки) +- Workflow use-case как тест: + 1. Написать тест с реальным сценарием (sync → import → index → search) + 2. Проверить что файлы созданы + 3. Проверить что поиск возвращает ожидаемые результаты +- A/B тест: сравнить два последовательных sync на идентичность +- Integration: тест через `oo` CLI с `--dry-run` + +### Cross-check (detective method) + +Перед записью в OO CRM — минимум 2 независимых источника: +- S1: Чат (текст сообщения, автор, дата) +- S2: OO CRM (существующий контакт/компания/deal) через `oo persons list` / `oo persons get` +- S3 (опционально): Corpus SoT (`knowledge-mesh-seed.yaml`) или web-search +- Факты с ≥2 источниками → `root=facts` в brain + `approve=true` для CRM +- Факты с 1 источником → предложение на review (catalog-паттерн) + +### Apply в OO CRM + +Через `oo` CLI: +- `oo persons update --about ... --job-title ...` +- `oo contacts info-add --type Phone|LinkedIn --value ...` +- `oo opportunities create --contact-id ... --title ... --stage ...` +- `oo persons update --company-id ...` (ассоциация) +- `oo projects contacts ` (связь с проектом) + +--- + +## 3. Implementation Plan + +### Phase 1: Telegram sync + import (MVP) + +1. Create `bin/chats/` directory structure (nested Go module like `bin/kbsearch/`) +2. Implement `Source` interface +3. Implement `TelegramSource` using `gotd/td` (import from `iyear/tdl` or direct) +4. Write JSONL output to `var/chats/telegram//messages.jsonl` +5. Implement `chats import` — reads all JSONL, writes MD +6. Write system test: real sync → import → verify output + +### Phase 2: Brain indexing + +1. Implement `chats index` — rebuild `var/kb.lbug` with corpus + chats +2. Write system test: sync → import → index → `bin/kb/search "query"` → verify recall + +### Phase 3: WhatsApp + +1. Implement `WhatsAppSource` using `tulir/whatsmeow` +2. First run requires QR auth (save session for later reuse) +3. Write sync output to `var/chats/whatsapp//messages.jsonl` + +### Phase 4: Facts extraction + cross-check + +1. Implement `chats facts`: + - Phone regex: `\+?\d{7,15}` + - LinkedIn URL regex: `linkedin\.com/in/[\w-]+` + - Cross-check: match extracted values against OO CRM API +2. Write facts to brain as `root=facts` + +### Phase 5: LinkedIn + Apply + +1. Implement `LinkedInSource` using Voyager API + cookies from `storage-state.json` +2. Implement `chats apply`: + - Build YAML catalog (as `oo-workspace` does) + - Run `oo catalog match` → review → `oo catalog apply` + +--- + +## 4. Existing code references + +| Code | Path | Purpose | +|------|------|---------| +| Mail sync | `bin/mail/sync.go` + `bin/mail/sync/` | Pattern for chat sync (Go subprocess pattern) | +| Mail import | `bin/mail/import` | Pattern for JSONL → MD conversion | +| Mail index | `bin/mail/index_mail` | Pattern for brain rebuild with new corpus | +| kbsearch | `bin/kbsearch/` | Go module pattern (nested, `system_ladybug` tag) | +| kb/index | `bin/kb/index` | Python brain index (upsert_leaf, init_schema) | +| kblib | `bin/tools/kblib.py` | Python brain core (connect, upsert_leaf, hybrid_search) | +| facts/crm | `bin/facts/crm` | Cross-check CRM facts pattern | +| facts/extract | `bin/facts/extract` | 2-source fact extraction pattern | +| oo-workspace catalog | `/mnt/8TB/projects/eslider/oo-workspace/catalog/` | Catalog scan/match/apply pattern (VCF, Thunderbird, projects) | +| WhatsApp bridge | `/mnt/8TB/projects/eslider/mcp-servers/whatsapp-mcp/whatsapp-bridge/` | Existing `whatsmeow` Go bridge | +| go-onlyoffice | `/mnt/8TB/projects/eslider/go-onlyoffice/` | OO CRM API library | +| oo CLI | `/home/ano/go/bin/oo` | Built go-onlyoffice CLI | + +--- + +## 5. Critical files + +| File | Content | +|------|---------| +| `~/.config/gitea/produktor.env` | Gitea URL + token | +| `/mnt/8TB/projects/eslider/mcp-servers/telegram.env` | Telegram API ID/Hash + phone + session | +| `/mnt/8TB/projects/eslider/mcp-servers/telegram-mcp/.env` | Telegram session string | +| `/mnt/8TB/projects/ai/2dph/.env` | ONLYOFFICE URL/USER/PASS | +| `/mnt/8TB/projects/eslider/cv/.env` | LinkedIn auth token | +| `~/.linkedin-mcp/storage-state.json` | LinkedIn active cookies | +| `/mnt/8TB/projects/eslider/go-onlyoffice/.env` | OO creds for oo CLI | +| `/mnt/8TB/projects/eslider/go-onlyoffice/go.mod` | Deps for go-onlyoffice | +| `/mnt/8TB/projects/eslider/oo-workspace/go.mod` | Deps for oo-workspace | + +--- + +## 6. Agent prompt (для запуска новой сессии) + +Скопируйте этот промпт при старте новой сессии: + +``` +Ты — ассистент для разработки chat import pipeline в проекте 2dph (eSlider/2dph). + +Прочитай docs/chat-import-plan.md полностью. Это план, находки и дизайн. + +Задача: реализовать bin/chats — единый Go бинарник для sync/import/index/facts/apply +личных чатов из Telegram, WhatsApp, LinkedIn с последующей записью фактов в OnlyOffice CRM. + +Правила: +1. TDD first — workflow use-case как тест. Реальные данные, без моков. +2. Go, всё в один бинарник bin/chats. +3. Source interface — каждая платформа плагином. +4. var/chats/telegram/ — полные имена платформ. +5. Начни с Telegram (session string активна). +6. После реализации — создай/обнови тесты. +7. Сохрани прогресс в docs/chat-import-plan.md. +8. Запроси разрешение перед apply в OO CRM. + +Ключевые пути: +- /mnt/8TB/projects/ai/2dph/ — корень проекта +- /mnt/8TB/projects/eslider/mcp-servers/telegram-mcp/ — Telegram MCP +- /mnt/8TB/projects/eslider/mcp-servers/telegram.env — API ID/Hash + session +- /mnt/8TB/projects/eslider/go-onlyoffice/ — go-onlyoffice библиотека +- /home/ano/go/bin/oo — OO CLI + +Готов начать. +``` +======= +--- + +## 1. Находки / Discoveries + +### Доступные платформы и данные + +| Platform | Status | Credentials (path, not values) | Go Library | +|----------|--------|-------------------------------|------------| +| **Telegram** | ✅ Active MCP server, session string exists | API creds in `telegram.env` (see §5). Session string in `telegram-mcp/.env`. | MCP JSON-RPC (через telegram-mcp server) | +| **WhatsApp** | ✅ MCP server exists + Go bridge. **No `messages.db` yet** (empty). Needs QR re-auth + sync first | MCP: `/mnt/8TB/projects/eslider/mcp-servers/whatsapp-mcp/`. Go bridge uses `tulir/whatsmeow` | [`tulir/whatsmeow`](https://github.com/tulir/whatsmeow) (7k★, Go, multi-device API) | +| **LinkedIn** | ✅ Connected via `mcp-server-linkedin`. Full browser profile + cookies | Browser profile: `~/.linkedin-mcp/profile/`. Active cookies: `~/.linkedin-mcp/storage-state.json`. Cookie backup: `~/.linkedin-mcp/cookies.json.bak-agent`. Script cookies: `/mnt/8TB/projects/eslider/cv/bin/creds/linkedin.cookies.json`. JobHunt session: `/mnt/8TB/projects/eslider/JobHunt/sessions/linkedin.json`. Env: `/mnt/8TB/projects/eslider/cv/.env` (`LINKEDIN_AUTH_TOKEN`). | [`swiftlysingh/lnk`](https://github.com/swiftlysingh/lnk) (Go CLI, Voyager API) | +| **Gmail** | ✅ Already working pipeline (`bin/mail/sync.go` + `bin/mail/import`) | OAuth at `~/.gmail-mcp/` | Already in 2dph | +| **Twitter/X.com** | ⚠️ Password only, no MCP, no session. Not needed now. | `/mnt/8TB/projects/eslider/scratches/Accounts/twitter-pass.txt` | [`benoitpetit/xsh`](https://github.com/benoitpetit/xsh) (Go CLI, cookie auth) | +| **Google Calendar** | ❌ Not connected yet. Gmail OAuth could extend | None | Нужен Google Calendar API | + +### Telegram session (проверен, активен) + +- API credentials in: `/mnt/8TB/projects/eslider/mcp-servers/telegram.env` + - `TELEGRAM_API_ID`, `TELEGRAM_API_HASH`, `TELEGRAM_PHONE` — загружаются из env + - `TELEGRAM_SESSION_STRING` — активная Telethon session (в env) +- MCP server launcher: `/mnt/8TB/projects/eslider/mcp-servers/run-telegram-mcp.sh` +- Session generator: `/mnt/8TB/projects/eslider/mcp-servers/telegram-mcp/session_string_generator.py` +- Regenerator: `/mnt/8TB/projects/eslider/mcp-servers/regen-telegram-session.py` + +### LinkedIn authentication (работает) + +- Запускается через Cursor MCP: `uvx mcp-server-linkedin@latest --user-data-dir ~/.linkedin-mcp/profile --no-auto-import` +- Cookie файлы: + - `~/.linkedin-mcp/storage-state.json` — Playwright storage state (активные cookies) + - `~/.linkedin-mcp/cookies.json.bak-agent` — резерв + - `~/.linkedin-mcp/cookies.json.bak-webtop-20260728T143535Z` — резерв + - `/mnt/8TB/projects/eslider/cv/bin/creds/linkedin.cookies.json` — для скриптов + - `/mnt/8TB/projects/eslider/JobHunt/sessions/linkedin.json` — JobHunt +- `li_at` token: в `storage-state.json` (см. §5) +- Для Go подхода: можно читать cookie из `storage-state.json` и использовать LinkedIn Voyager API напрямую + +### WhatsApp bridge (не синхронизирован) + +- Go bridge at: `/mnt/8TB/projects/eslider/mcp-servers/whatsapp-mcp/whatsapp-bridge/` +- Использует `tulir/whatsmeow` Go library (уже в `go.mod`) +- SQLite store: `whatsapp-bridge/store/messages.db` — **не существует** (0 байт), нужна первая авторизация по QR +- Launcher: `/mnt/8TB/projects/eslider/mcp-servers/run-whatsapp-bridge-lharries.sh` +- QR images готовились: `/mnt/8TB/projects/produktor/work/data/export/whatsapp-qr.png` +- Пароль: в `.env` OnlyOffice + +### OnlyOffice доступ + +- `.env` в корне 2dph: `ONLYOFFICE_URL`, `ONLYOFFICE_USER`, `ONLYOFFICE_PASS` +- `oo` CLI установлен: `/home/ano/go/bin/oo` (go-onlyoffice) +- go-onlyoffice repo: `/mnt/8TB/projects/eslider/go-onlyoffice/` (main, public) +- oo-workspace (каталог, private): `/mnt/8TB/projects/eslider/oo-workspace/` + +### Gitea + +- Instance: `https://git.produktor.io` (через `gitea-api` wrapper) +- Credentials: `~/.config/gitea/produktor.env` +- User: `eSlider`, admin +- Repos: 30+ приватных репозиториев +- **`eSlider/2dph` не существует на Gitea** — проект только на GitHub +- Для задачи: создать issue в существующем репо (например `eSlider/JobHunt`) или создать новый репо + +### Rambox + +Не используется. IndexedDB база не существует. + +--- + +## 2. Design Decisions + +### One binary: `bin/chats` + +```go +// Source interface — каждая платформа реализует +type Source interface { + Name() string // "telegram", "whatsapp", "linkedin" + Sync(ctx context.Context, out string) error +} + +// CLI subcommands +chats sync telegram [--limit N] [--since DATE] +chats sync whatsapp [--qr] [--limit N] +chats sync linkedin [--limit N] +chats import // JSONL → markdown (all sources) +chats index // rebuild var/kb.lbug +chats facts // extract + cross-check +chats apply // push to OO CRM [--dry-run] +``` + +### Directory layout (полные имена платформ) + +``` +var/chats/ + telegram//messages.jsonl + whatsapp//messages.jsonl + linkedin//messages.jsonl + md///message.md +``` + +### JSONL format (одна строка = одно сообщение) + +```json +{"id":"123","ts":"2026-01-15T10:30:00Z","from":"me","text":"Hello!","media":null,"platform":"telegram"} +``` + +### MD format (YAML frontmatter) + +```markdown +--- +id: tg_12345 +platform: telegram +chat_id: "-100123456" +chat_name: "John Doe" +participants: ["me", "John Doe"] +message_count: 1500 +type: personal +--- + +# Чат с John Doe + +**2026-01-15 10:30** — me: Hello! +``` + +### Фильтр "личный чат" + +- ≤3 участников +- Боты не считаются (Telegram: `user.is_bot == false`) +- Не канал, не публичная группа +- Telegram: `dialog.Type == "user"` или супергруппа с `participants_count <= 3` (исключая ботов) +- WhatsApp: `@s.whatsapp.net` (не `@g.us`) +- LinkedIn: 1:1 conversation + +### Очередность реализации + +1. **Telegram** — самый простой (session string + API ID/Hash активны) +2. **Import** — конвертер JSONL → MD +3. **Index** — интеграция с brain +4. **WhatsApp** — после первого QR +5. **Facts** — извлечение контактных данных +6. **LinkedIn** — после проверки cookie +7. **Apply** — запись в OO CRM + +### Тестирование (TDD, system tests only) + +- Никаких unit-тестов, никаких моков +- Тесты используют синтетические данные (слепки) +- Workflow use-case как тест: + 1. Написать тест с реальным сценарием (sync → import → index → search) + 2. Проверить что файлы созданы + 3. Проверить что поиск возвращает ожидаемые результаты +- A/B тест: сравнить два последовательных sync на идентичность +- Integration: тест через `oo` CLI с `--dry-run` + +### Cross-check (detective method) + +Перед записью в OO CRM — минимум 2 независимых источника: +- S1: Чат (текст сообщения, автор, дата) +- S2: OO CRM (существующий контакт/компания/deal) через `oo persons list` / `oo persons get` +- S3 (опционально): Corpus SoT (`knowledge-mesh-seed.yaml`) или web-search +- Факты с ≥2 источниками → `root=facts` в brain + `approve=true` для CRM +- Факты с 1 источником → предложение на review (catalog-паттерн) + +### Apply в OO CRM + +Через `oo` CLI: +- `oo persons update --about ... --job-title ...` +- `oo contacts info-add --type Phone|LinkedIn --value ...` +- `oo opportunities create --contact-id ... --title ... --stage ...` +- `oo persons update --company-id ...` (ассоциация) +- `oo projects contacts ` (связь с проектом) + +--- + +## 3. Implementation Plan + +### Phase 1: Telegram sync + import (MVP) + +1. Create `bin/chats/` directory structure (nested Go module like `bin/kbsearch/`) +2. Implement `Source` interface +3. Implement `TelegramSource` — MCP JSON-RPC клиент к telegram-mcp +4. Write JSONL output to `var/chats/telegram//messages.jsonl` +5. Implement `chats import` — reads all JSONL, writes MD +6. Write system test: sync → import → verify output + +### Phase 2: Brain indexing + +1. Implement `chats index` — rebuild `var/kb.lbug` with corpus + chats +2. Write system test: sync → import → index → `bin/kb/search "query"` → verify recall + +### Phase 3: WhatsApp + +1. Implement `WhatsAppSource` using `tulir/whatsmeow` +2. First run requires QR auth (save session for later reuse) +3. Write sync output to `var/chats/whatsapp//messages.jsonl` + +### Phase 4: Facts extraction + cross-check + +1. Implement `chats facts`: + - Phone regex: `\+?\d{7,15}` + validation (exclude dates/amounts/cards) + - LinkedIn URL regex: `linkedin\.com/in/[\w-]+` + - Cross-check: match extracted values against OO CRM API +2. Write facts to brain as `root=facts` + +### Phase 5: LinkedIn + Apply + +1. Implement `LinkedInSource` using Voyager API + cookies from `storage-state.json` +2. Implement `chats apply`: + - Build YAML catalog (as `oo-workspace` does) + - Run `oo catalog match` → review → `oo catalog apply` + +--- + +## 4. Existing code references + +| Code | Path | Purpose | +|------|------|---------| +| Mail sync | `bin/mail/sync.go` + `bin/mail/sync/` | Pattern for chat sync (Go subprocess pattern) | +| Mail import | `bin/mail/import` | Pattern for JSONL → MD conversion | +| Mail index | `bin/mail/index_mail` | Pattern for brain rebuild with new corpus | +| kbsearch | `bin/kbsearch/` | Go module pattern (nested, `system_ladybug` tag) | +| kb/index | `bin/kb/index` | Python brain index (upsert_leaf, init_schema) | +| kblib | `bin/tools/kblib.py` | Python brain core (connect, upsert_leaf, hybrid_search) | +| facts/crm | `bin/facts/crm` | Cross-check CRM facts pattern | +| facts/extract | `bin/facts/extract` | 2-source fact extraction pattern | +| oo-workspace catalog | `/mnt/8TB/projects/eslider/oo-workspace/catalog/` | Catalog scan/match/apply pattern (VCF, Thunderbird, projects) | +| WhatsApp bridge | `/mnt/8TB/projects/eslider/mcp-servers/whatsapp-mcp/whatsapp-bridge/` | Existing `whatsmeow` Go bridge | +| go-onlyoffice | `/mnt/8TB/projects/eslider/go-onlyoffice/` | OO CRM API library | +| oo CLI | `/home/ano/go/bin/oo` | Built go-onlyoffice CLI | + +--- + +## 5. Critical files (paths only, no secrets) + +| File | Content | +|------|---------| +| `~/.config/gitea/produktor.env` | Gitea URL + token | +| `/mnt/8TB/projects/eslider/mcp-servers/telegram.env` | Telegram API ID/Hash + phone + session | +| `/mnt/8TB/projects/eslider/mcp-servers/telegram-mcp/.env` | Telegram session string | +| `/mnt/8TB/projects/ai/2dph/.env` | ONLYOFFICE URL/USER/PASS | +| `/mnt/8TB/projects/eslider/cv/.env` | LinkedIn auth token | +| `~/.linkedin-mcp/storage-state.json` | LinkedIn active cookies | +| `/mnt/8TB/projects/eslider/go-onlyoffice/.env` | OO creds for oo CLI | +| `/mnt/8TB/projects/eslider/go-onlyoffice/go.mod` | Deps for go-onlyoffice | +| `/mnt/8TB/projects/eslider/oo-workspace/go.mod` | Deps for oo-workspace | + +--- + +## 6. Agent prompt (для запуска новой сессии) + +Скопируйте этот промпт при старте новой сессии: + +``` +Ты — ассистент для разработки chat import pipeline в проекте 2dph (eSlider/2dph). + +Прочитай docs/chat-import-plan.md полностью. Это план, находки и дизайн. + +Задача: реализовать bin/chats — единый Go бинарник для sync/import/index/facts/apply +личных чатов из Telegram, WhatsApp, LinkedIn с последующей записью фактов в OnlyOffice CRM. + +Правила: +1. TDD first — workflow use-case как тест. Синтетические данные (Alice, Bob). +2. Go, всё в один бинарник bin/chats. +3. Source interface — каждая платформа плагином. +4. var/chats/telegram/ — полные имена платформ. +5. Начни с Telegram (session string активна, через MCP server). +6. После реализации — создай/обнови тесты. +7. Сохрани прогресс в docs/chat-import-plan.md. +8. Запроси разрешение перед apply в OO CRM. +9. НИКАКИХ реальных имён, телефонов, email в коммитах — только ссылки на env файлы. + +Ключевые пути: +- /mnt/8TB/projects/ai/2dph/ — корень проекта +- /mnt/8TB/projects/eslider/mcp-servers/telegram-mcp/ — Telegram MCP +- /mnt/8TB/projects/eslider/mcp-servers/telegram.env — API ID/Hash + session (не коммитить!) +- /mnt/8TB/projects/eslider/go-onlyoffice/ — go-onlyoffice библиотека +- /home/ano/go/bin/oo — OO CLI + +Готов начать. +``` + +--- + +## 7. Implementation Progress + +### Session: 2026-08-12 — Phase 1 complete, sync проверен с реальными данными + +**Результаты тестового синка (--limit=100, 31 личный чат):** +- Подключение к Telegram MCP server: 1.5с +- Получен 31 личный чат (отфильтровано боты + Telegram service) +- Синк всех чатов: 7.7с (1 flood wait на 3с, обработан MCP сервером) +- Записано 922 сообщения в JSONL +- Import: 30 чатов в MD (1 пустой — Saved Messages) +- Факты: 14 phone + 2 email после фильтрации + +**Ключевое изменение: `gotd/td` → Telegram MCP Server** +- Вместо прямого gotd/td подключения (FLOOD_WAIT из-за 5 уже запущенных MCP серверов) +- Используется MCP JSON-RPC (JSONL, `\n`-delimited, **не** Content-Length headers) +- Переиспользует существующий Telethon session string (без phone auth) +- Стартует транзиентный MCP сервер на время синка, убивает после +- Инструменты: `list_chats` + `get_history` через MCP protocol v1.8+ + +**Файлы:** + +| Файл | Роль | +|------|------| +| `bin/chats/mcpclient.go` | MCP JSON-RPC client + `TelegramMCPSource` | +| `bin/chats/sync_cmd.go` | CLI: `chats sync telegram --limit N` | +| `bin/chats/import_cmd.go` | JSONL → MD с YAML frontmatter | +| `bin/chats/index_cmd.go` | Делегирует `bin/kb/index --corpus` | +| `bin/chats/facts_cmd.go` | Regex extraction (phone/email/linkedin) с валидацией | +| `bin/chats/apply_cmd.go` | OO CRM cross-check + apply через `oo` CLI + --dry-run | +| `bin/chat` | build+exec wrapper (как `bin/kb/search`) | + +**Phone regex улучшен:** +- Фильтр: исключает даты (`2026-06-14`), суммы (`2 500 000`), номера карт (16 digits), инвойсы (`25-002332001`) +- Валидация: >=7 digits, <=15 digits, без `000` pattern, не дата +- Результат: 37 → 16 фактов (14 phone + 2 email) + +**Cross-check + Apply в OO CRM (без персональных данных в файлах):** +- Найден существующий контакт через first name: добавлен phone + email +- Создан новый контакт: phone добавлен +- Apply через `oo contacts info-add --type Phone|Email --value ...` + +**Brain:** 132 leafs (120 info + 12 facts), чаты заиндексены + +**Безопасность:** +- `var/` в `.gitignore` — сырые чат-данные не коммитятся +- credentials в env файлах, не в коде +- Тесты используют синтетические данные (Alice, Bob, Charlie, Diana) +- Этот документ не содержит реальных значений API ключей, токенов, паролей или персональных данных третьих лиц + +**Что дальше (следующая сессия):** +1. WhatsApp source (`tulir/whatsmeow`) — после первого QR +2. LinkedIn source (Voyager API через cookies) +3. Full history sync (без --limit) для всех чатов +4. Telegram session reuse через MCP server (уже работает) +5. Apply через `bin/chats apply` (уже работает, улучшен поиск контактов) +>>>>>>> 313599b (bin/chats: Phase 1 MVP — Telegram sync/import/index/facts/apply)