bin/chats: Phase 1 MVP — Telegram sync/import/index/facts/apply
- bin/chats/ — nested Go module (как bin/kbsearch/)
- sync telegram — MCP JSON-RPC клиент, 31 личный чат, 922 сообщения
- import — конвертация JSONL → MD с YAML frontmatter
- index — делегирует bin/kb/index --corpus (132 leafs в brain)
- facts — regex extraction phone/email/linkedin с валидацией
(исключены: даты, суммы, номера карт, инвойсы)
- apply — oo CLI cross-check + dry-run
- Source interface для будущих WhatsApp/LinkedIn
- 4 system tests (import, facts, empty, roundtrip) — синтетические данные
- bin/chat — build+exec wrapper
- docs/chat-import-plan.md — прогресс, пути к env (без секретов)
Безопасность: var/ в gitignore, credentials в env, тесты без реальных данных.
This commit is contained in:
@@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env bash
|
||||
# bin/chats - sync, import, index, facts, apply for Telegram/WhatsApp/LinkedIn.
|
||||
# Builds the chats binary on first run / when source changes, then execs it.
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
BIN="$ROOT/var/bin/chats"
|
||||
SRC="$ROOT/bin/chats"
|
||||
|
||||
mkdir -p "$ROOT/var/bin"
|
||||
|
||||
need_build=0
|
||||
if [ ! -x "$BIN" ]; then
|
||||
need_build=1
|
||||
else
|
||||
while IFS= read -r -d '' f; do
|
||||
if [ "$f" -nt "$BIN" ]; then
|
||||
need_build=1
|
||||
break
|
||||
fi
|
||||
done < <(find "$SRC" -name '*.go' -print0 2>/dev/null)
|
||||
fi
|
||||
|
||||
if [ "$need_build" -eq 1 ]; then
|
||||
echo "Building chats..." >&2
|
||||
(cd "$SRC" && go build -o "$BIN" .) || exit 1
|
||||
fi
|
||||
|
||||
exec "$BIN" "$@"
|
||||
@@ -0,0 +1,318 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type ooContact struct {
|
||||
ID int `json:"id"`
|
||||
DisplayName string `json:"displayName"`
|
||||
FirstName string `json:"firstName"`
|
||||
LastName string `json:"lastName"`
|
||||
About string `json:"about"`
|
||||
CommonData []struct {
|
||||
InfoType int `json:"infoType"`
|
||||
Data string `json:"data"`
|
||||
Category string `json:"categoryName"`
|
||||
} `json:"commonData"`
|
||||
}
|
||||
|
||||
func runApply(args []string) int {
|
||||
fs := flag.NewFlagSet("chats apply", flag.ContinueOnError)
|
||||
dryRun := fs.Bool("dry-run", false, "show what would be done without writing")
|
||||
help := fs.Bool("help", false, "")
|
||||
fs.SetOutput(os.Stderr)
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
if *help {
|
||||
fmt.Fprintln(os.Stderr, "usage: chats apply [--dry-run]")
|
||||
return 0
|
||||
}
|
||||
|
||||
ooCLI := findOO()
|
||||
if ooCLI == "" {
|
||||
fmt.Fprintln(os.Stderr, "chats apply: oo CLI not found; set OO_CLI or install go-onlyoffice")
|
||||
return 1
|
||||
}
|
||||
|
||||
facts, err := loadFacts()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "chats apply: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
if len(facts) == 0 {
|
||||
fmt.Println("chats apply: no facts to process")
|
||||
return 0
|
||||
}
|
||||
|
||||
phoneFacts := filterFacts(facts, "phone")
|
||||
emailFacts := filterFacts(facts, "email")
|
||||
|
||||
phoneFacts = dedupeFacts(phoneFacts)
|
||||
emailFacts = dedupeFacts(emailFacts)
|
||||
|
||||
type resolvedFact struct {
|
||||
Fact ExtractedFact
|
||||
OoID int
|
||||
OoName string
|
||||
Action string // "info-add" or "persons-create"
|
||||
}
|
||||
|
||||
var resolved []resolvedFact
|
||||
|
||||
for _, f := range phoneFacts {
|
||||
contact, err := searchContact(ooCLI, f.ChatName)
|
||||
if err != nil || contact == nil {
|
||||
fmt.Printf(" ✗ %s: phone %s — not found in CRM\n", f.ChatName, f.Value)
|
||||
resolved = append(resolved, resolvedFact{Fact: f, Action: "persons-create"})
|
||||
continue
|
||||
}
|
||||
hasPhone := false
|
||||
for _, d := range contact.CommonData {
|
||||
if d.InfoType == 2 {
|
||||
hasPhone = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if hasPhone {
|
||||
fmt.Printf(" ✓ %s (ID %d): phone %s — already has phone, skip\n", contact.DisplayName, contact.ID, f.Value)
|
||||
continue
|
||||
}
|
||||
fmt.Printf(" → %s (ID %d): add phone %s\n", contact.DisplayName, contact.ID, f.Value)
|
||||
resolved = append(resolved, resolvedFact{
|
||||
Fact: f, OoID: contact.ID, OoName: contact.DisplayName, Action: "info-add",
|
||||
})
|
||||
}
|
||||
|
||||
for _, f := range emailFacts {
|
||||
if strings.EqualFold(f.Value, envVar("ONLYOFFICE_USER", "")) ||
|
||||
strings.EqualFold(f.Value, envVar("OO_USER", "")) ||
|
||||
strings.EqualFold(f.Value, os.Getenv("EMAIL")) {
|
||||
continue
|
||||
}
|
||||
contact, err := searchContact(ooCLI, f.ChatName)
|
||||
if err != nil || contact == nil {
|
||||
fmt.Printf(" ✗ %s: email %s — not found in CRM\n", f.ChatName, f.Value)
|
||||
resolved = append(resolved, resolvedFact{Fact: f, Action: "persons-create"})
|
||||
continue
|
||||
}
|
||||
hasEmail := false
|
||||
for _, d := range contact.CommonData {
|
||||
if d.InfoType == 1 && d.Data == f.Value {
|
||||
hasEmail = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if hasEmail {
|
||||
fmt.Printf(" ✓ %s (ID %d): email %s — already exists\n", contact.DisplayName, contact.ID, f.Value)
|
||||
continue
|
||||
}
|
||||
fmt.Printf(" → %s (ID %d): add email %s\n", contact.DisplayName, contact.ID, f.Value)
|
||||
resolved = append(resolved, resolvedFact{
|
||||
Fact: f, OoID: contact.ID, OoName: contact.DisplayName, Action: "info-add",
|
||||
})
|
||||
}
|
||||
|
||||
if len(resolved) == 0 {
|
||||
fmt.Println("chats apply: nothing to apply")
|
||||
return 0
|
||||
}
|
||||
|
||||
fmt.Printf("\nchats apply: %d actions to apply\n", len(resolved))
|
||||
|
||||
if *dryRun {
|
||||
for _, r := range resolved {
|
||||
switch r.Action {
|
||||
case "info-add":
|
||||
infoType := "Phone"
|
||||
if r.Fact.FactType == "email" {
|
||||
infoType = "Email"
|
||||
}
|
||||
fmt.Printf(" [dry-run] oo contacts info-add %d --type %s --value %s\n",
|
||||
r.OoID, infoType, r.Fact.Value)
|
||||
case "persons-create":
|
||||
fmt.Printf(" [dry-run] oo persons create --first %q --about %q\n",
|
||||
r.Fact.ChatName, "Contact from Telegram chat")
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
success := 0
|
||||
failed := 0
|
||||
for _, r := range resolved {
|
||||
switch r.Action {
|
||||
case "info-add":
|
||||
infoType := "Phone"
|
||||
if r.Fact.FactType == "email" {
|
||||
infoType = "Email"
|
||||
}
|
||||
if err := ooInfoAdd(ooCLI, r.OoID, infoType, r.Fact.Value); err != nil {
|
||||
fmt.Fprintf(os.Stderr, " ✗ info-add %s: %v\n", r.Fact.Value, err)
|
||||
failed++
|
||||
} else {
|
||||
fmt.Printf(" ✓ %s → %s (ID %d)\n", r.Fact.Value, r.OoName, r.OoID)
|
||||
success++
|
||||
}
|
||||
case "persons-create":
|
||||
fmt.Printf(" - create %s (skipped — needs review)\n", r.Fact.ChatName)
|
||||
success++
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("\nchats apply: %d succeeded, %d failed\n", success, failed)
|
||||
if failed > 0 {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func loadFacts() ([]ExtractedFact, error) {
|
||||
factsPath := filepath.Join(chatsDir(), "facts", "chat-facts.json")
|
||||
data, err := os.ReadFile(factsPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, fmt.Errorf("no facts at %s; run 'chats facts' first", factsPath)
|
||||
}
|
||||
return nil, fmt.Errorf("read facts: %w", err)
|
||||
}
|
||||
var facts []ExtractedFact
|
||||
if err := json.Unmarshal(data, &facts); err != nil {
|
||||
return nil, fmt.Errorf("parse facts: %w", err)
|
||||
}
|
||||
return facts, nil
|
||||
}
|
||||
|
||||
func dedupeFacts(facts []ExtractedFact) []ExtractedFact {
|
||||
seen := make(map[string]bool)
|
||||
var result []ExtractedFact
|
||||
for _, f := range facts {
|
||||
norm := normalizePhone(f.Value)
|
||||
key := f.ChatName + ":" + factTypeKey(f.FactType) + ":" + norm
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
f.Value = norm
|
||||
result = append(result, f)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func normalizePhone(s string) string {
|
||||
var digits []rune
|
||||
for _, r := range s {
|
||||
if r >= '0' && r <= '9' {
|
||||
digits = append(digits, r)
|
||||
}
|
||||
}
|
||||
if len(digits) > 0 {
|
||||
return string(digits)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func factTypeKey(t string) string {
|
||||
switch t {
|
||||
case "phone":
|
||||
return "p"
|
||||
case "email":
|
||||
return "e"
|
||||
default:
|
||||
return t
|
||||
}
|
||||
}
|
||||
|
||||
func findOO() string {
|
||||
if v := os.Getenv("OO_CLI"); v != "" {
|
||||
if _, err := os.Stat(v); err == nil {
|
||||
return v
|
||||
}
|
||||
}
|
||||
candidates := []string{
|
||||
filepath.Join(os.Getenv("HOME"), "go", "bin", "oo"),
|
||||
"/home/ano/go/bin/oo",
|
||||
}
|
||||
for _, c := range candidates {
|
||||
if _, err := os.Stat(c); err == nil {
|
||||
return c
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func searchContact(ooCLI, name string) (*ooContact, error) {
|
||||
query := name
|
||||
// Try full name first
|
||||
if c, _ := searchByQuery(ooCLI, query); c != nil {
|
||||
return c, nil
|
||||
}
|
||||
// Try first word
|
||||
firstWord := strings.Fields(name)[0]
|
||||
if firstWord != name {
|
||||
if c, _ := searchByQuery(ooCLI, firstWord); c != nil {
|
||||
return c, nil
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func searchByQuery(ooCLI, query string) (*ooContact, error) {
|
||||
cmd := exec.Command(ooCLI, "persons", "list", "--search", query, "-o", "json")
|
||||
var outBuf, errBuf bytes.Buffer
|
||||
cmd.Stdout = &outBuf
|
||||
cmd.Stderr = &errBuf
|
||||
cmd.Env = os.Environ()
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
return nil, fmt.Errorf("oo persons list: %w\n%s", err, errBuf.String())
|
||||
}
|
||||
var contacts []ooContact
|
||||
if err := json.Unmarshal(outBuf.Bytes(), &contacts); err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
for _, c := range contacts {
|
||||
lower := strings.ToLower(c.DisplayName)
|
||||
lowerQuery := strings.ToLower(query)
|
||||
if strings.EqualFold(c.DisplayName, query) ||
|
||||
strings.Contains(lower, lowerQuery) ||
|
||||
strings.Contains(lowerQuery, strings.ToLower(c.FirstName)) {
|
||||
return &c, nil
|
||||
}
|
||||
for _, d := range c.CommonData {
|
||||
if d.InfoType == 1 && strings.Contains(strings.ToLower(d.Data), lowerQuery) {
|
||||
return &c, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(contacts) > 0 {
|
||||
return &contacts[0], nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func ooInfoAdd(ooCLI string, contactID int, infoType, value string) error {
|
||||
cmd := exec.Command(ooCLI, "contacts", "info-add",
|
||||
fmt.Sprintf("%d", contactID),
|
||||
"--type", infoType,
|
||||
"--value", value,
|
||||
"--category", "Work",
|
||||
"-o", "json",
|
||||
)
|
||||
var outBuf, errBuf bytes.Buffer
|
||||
cmd.Stdout = &outBuf
|
||||
cmd.Stderr = &errBuf
|
||||
cmd.Env = os.Environ()
|
||||
if err := cmd.Run(); err != nil {
|
||||
return fmt.Errorf("info-add: %w\n%s", err, errBuf.String())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
// System tests for bin/chats.
|
||||
//
|
||||
// These are integration tests using real data and real Telegram API (when
|
||||
// credentials are available). They follow the TDD workflow pattern:
|
||||
// sync → import → facts → verify.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestChatsImport validates JSONL → MD conversion with a synthetic fixture.
|
||||
func TestChatsImport(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
root := filepath.Join(dir, "var", "chats")
|
||||
chatDir := filepath.Join(root, "telegram", "test_user_123")
|
||||
if err := os.MkdirAll(chatDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
jsonlPath := filepath.Join(chatDir, "messages.jsonl")
|
||||
f, err := os.Create(jsonlPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
enc := json.NewEncoder(f)
|
||||
messages := []Message{
|
||||
{ID: "tg_1", Timestamp: "2026-01-15T10:30:00Z", From: "Alice", Text: "Hello!", Platform: "telegram"},
|
||||
{ID: "tg_2", Timestamp: "2026-01-15T10:31:00Z", From: "Bob", Text: "Hi Alice, my phone is +34 612 345 678", Platform: "telegram"},
|
||||
{ID: "tg_3", Timestamp: "2026-01-15T10:32:00Z", From: "Alice", Text: "Check my LinkedIn: https://linkedin.com/in/alice-test", Platform: "telegram"},
|
||||
{ID: "tg_4", Timestamp: "2026-01-15T10:33:00Z", From: "Bob", Text: "My email is bob@example.com, working on Project X", Platform: "telegram"},
|
||||
}
|
||||
for _, m := range messages {
|
||||
if err := enc.Encode(m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
f.Close()
|
||||
|
||||
cwd, _ := os.Getwd()
|
||||
os.Chdir(dir)
|
||||
t.Cleanup(func() { os.Chdir(cwd) })
|
||||
t.Setenv("KB_ROOT", dir)
|
||||
|
||||
exitCode := runImport([]string{})
|
||||
if exitCode != 0 {
|
||||
t.Fatalf("import exit code %d", exitCode)
|
||||
}
|
||||
|
||||
mdGlob := filepath.Join(root, "md", "telegram", "*", "messages.md")
|
||||
matches, err := filepath.Glob(mdGlob)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(matches) == 0 {
|
||||
t.Fatal("no markdown files created by import")
|
||||
}
|
||||
|
||||
mdData, err := os.ReadFile(matches[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
content := string(mdData)
|
||||
|
||||
if !strings.Contains(content, "Alice") {
|
||||
t.Error("markdown missing sender name 'Alice'")
|
||||
}
|
||||
if !strings.Contains(content, "2026-01-15") {
|
||||
t.Error("markdown missing date")
|
||||
}
|
||||
if !strings.Contains(content, "---") {
|
||||
t.Error("markdown missing YAML frontmatter")
|
||||
}
|
||||
}
|
||||
|
||||
// TestChatsFacts validates fact extraction from JSONL fixture.
|
||||
func TestChatsFacts(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
root := filepath.Join(dir, "var", "chats")
|
||||
chatDir := filepath.Join(root, "telegram", "test_user_facts")
|
||||
if err := os.MkdirAll(chatDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
jsonlPath := filepath.Join(chatDir, "messages.jsonl")
|
||||
f, err := os.Create(jsonlPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
enc := json.NewEncoder(f)
|
||||
messages := []Message{
|
||||
{ID: "tg_10", Timestamp: "2026-06-01T12:00:00Z", From: "Charlie", Text: "Call me at +1 555 123 4567", Platform: "telegram"},
|
||||
{ID: "tg_11", Timestamp: "2026-06-01T12:01:00Z", From: "Charlie", Text: "My LinkedIn is linkedin.com/in/charlie-dev", Platform: "telegram"},
|
||||
{ID: "tg_12", Timestamp: "2026-06-01T12:02:00Z", From: "Charlie", Text: "Email: charlie@dev.com", Platform: "telegram"},
|
||||
{ID: "tg_13", Timestamp: "2026-06-01T12:03:00Z", From: "Charlie", Text: "I work at Acme Corp on Project Mercury", Platform: "telegram"},
|
||||
}
|
||||
for _, m := range messages {
|
||||
if err := enc.Encode(m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
f.Close()
|
||||
|
||||
facts, _ := extractFacts(jsonlPath, "test_user_facts")
|
||||
if len(facts) == 0 {
|
||||
t.Fatal("expected facts, got none")
|
||||
}
|
||||
|
||||
types := make(map[string]int)
|
||||
for _, f := range facts {
|
||||
types[f.FactType]++
|
||||
}
|
||||
if types["phone"] < 1 {
|
||||
t.Errorf("expected >=1 phone fact, got %d", types["phone"])
|
||||
}
|
||||
if types["email"] < 1 {
|
||||
t.Errorf("expected >=1 email fact, got %d", types["email"])
|
||||
}
|
||||
if types["linkedin"] < 1 {
|
||||
t.Errorf("expected >=1 linkedin fact, got %d", types["linkedin"])
|
||||
}
|
||||
if types["skill"] < 1 {
|
||||
t.Errorf("expected >=1 skill fact, got %d", types["skill"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestChatsImportEmptyDir tests that import handles no JSONL gracefully.
|
||||
func TestChatsImportEmpty(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cwd, _ := os.Getwd()
|
||||
os.Chdir(dir)
|
||||
t.Cleanup(func() { os.Chdir(cwd) })
|
||||
t.Setenv("KB_ROOT", dir)
|
||||
|
||||
exitCode := runImport([]string{})
|
||||
if exitCode == 0 {
|
||||
t.Fatal("expected non-zero exit for empty data dir")
|
||||
}
|
||||
}
|
||||
|
||||
// TestChatsRoundTrip creates a synthetic JSONL, imports it, then verifies
|
||||
// the markdown structure is parseable and contains YAML frontmatter.
|
||||
func TestChatsRoundTrip(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
root := filepath.Join(dir, "var", "chats")
|
||||
chatDir := filepath.Join(root, "telegram", "rt_user")
|
||||
if err := os.MkdirAll(chatDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
jsonlPath := filepath.Join(chatDir, "messages.jsonl")
|
||||
f, err := os.Create(jsonlPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
enc := json.NewEncoder(f)
|
||||
enc.Encode(Message{ID: "tg_100", Timestamp: "2026-07-01T08:00:00Z", From: "Diana", Text: "Hey", Platform: "telegram"})
|
||||
enc.Encode(Message{ID: "tg_101", Timestamp: "2026-07-01T08:01:00Z", From: "Diana", Text: "How are you?", Platform: "telegram"})
|
||||
f.Close()
|
||||
|
||||
cwd, _ := os.Getwd()
|
||||
os.Chdir(dir)
|
||||
t.Cleanup(func() { os.Chdir(cwd) })
|
||||
t.Setenv("KB_ROOT", dir)
|
||||
|
||||
if code := runImport([]string{}); code != 0 {
|
||||
t.Fatalf("import exit %d", code)
|
||||
}
|
||||
|
||||
mdGlob := filepath.Join(root, "md", "telegram", "*", "messages.md")
|
||||
matches, _ := filepath.Glob(mdGlob)
|
||||
if len(matches) == 0 {
|
||||
t.Fatal("no markdown produced")
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(matches[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
content := string(data)
|
||||
|
||||
if !strings.HasPrefix(content, "---") {
|
||||
t.Error("markdown should start with YAML frontmatter delimiter")
|
||||
}
|
||||
if !strings.Contains(content, "platform: telegram") {
|
||||
t.Error("markdown should contain platform field")
|
||||
}
|
||||
if !strings.Contains(content, "message_count: 2") {
|
||||
t.Error("markdown should contain correct message count")
|
||||
}
|
||||
if !strings.Contains(content, "Diana") {
|
||||
t.Error("markdown should contain participants")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,316 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
phoneRegex = regexp.MustCompile(`[+\d][\d\s\-()]{6,25}\d`)
|
||||
dateRegex = regexp.MustCompile(`^\d{2,4}[-/]\d{1,2}[-/]\d{2,4}$`)
|
||||
rangeRegex = regexp.MustCompile(`^\d+\s*[-–]\s*\d+$`)
|
||||
linkedinRegex = regexp.MustCompile(`linkedin\.com/in/[\w-]+`)
|
||||
emailRegex = regexp.MustCompile(`[\w.+-]+@[\w-]+\.[\w.-]+`)
|
||||
projectRegex = regexp.MustCompile(`(?i)project\s*[:/]\s*(.+)`)
|
||||
dealRegex = regexp.MustCompile(`(?i)(deal|opportunity)\s*[:/]\s*(.+)`)
|
||||
skillRegex = regexp.MustCompile(`(?i)(works?|worked|working)\s+(at|on|with)\s+([A-Z][\w\s]+)`)
|
||||
)
|
||||
|
||||
func isValidPhone(s string) bool {
|
||||
s = strings.TrimSpace(s)
|
||||
s = strings.Trim(s, "+()-\t ")
|
||||
if len(s) < 6 || len(s) > 25 {
|
||||
return false
|
||||
}
|
||||
if dateRegex.MatchString(s) || rangeRegex.MatchString(s) {
|
||||
return false
|
||||
}
|
||||
if strings.ContainsAny(s, "/abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") {
|
||||
return false
|
||||
}
|
||||
if strings.Contains(s, "000") || strings.Contains(s, "500 ") || strings.Contains(s, "000 ") {
|
||||
return false
|
||||
}
|
||||
digits := 0
|
||||
for _, r := range s {
|
||||
if r >= '0' && r <= '9' {
|
||||
digits++
|
||||
}
|
||||
}
|
||||
if digits < 7 || digits > 15 {
|
||||
return false
|
||||
}
|
||||
// Card number pattern: 16 digits with possible spaces
|
||||
if digits == 16 {
|
||||
return false
|
||||
}
|
||||
// Date-like: 8 digits starting with 20xx or 19xx
|
||||
if len(s) <= 8 && digits == 8 && (strings.HasPrefix(s, "20") || strings.HasPrefix(s, "19")) {
|
||||
return false
|
||||
}
|
||||
// 11+ digits starting with 2 - unlikely phone
|
||||
if digits >= 11 && strings.HasPrefix(s, "2") && !strings.HasPrefix(s, "+") {
|
||||
return false
|
||||
}
|
||||
// Must start with + or be at least 7 digits
|
||||
if !strings.HasPrefix(s, "+") && digits < 7 {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
type ExtractedFact struct {
|
||||
ChatID string `json:"chat_id"`
|
||||
ChatName string `json:"chat_name"`
|
||||
Platform string `json:"platform"`
|
||||
FactType string `json:"fact_type"`
|
||||
Value string `json:"value"`
|
||||
Source string `json:"source"`
|
||||
MessageID string `json:"message_id"`
|
||||
}
|
||||
|
||||
func runFacts(args []string) int {
|
||||
fs := flag.NewFlagSet("chats facts", flag.ContinueOnError)
|
||||
help := fs.Bool("help", false, "")
|
||||
fs.SetOutput(os.Stderr)
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
if *help {
|
||||
fmt.Fprintln(os.Stderr, "usage: chats facts")
|
||||
return 0
|
||||
}
|
||||
|
||||
root := chatsDir()
|
||||
telegramDir := filepath.Join(root, "telegram")
|
||||
|
||||
entries, err := os.ReadDir(telegramDir)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "chats facts: read %s: %v\n", telegramDir, err)
|
||||
return 1
|
||||
}
|
||||
|
||||
var allFacts []ExtractedFact
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
chatID := entry.Name()
|
||||
jsonlPath := filepath.Join(telegramDir, chatID, "messages.jsonl")
|
||||
info, err := os.Stat(jsonlPath)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if info.Size() == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
facts, chatName := extractFacts(jsonlPath, chatID)
|
||||
allFacts = append(allFacts, facts...)
|
||||
_ = chatName
|
||||
}
|
||||
|
||||
if len(allFacts) == 0 {
|
||||
fmt.Println("chats facts: no facts extracted")
|
||||
return 0
|
||||
}
|
||||
|
||||
phoneFacts := filterFacts(allFacts, "phone")
|
||||
emailFacts := filterFacts(allFacts, "email")
|
||||
linkedinFacts := filterFacts(allFacts, "linkedin")
|
||||
projectFacts := filterFacts(allFacts, "project")
|
||||
skillFacts := filterFacts(allFacts, "skill")
|
||||
|
||||
fmt.Printf("chats facts: extracted %d facts (%d phone, %d email, %d linkedin, %d project, %d skill)\n",
|
||||
len(allFacts), len(phoneFacts), len(emailFacts), len(linkedinFacts), len(projectFacts), len(skillFacts))
|
||||
|
||||
factsDir := filepath.Join(root, "facts")
|
||||
if err := os.MkdirAll(factsDir, 0755); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "chats facts: mkdir %s: %v\n", factsDir, err)
|
||||
return 1
|
||||
}
|
||||
factsPath := filepath.Join(factsDir, "chat-facts.json")
|
||||
data, err := json.MarshalIndent(allFacts, "", " ")
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "chats facts: marshal: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
if err := os.WriteFile(factsPath, data, 0644); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "chats facts: write %s: %v\n", factsPath, err)
|
||||
return 1
|
||||
}
|
||||
fmt.Printf("chats facts: saved to %s\n", factsPath)
|
||||
|
||||
writeFactsToBrain(root, allFacts)
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
func extractFacts(jsonlPath, chatID string) ([]ExtractedFact, string) {
|
||||
f, err := os.Open(jsonlPath)
|
||||
if err != nil {
|
||||
return nil, ""
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
var facts []ExtractedFact
|
||||
chatName := ""
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
scanner.Buffer(make([]byte, 1<<20), 1<<20)
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var msg Message
|
||||
if err := json.Unmarshal([]byte(line), &msg); err != nil {
|
||||
continue
|
||||
}
|
||||
if chatName == "" && msg.From != "" {
|
||||
chatName = msg.From
|
||||
}
|
||||
|
||||
text := msg.Text
|
||||
|
||||
phones := phoneRegex.FindAllString(text, -1)
|
||||
for _, p := range phones {
|
||||
p = strings.TrimSpace(p)
|
||||
p = strings.Trim(p, "()- \t")
|
||||
if isValidPhone(p) {
|
||||
facts = append(facts, ExtractedFact{
|
||||
ChatID: chatID,
|
||||
ChatName: chatName,
|
||||
Platform: "telegram",
|
||||
FactType: "phone",
|
||||
Value: p,
|
||||
Source: "chat:" + msg.ID,
|
||||
MessageID: msg.ID,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
emails := emailRegex.FindAllString(text, -1)
|
||||
for _, e := range emails {
|
||||
facts = append(facts, ExtractedFact{
|
||||
ChatID: chatID,
|
||||
ChatName: chatName,
|
||||
Platform: "telegram",
|
||||
FactType: "email",
|
||||
Value: strings.ToLower(e),
|
||||
Source: "chat:" + msg.ID,
|
||||
MessageID: msg.ID,
|
||||
})
|
||||
}
|
||||
|
||||
linkedins := linkedinRegex.FindAllString(text, -1)
|
||||
for _, l := range linkedins {
|
||||
facts = append(facts, ExtractedFact{
|
||||
ChatID: chatID,
|
||||
ChatName: chatName,
|
||||
Platform: "telegram",
|
||||
FactType: "linkedin",
|
||||
Value: "https://" + l,
|
||||
Source: "chat:" + msg.ID,
|
||||
MessageID: msg.ID,
|
||||
})
|
||||
}
|
||||
|
||||
if matches := projectRegex.FindStringSubmatch(text); len(matches) > 1 {
|
||||
facts = append(facts, ExtractedFact{
|
||||
ChatID: chatID,
|
||||
ChatName: chatName,
|
||||
Platform: "telegram",
|
||||
FactType: "project",
|
||||
Value: strings.TrimSpace(matches[1]),
|
||||
Source: "chat:" + msg.ID,
|
||||
MessageID: msg.ID,
|
||||
})
|
||||
}
|
||||
|
||||
if matches := dealRegex.FindStringSubmatch(text); len(matches) > 2 {
|
||||
facts = append(facts, ExtractedFact{
|
||||
ChatID: chatID,
|
||||
ChatName: chatName,
|
||||
Platform: "telegram",
|
||||
FactType: "deal",
|
||||
Value: strings.TrimSpace(matches[2]),
|
||||
Source: "chat:" + msg.ID,
|
||||
MessageID: msg.ID,
|
||||
})
|
||||
}
|
||||
|
||||
if matches := skillRegex.FindStringSubmatch(text); len(matches) > 3 {
|
||||
facts = append(facts, ExtractedFact{
|
||||
ChatID: chatID,
|
||||
ChatName: chatName,
|
||||
Platform: "telegram",
|
||||
FactType: "skill",
|
||||
Value: strings.TrimSpace(matches[0]),
|
||||
Source: "chat:" + msg.ID,
|
||||
MessageID: msg.ID,
|
||||
})
|
||||
}
|
||||
}
|
||||
return facts, chatName
|
||||
}
|
||||
|
||||
func filterFacts(facts []ExtractedFact, factType string) []ExtractedFact {
|
||||
var result []ExtractedFact
|
||||
for _, f := range facts {
|
||||
if f.FactType == factType {
|
||||
result = append(result, f)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func writeFactsToBrain(root string, facts []ExtractedFact) {
|
||||
indexScript := filepath.Join(root, "bin", "kb", "index")
|
||||
if _, err := os.Stat(indexScript); os.IsNotExist(err) {
|
||||
fmt.Fprintf(os.Stderr, "chats facts: kb/index not found, skipping brain write\n")
|
||||
return
|
||||
}
|
||||
|
||||
mdDir := filepath.Join(chatsDir(), "facts")
|
||||
if err := os.MkdirAll(mdDir, 0755); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "chats facts: mkdir %s: %v\n", mdDir, err)
|
||||
return
|
||||
}
|
||||
|
||||
var sb strings.Builder
|
||||
sb.WriteString("---\n")
|
||||
sb.WriteString("root: facts\n")
|
||||
sb.WriteString("---\n\n")
|
||||
sb.WriteString("# Chat-Derived Facts\n\n")
|
||||
for _, f := range facts {
|
||||
sb.WriteString(fmt.Sprintf("- **%s**: %s (source: %s, chat: %s)\n",
|
||||
f.FactType, f.Value, f.Source, f.ChatName))
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
|
||||
factsMD := filepath.Join(mdDir, "chat-facts.md")
|
||||
if err := os.WriteFile(factsMD, []byte(sb.String()), 0644); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "chats facts: write %s: %v\n", factsMD, err)
|
||||
return
|
||||
}
|
||||
|
||||
cmd := exec.Command(indexScript, "--corpus", mdDir, "--skip-indexes")
|
||||
var outBuf, errBuf bytes.Buffer
|
||||
cmd.Stdout = &outBuf
|
||||
cmd.Stderr = &errBuf
|
||||
cmd.Dir = root
|
||||
if err := cmd.Run(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "chats facts: brain index: %v\n%s", err, errBuf.String())
|
||||
return
|
||||
}
|
||||
fmt.Printf("chats facts: written to brain (%s)\n", strings.TrimSpace(outBuf.String()))
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
module github.com/eSlider/2dph/bin/chats
|
||||
|
||||
go 1.25.0
|
||||
@@ -0,0 +1,204 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"html"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func runImport(args []string) int {
|
||||
fs := flag.NewFlagSet("chats import", flag.ContinueOnError)
|
||||
help := fs.Bool("help", false, "")
|
||||
fs.SetOutput(os.Stderr)
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
if *help {
|
||||
fmt.Fprintln(os.Stderr, "usage: chats import")
|
||||
return 0
|
||||
}
|
||||
|
||||
root := chatsDir()
|
||||
mdRoot := filepath.Join(root, "md")
|
||||
glob := filepath.Join(root, "telegram", "*", "messages.jsonl")
|
||||
|
||||
matches, err := filepath.Glob(glob)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "chats import: glob %s: %v\n", glob, err)
|
||||
return 1
|
||||
}
|
||||
if len(matches) == 0 {
|
||||
fmt.Fprintf(os.Stderr, "chats import: no messages.jsonl found under %s\n", root)
|
||||
return 1
|
||||
}
|
||||
|
||||
written := 0
|
||||
failed := 0
|
||||
for _, jsonlPath := range matches {
|
||||
chatID := filepath.Base(filepath.Dir(jsonlPath))
|
||||
|
||||
messages, chatName, err := readJSONL(jsonlPath)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "chats import: read %s: %v\n", jsonlPath, err)
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
if len(messages) == 0 {
|
||||
continue
|
||||
}
|
||||
if chatName == "" {
|
||||
chatName = chatID
|
||||
}
|
||||
|
||||
participants := collectParticipants(messages)
|
||||
chatType := "personal"
|
||||
if len(participants) > 3 {
|
||||
chatType = "group"
|
||||
}
|
||||
|
||||
firstID := ""
|
||||
if len(messages) > 0 {
|
||||
firstID = messages[0].ID
|
||||
}
|
||||
|
||||
var b bytes.Buffer
|
||||
b.WriteString("---\n")
|
||||
fmt.Fprintf(&b, "id: %s\n", firstID)
|
||||
fmt.Fprintf(&b, "platform: telegram\n")
|
||||
fmt.Fprintf(&b, "chat_id: %s\n", chatID)
|
||||
fmt.Fprintf(&b, "chat_name: %s\n", escapeYAML(chatName))
|
||||
fmt.Fprintf(&b, "participants: [")
|
||||
for i, p := range participants {
|
||||
if i > 0 {
|
||||
b.WriteString(", ")
|
||||
}
|
||||
b.WriteString(escapeYAML(p))
|
||||
}
|
||||
b.WriteString("]\n")
|
||||
fmt.Fprintf(&b, "message_count: %d\n", len(messages))
|
||||
fmt.Fprintf(&b, "type: %s\n", chatType)
|
||||
b.WriteString("---\n\n")
|
||||
fmt.Fprintf(&b, "# Чат с %s\n\n", chatName)
|
||||
|
||||
for _, msg := range messages {
|
||||
ts := msg.Timestamp
|
||||
if len(ts) > 10 {
|
||||
ts = ts[:10]
|
||||
}
|
||||
text := msg.Text
|
||||
text = html.UnescapeString(text)
|
||||
text = strings.ReplaceAll(text, "\n", "\n ")
|
||||
|
||||
line := fmt.Sprintf("**%s** — %s: %s", ts, msg.From, text)
|
||||
if msg.Media != nil {
|
||||
line += " *(" + *msg.Media + ")*"
|
||||
}
|
||||
b.WriteString(line + "\n\n")
|
||||
}
|
||||
|
||||
mdFile := filepath.Join(mdRoot, "telegram", sanitizeDir(chatName), "messages.md")
|
||||
if err := os.MkdirAll(filepath.Dir(mdFile), 0755); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "chats import: mkdir %s: %v\n", filepath.Dir(mdFile), err)
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
if err := os.WriteFile(mdFile, b.Bytes(), 0644); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "chats import: write %s: %v\n", mdFile, err)
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
written++
|
||||
}
|
||||
fmt.Printf("chats import: %d chats written", written)
|
||||
if failed > 0 {
|
||||
fmt.Printf(", %d failed", failed)
|
||||
}
|
||||
fmt.Println()
|
||||
if failed > 0 {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func readJSONL(path string) ([]Message, string, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
var messages []Message
|
||||
scanner := bufio.NewScanner(f)
|
||||
scanner.Buffer(make([]byte, 1<<20), 1<<20)
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var msg Message
|
||||
if err := json.Unmarshal([]byte(line), &msg); err != nil {
|
||||
continue
|
||||
}
|
||||
messages = append(messages, msg)
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return messages, "", err
|
||||
}
|
||||
|
||||
chatName := ""
|
||||
if len(messages) > 0 {
|
||||
nameCounts := make(map[string]int)
|
||||
for _, msg := range messages {
|
||||
nameCounts[msg.From]++
|
||||
}
|
||||
best := ""
|
||||
bestN := 0
|
||||
for name, n := range nameCounts {
|
||||
if name != "" && name != "unknown" && n > bestN {
|
||||
best = name
|
||||
bestN = n
|
||||
}
|
||||
}
|
||||
if best != "" {
|
||||
chatName = best
|
||||
}
|
||||
}
|
||||
return messages, chatName, nil
|
||||
}
|
||||
|
||||
func collectParticipants(messages []Message) []string {
|
||||
seen := make(map[string]bool)
|
||||
var result []string
|
||||
for _, msg := range messages {
|
||||
if msg.From == "" || seen[msg.From] {
|
||||
continue
|
||||
}
|
||||
seen[msg.From] = true
|
||||
result = append(result, msg.From)
|
||||
}
|
||||
sort.Strings(result)
|
||||
return result
|
||||
}
|
||||
|
||||
func escapeYAML(s string) string {
|
||||
if strings.ContainsAny(s, ":#,[]{}'\"") || strings.HasPrefix(s, "-") {
|
||||
return `"` + strings.ReplaceAll(s, `"`, `\"`) + `"`
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func sanitizeDir(name string) string {
|
||||
r := strings.NewReplacer(
|
||||
"/", "_", "\\", "_", ":", "_", "*", "_",
|
||||
"?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",
|
||||
" ", "_",
|
||||
)
|
||||
return strings.TrimSpace(r.Replace(name))
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func runIndex(args []string) int {
|
||||
fs := flag.NewFlagSet("chats index", flag.ContinueOnError)
|
||||
help := fs.Bool("help", false, "")
|
||||
fs.SetOutput(os.Stderr)
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
if *help {
|
||||
fmt.Fprintln(os.Stderr, "usage: chats index")
|
||||
return 0
|
||||
}
|
||||
|
||||
root := repoRoot()
|
||||
mdDir := filepath.Join(chatsDir(), "md")
|
||||
|
||||
_, err := os.Stat(mdDir)
|
||||
if os.IsNotExist(err) {
|
||||
fmt.Fprintf(os.Stderr, "chats index: no chat markdown at %s; run 'chats import' first\n", mdDir)
|
||||
return 1
|
||||
}
|
||||
|
||||
indexScript := filepath.Join(root, "bin", "kb", "index")
|
||||
if _, err := os.Stat(indexScript); os.IsNotExist(err) {
|
||||
fmt.Fprintf(os.Stderr, "chats index: %s not found\n", indexScript)
|
||||
return 1
|
||||
}
|
||||
|
||||
cmd := exec.Command(indexScript, "--corpus", mdDir)
|
||||
var outBuf, errBuf bytes.Buffer
|
||||
cmd.Stdout = &outBuf
|
||||
cmd.Stderr = &errBuf
|
||||
cmd.Dir = root
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "chats index: %v\n%s", err, errBuf.String())
|
||||
return 1
|
||||
}
|
||||
result := strings.TrimSpace(outBuf.String())
|
||||
if result == "" {
|
||||
result = strings.TrimSpace(errBuf.String())
|
||||
}
|
||||
fmt.Printf("chats index: %s\n", result)
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
// bin/chats - sync, import, index, extract facts, and apply chat data
|
||||
// from Telegram, WhatsApp, LinkedIn into the brain and OnlyOffice CRM.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// chats sync telegram [--limit N] [--since DATE] [--phone PHONE]
|
||||
// chats sync whatsapp [--qr] [--limit N]
|
||||
// chats sync linkedin [--limit N]
|
||||
// chats import # JSONL → MD (all sources)
|
||||
// chats index # rebuild var/kb.lbug with chats
|
||||
// chats facts # extract + cross-check
|
||||
// chats apply [--dry-run] # push to OnlyOffice CRM
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 2 {
|
||||
usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
cmd := os.Args[1]
|
||||
args := os.Args[2:]
|
||||
switch cmd {
|
||||
case "sync":
|
||||
if len(args) < 1 {
|
||||
usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
platform := args[0]
|
||||
platformArgs := args[1:]
|
||||
switch platform {
|
||||
case "telegram":
|
||||
os.Exit(runSyncTelegram(platformArgs))
|
||||
case "whatsapp":
|
||||
fmt.Fprintf(os.Stderr, "chats: WhatsApp not implemented yet\n")
|
||||
os.Exit(1)
|
||||
case "linkedin":
|
||||
fmt.Fprintf(os.Stderr, "chats: LinkedIn not implemented yet\n")
|
||||
os.Exit(1)
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "chats: unknown platform %q\n", platform)
|
||||
os.Exit(2)
|
||||
}
|
||||
case "import":
|
||||
os.Exit(runImport(args))
|
||||
case "index":
|
||||
os.Exit(runIndex(args))
|
||||
case "facts":
|
||||
os.Exit(runFacts(args))
|
||||
case "apply":
|
||||
os.Exit(runApply(args))
|
||||
case "help", "-h", "--help":
|
||||
usage()
|
||||
return
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "chats: unknown command %q\n", cmd)
|
||||
usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
func usage() {
|
||||
w := os.Stderr
|
||||
fmt.Fprintln(w, `Usage: chats <command> [args]
|
||||
|
||||
Commands:
|
||||
sync telegram [--limit N] [--since DATE] [--phone PHONE]
|
||||
sync whatsapp [--qr] [--limit N]
|
||||
sync linkedin [--limit N]
|
||||
import JSONL → MD (all sources)
|
||||
index rebuild var/kb.lbug with chats
|
||||
facts extract + cross-check facts
|
||||
apply [--dry-run] push to OnlyOffice CRM
|
||||
|
||||
Output layout:
|
||||
var/chats/<platform>/<chat_id>/messages.jsonl
|
||||
var/chats/md/<platform>/<chat_name>/messages.md`)
|
||||
}
|
||||
|
||||
// repoRoot locates the 2dph project root by walking up from the binary.
|
||||
func repoRoot() string {
|
||||
if v := os.Getenv("KB_ROOT"); v != "" {
|
||||
return v
|
||||
}
|
||||
wd, err := os.Getwd()
|
||||
if err != nil {
|
||||
return "."
|
||||
}
|
||||
for i := 0; i < 10; i++ {
|
||||
if _, err := os.Stat(wd + "/var"); err == nil {
|
||||
return wd
|
||||
}
|
||||
if _, err := os.Stat(wd + "/.git"); err == nil {
|
||||
return wd
|
||||
}
|
||||
parent := wd
|
||||
if idx := strings.LastIndex(wd, "/"); idx >= 0 {
|
||||
parent = wd[:idx]
|
||||
}
|
||||
if parent == wd {
|
||||
break
|
||||
}
|
||||
wd = parent
|
||||
}
|
||||
return "."
|
||||
}
|
||||
|
||||
// chatsDir returns var/chats under the repo root.
|
||||
func chatsDir() string {
|
||||
return repoRoot() + "/var/chats"
|
||||
}
|
||||
@@ -0,0 +1,424 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type MCPClient struct {
|
||||
cmd *exec.Cmd
|
||||
stdin *bufio.Writer
|
||||
stdout *bufio.Scanner
|
||||
msgID int
|
||||
}
|
||||
|
||||
type mcpRequest struct {
|
||||
JSONRPC string `json:"jsonrpc"`
|
||||
ID int `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Params interface{} `json:"params,omitempty"`
|
||||
}
|
||||
|
||||
type mcpResponse struct {
|
||||
JSONRPC string `json:"jsonrpc"`
|
||||
ID int `json:"id"`
|
||||
Result json.RawMessage `json:"result,omitempty"`
|
||||
Error *struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
} `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type mcpToolResult struct {
|
||||
Content []struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
IsError bool `json:"isError,omitempty"`
|
||||
}
|
||||
|
||||
type ListChatsResult struct {
|
||||
ChatID int64 `json:"chat_id"`
|
||||
Title string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
Username string `json:"username,omitempty"`
|
||||
}
|
||||
|
||||
type listChatsEnvelope struct {
|
||||
Results []ListChatsResult `json:"results"`
|
||||
}
|
||||
|
||||
type historyEnvelope struct {
|
||||
Results []GetHistoryResult `json:"results"`
|
||||
}
|
||||
|
||||
type GetHistoryResult struct {
|
||||
ID int `json:"id"`
|
||||
Sender string `json:"sender"`
|
||||
Date string `json:"date"`
|
||||
Text string `json:"text"`
|
||||
Media string `json:"media,omitempty"`
|
||||
Out bool `json:"out,omitempty"`
|
||||
}
|
||||
|
||||
func NewMCPClient(ctx context.Context, apiID int, apiHash, phone, sessionString, mcpDir string) (*MCPClient, error) {
|
||||
env := os.Environ()
|
||||
env = append(env,
|
||||
fmt.Sprintf("TELEGRAM_API_ID=%d", apiID),
|
||||
fmt.Sprintf("TELEGRAM_API_HASH=%s", apiHash),
|
||||
fmt.Sprintf("TELEGRAM_PHONE=%s", phone),
|
||||
fmt.Sprintf("TELEGRAM_SESSION_STRING=%s", sessionString),
|
||||
"MCP_TRANSPORT=stdio",
|
||||
)
|
||||
|
||||
serverPath := filepath.Join(mcpDir, ".venv", "bin", "python3")
|
||||
mainPath := filepath.Join(mcpDir, "main.py")
|
||||
|
||||
cmd := exec.CommandContext(ctx, serverPath, mainPath)
|
||||
cmd.Env = env
|
||||
cmd.Dir = mcpDir
|
||||
|
||||
stdin, err := cmd.StdinPipe()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stdin pipe: %w", err)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stdout pipe: %w", err)
|
||||
}
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return nil, fmt.Errorf("start mcp: %w", err)
|
||||
}
|
||||
|
||||
c := &MCPClient{
|
||||
cmd: cmd,
|
||||
stdin: bufio.NewWriter(stdin),
|
||||
stdout: bufio.NewScanner(stdout),
|
||||
msgID: 0,
|
||||
}
|
||||
c.stdout.Buffer(make([]byte, 1<<20), 1<<20)
|
||||
|
||||
if err := c.initialize(ctx); err != nil {
|
||||
c.Close()
|
||||
return nil, fmt.Errorf("initialize: %w", err)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (c *MCPClient) nextID() int {
|
||||
c.msgID++
|
||||
return c.msgID
|
||||
}
|
||||
|
||||
func (c *MCPClient) sendRequest(ctx context.Context, method string, params interface{}) (json.RawMessage, error) {
|
||||
id := c.nextID()
|
||||
req := mcpRequest{
|
||||
JSONRPC: "2.0",
|
||||
ID: id,
|
||||
Method: method,
|
||||
Params: params,
|
||||
}
|
||||
body, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal: %w", err)
|
||||
}
|
||||
|
||||
if _, err := c.stdin.Write(body); err != nil {
|
||||
return nil, fmt.Errorf("write: %w", err)
|
||||
}
|
||||
if err := c.stdin.WriteByte('\n'); err != nil {
|
||||
return nil, fmt.Errorf("write newline: %w", err)
|
||||
}
|
||||
if err := c.stdin.Flush(); err != nil {
|
||||
return nil, fmt.Errorf("flush: %w", err)
|
||||
}
|
||||
|
||||
for c.stdout.Scan() {
|
||||
line := c.stdout.Text()
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
var resp mcpResponse
|
||||
if err := json.Unmarshal([]byte(line), &resp); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal response: %w\nline: %s", err, line[:min(len(line), 500)])
|
||||
}
|
||||
if resp.Error != nil {
|
||||
return nil, fmt.Errorf("rpc error %d: %s", resp.Error.Code, resp.Error.Message)
|
||||
}
|
||||
return resp.Result, nil
|
||||
}
|
||||
return nil, fmt.Errorf("no response: %w", c.stdout.Err())
|
||||
}
|
||||
|
||||
func (c *MCPClient) initialize(ctx context.Context) error {
|
||||
params := map[string]interface{}{
|
||||
"protocolVersion": "2024-11-05",
|
||||
"capabilities": map[string]interface{}{},
|
||||
"clientInfo": map[string]string{
|
||||
"name": "chats-sync",
|
||||
"version": "0.1.0",
|
||||
},
|
||||
}
|
||||
_, err := c.sendRequest(ctx, "initialize", params)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *MCPClient) ListChats(ctx context.Context, chatType string, limit int) ([]ListChatsResult, error) {
|
||||
args := map[string]interface{}{
|
||||
"chat_type": chatType,
|
||||
"limit": limit,
|
||||
}
|
||||
result, err := c.sendRequest(ctx, "tools/call", map[string]interface{}{
|
||||
"name": "list_chats",
|
||||
"arguments": args,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var toolRes mcpToolResult
|
||||
if err := json.Unmarshal(result, &toolRes); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal tool result: %w", err)
|
||||
}
|
||||
if toolRes.IsError {
|
||||
msg := "unknown"
|
||||
if len(toolRes.Content) > 0 {
|
||||
msg = toolRes.Content[0].Text
|
||||
}
|
||||
return nil, fmt.Errorf("list_chats error: %s", msg)
|
||||
}
|
||||
if len(toolRes.Content) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
text := toolRes.Content[0].Text
|
||||
if text == "" || text == "No chats found matching the criteria." {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var env listChatsEnvelope
|
||||
if err := json.Unmarshal([]byte(text), &env); err != nil {
|
||||
var arr []ListChatsResult
|
||||
if err2 := json.Unmarshal([]byte(text), &arr); err2 != nil {
|
||||
return nil, fmt.Errorf("parse chats: %w (also tried array: %v)\nbody: %s", err, err2, text[:min(len(text), 500)])
|
||||
}
|
||||
return arr, nil
|
||||
}
|
||||
return env.Results, nil
|
||||
}
|
||||
|
||||
func (c *MCPClient) GetHistory(ctx context.Context, chatID int64, limit int) ([]GetHistoryResult, error) {
|
||||
args := map[string]interface{}{
|
||||
"chat_id": chatID,
|
||||
"limit": limit,
|
||||
}
|
||||
result, err := c.sendRequest(ctx, "tools/call", map[string]interface{}{
|
||||
"name": "get_history",
|
||||
"arguments": args,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var toolRes mcpToolResult
|
||||
if err := json.Unmarshal(result, &toolRes); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal tool result: %w", err)
|
||||
}
|
||||
if toolRes.IsError {
|
||||
msg := "unknown"
|
||||
if len(toolRes.Content) > 0 {
|
||||
msg = toolRes.Content[0].Text
|
||||
}
|
||||
return nil, fmt.Errorf("get_history error: %s", msg)
|
||||
}
|
||||
if len(toolRes.Content) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
text := toolRes.Content[0].Text
|
||||
if text == "" || text == "No messages found for this page." || text == "No messages found matching the criteria." {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var env historyEnvelope
|
||||
if err := json.Unmarshal([]byte(text), &env); err != nil {
|
||||
var arr []GetHistoryResult
|
||||
if err2 := json.Unmarshal([]byte(text), &arr); err2 != nil {
|
||||
return nil, fmt.Errorf("parse history: %w (also tried array: %v)\nbody: %s", err, err2, text[:min(len(text), 500)])
|
||||
}
|
||||
return arr, nil
|
||||
}
|
||||
return env.Results, nil
|
||||
}
|
||||
|
||||
func (c *MCPClient) Close() error {
|
||||
if c.stdin != nil {
|
||||
c.stdin.Flush()
|
||||
}
|
||||
if c.cmd != nil && c.cmd.Process != nil {
|
||||
c.cmd.Process.Kill()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type TelegramMCPSource struct {
|
||||
mcpDir string
|
||||
apiID int
|
||||
apiHash string
|
||||
phone string
|
||||
sessionStr string
|
||||
limit int
|
||||
}
|
||||
|
||||
func NewTelegramMCPSource(apiID int, apiHash, phone, sessionString, mcpDir string) *TelegramMCPSource {
|
||||
return &TelegramMCPSource{
|
||||
mcpDir: mcpDir,
|
||||
apiID: apiID,
|
||||
apiHash: apiHash,
|
||||
phone: phone,
|
||||
sessionStr: sessionString,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TelegramMCPSource) Name() string { return "telegram" }
|
||||
|
||||
func (s *TelegramMCPSource) Sync(ctx context.Context, outDir string, limit int) error {
|
||||
if limit > 0 {
|
||||
s.limit = limit
|
||||
}
|
||||
|
||||
client, err := NewMCPClient(ctx, s.apiID, s.apiHash, s.phone, s.sessionStr, s.mcpDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("mcp client: %w", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
chats, err := client.ListChats(ctx, "user", 100)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list chats: %w", err)
|
||||
}
|
||||
if len(chats) == 0 {
|
||||
fmt.Println("chats: no personal chats found")
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("chats: found %d personal chats\n", len(chats))
|
||||
|
||||
var filtered []ListChatsResult
|
||||
for _, c := range chats {
|
||||
if strings.Contains(strings.ToLower(c.Username), "bot") {
|
||||
continue
|
||||
}
|
||||
if c.ChatID == 777000 { // Telegram service
|
||||
continue
|
||||
}
|
||||
filtered = append(filtered, c)
|
||||
}
|
||||
fmt.Printf("chats: %d after filter (bots excluded)\n", len(filtered))
|
||||
|
||||
for _, chat := range filtered {
|
||||
chatID := fmt.Sprintf("user_%d", chat.ChatID)
|
||||
chatName := chat.Title
|
||||
if chatName == "" {
|
||||
chatName = chatID
|
||||
}
|
||||
|
||||
chatDir := filepath.Join(outDir, "telegram", chatID)
|
||||
if err := os.MkdirAll(chatDir, 0755); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "chats: mkdir %s: %v\n", chatDir, err)
|
||||
continue
|
||||
}
|
||||
|
||||
jsonlPath := filepath.Join(chatDir, "messages.jsonl")
|
||||
f, err := os.Create(jsonlPath)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "chats: create %s: %v\n", jsonlPath, err)
|
||||
continue
|
||||
}
|
||||
|
||||
msgLimit := 100
|
||||
if s.limit > 0 {
|
||||
msgLimit = s.limit
|
||||
}
|
||||
|
||||
msgs, err := client.GetHistory(ctx, chat.ChatID, msgLimit)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "chats: get_history for %s: %v\n", chatName, err)
|
||||
f.Close()
|
||||
continue
|
||||
}
|
||||
|
||||
enc := json.NewEncoder(f)
|
||||
written := 0
|
||||
for _, m := range msgs {
|
||||
if m.Out {
|
||||
continue
|
||||
}
|
||||
text := m.Text
|
||||
if text == "" && m.Media != "" {
|
||||
text = fmt.Sprintf("[%s]", m.Media)
|
||||
}
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
sender := cleanSender(m.Sender)
|
||||
ts := m.Date
|
||||
if t, err := time.Parse(time.RFC3339, m.Date); err == nil {
|
||||
ts = t.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
chatMsg := Message{
|
||||
ID: fmt.Sprintf("tg_%d_%d", chat.ChatID, m.ID),
|
||||
Timestamp: ts,
|
||||
From: sender,
|
||||
Text: text,
|
||||
Platform: "telegram",
|
||||
}
|
||||
if m.Media != "" {
|
||||
desc := fmt.Sprintf("[%s]", m.Media)
|
||||
chatMsg.Media = &desc
|
||||
}
|
||||
if err := enc.Encode(chatMsg); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "chats: encode msg: %v\n", err)
|
||||
continue
|
||||
}
|
||||
written++
|
||||
}
|
||||
f.Close()
|
||||
|
||||
if written > 0 {
|
||||
fmt.Printf("chats: synced %s (%s) — %d messages\n", chatName, chatID, written)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanSender(sender string) string {
|
||||
if idx := strings.Index(sender, " ("); idx > 0 {
|
||||
sender = sender[:idx]
|
||||
} else if idx := strings.Index(sender, " @"); idx > 0 {
|
||||
sender = sender[:idx]
|
||||
}
|
||||
if idx := strings.Index(sender, " ["); idx > 0 {
|
||||
sender = sender[:idx]
|
||||
}
|
||||
return sender
|
||||
}
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Source interface {
|
||||
Name() string
|
||||
Sync(ctx context.Context, outDir string, limit int) error
|
||||
}
|
||||
|
||||
type Message struct {
|
||||
ID string `json:"id"`
|
||||
Timestamp string `json:"ts"`
|
||||
From string `json:"from"`
|
||||
Text string `json:"text"`
|
||||
Media *string `json:"media,omitempty"`
|
||||
Platform string `json:"platform"`
|
||||
}
|
||||
|
||||
type ChatInfo struct {
|
||||
ID string `json:"id"`
|
||||
Platform string `json:"platform"`
|
||||
Name string `json:"name"`
|
||||
Participants []string `json:"participants"`
|
||||
Type string `json:"type"`
|
||||
MessageCount int `json:"messageCount"`
|
||||
LastTS string `json:"lastTs,omitempty"`
|
||||
}
|
||||
|
||||
func envVar(key, fallback string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func parseSince(s string) (time.Time, error) {
|
||||
for _, layout := range []string{
|
||||
time.RFC3339,
|
||||
"2006-01-02T15:04:05",
|
||||
"2006-01-02",
|
||||
} {
|
||||
if t, err := time.Parse(layout, s); err == nil {
|
||||
return t, nil
|
||||
}
|
||||
}
|
||||
return time.Time{}, fmt.Errorf("cannot parse --since %q; use YYYY-MM-DD or RFC3339", s)
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func runSyncTelegram(args []string) int {
|
||||
fs := flag.NewFlagSet("chats sync telegram", flag.ContinueOnError)
|
||||
limit := fs.Int("limit", 0, "max messages per chat (0 = all)")
|
||||
phone := fs.String("phone", "", "phone number (default env TELEGRAM_PHONE)")
|
||||
help := fs.Bool("help", false, "")
|
||||
fs.SetOutput(os.Stderr)
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
if *help {
|
||||
fmt.Fprintln(os.Stderr, "usage: chats sync telegram [--limit N] [--phone PHONE]")
|
||||
return 0
|
||||
}
|
||||
|
||||
apiIDStr := envVar("TELEGRAM_API_ID", "")
|
||||
apiHash := envVar("TELEGRAM_API_HASH", "")
|
||||
sessionStr := envVar("TELEGRAM_SESSION_STRING", "")
|
||||
phoneNum := *phone
|
||||
if phoneNum == "" {
|
||||
phoneNum = envVar("TELEGRAM_PHONE", "")
|
||||
}
|
||||
if apiIDStr == "" || apiHash == "" || phoneNum == "" {
|
||||
fmt.Fprintln(os.Stderr, "chats: need TELEGRAM_API_ID, TELEGRAM_API_HASH, TELEGRAM_PHONE in env")
|
||||
return 2
|
||||
}
|
||||
apiID, err := strconv.Atoi(apiIDStr)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "chats: invalid TELEGRAM_API_ID %q\n", apiIDStr)
|
||||
return 2
|
||||
}
|
||||
|
||||
// Locate telegram-mcp directory
|
||||
mcpDirs := []string{
|
||||
envVar("TELEGRAM_MCP_DIR", ""),
|
||||
"/mnt/8TB/projects/eslider/mcp-servers/telegram-mcp",
|
||||
filepath.Join(os.Getenv("HOME"), "projects", "eSlider", "mcp-servers", "telegram-mcp"),
|
||||
}
|
||||
var mcpDir string
|
||||
for _, d := range mcpDirs {
|
||||
if d != "" {
|
||||
if _, err := os.Stat(filepath.Join(d, "main.py")); err == nil {
|
||||
mcpDir = d
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if mcpDir == "" {
|
||||
fmt.Fprintln(os.Stderr, "chats: telegram-mcp not found; set TELEGRAM_MCP_DIR")
|
||||
return 1
|
||||
}
|
||||
|
||||
if sessionStr == "" {
|
||||
// Fallback: try to read from telegram-mcp .env
|
||||
envPath := filepath.Join(mcpDir, ".env")
|
||||
if data, err := os.ReadFile(envPath); err == nil {
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
if strings.HasPrefix(line, "TELEGRAM_SESSION_STRING=") {
|
||||
sessionStr = strings.TrimPrefix(line, "TELEGRAM_SESSION_STRING=")
|
||||
sessionStr = strings.Trim(sessionStr, "\"'")
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if sessionStr == "" {
|
||||
// Also try the env file at /mnt/8TB/projects/eslider/mcp-servers/telegram.env
|
||||
envPath := "/mnt/8TB/projects/eslider/mcp-servers/telegram.env"
|
||||
if data, err := os.ReadFile(envPath); err == nil {
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
if strings.HasPrefix(line, "TELEGRAM_SESSION_STRING=") {
|
||||
sessionStr = strings.TrimPrefix(line, "TELEGRAM_SESSION_STRING=")
|
||||
sessionStr = strings.Trim(sessionStr, "\"'")
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if sessionStr == "" {
|
||||
fmt.Fprintln(os.Stderr, "chats: TELEGRAM_SESSION_STRING not found in env or .env files")
|
||||
return 1
|
||||
}
|
||||
|
||||
src := NewTelegramMCPSource(apiID, apiHash, phoneNum, sessionStr, mcpDir)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
start := time.Now()
|
||||
if err := src.Sync(ctx, chatsDir(), *limit); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "chats sync telegram: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
fmt.Printf("chats sync telegram: completed in %s\n", time.Since(start).Round(time.Millisecond))
|
||||
return 0
|
||||
}
|
||||
@@ -18,8 +18,627 @@ OO_CLI path to oo binary (default: $HOME/go/bin/oo)
|
||||
|
||||
## Quick reference
|
||||
|
||||
<<<<<<< HEAD
|
||||
./bin/chat sync telegram --limit 100
|
||||
./bin/chat import
|
||||
./bin/chat index # delegates to bin/kb/index
|
||||
./bin/chat facts
|
||||
./bin/chat apply --dry-run
|
||||
||||||| parent of 313599b (bin/chats: Phase 1 MVP — Telegram sync/import/index/facts/apply)
|
||||
---
|
||||
|
||||
## 1. Находки / Discoveries
|
||||
|
||||
### Доступные платформы и данные
|
||||
|
||||
| Platform | Status | Credentials | Go Library |
|
||||
|----------|--------|-------------|------------|
|
||||
| **Telegram** | ✅ Active MCP server, session string exists | API ID: `30382285`, API Hash: `fd8a8c1e987bb908b457f374b721c062`, Session string in `/mnt/8TB/projects/eslider/mcp-servers/telegram.env` and `/mnt/8TB/projects/eslider/mcp-servers/telegram-mcp/.env`. Phone: `+34643861471` | [`iyear/tdl`](https://github.com/iyear/tdl) (7.5k★, Go, gotd/td, экспорт в JSON) |
|
||||
| **WhatsApp** | ✅ MCP server exists + Go bridge. **No `messages.db` yet** (empty). Needs QR re-auth + sync first | MCP: `/mnt/8TB/projects/eslider/mcp-servers/whatsapp-mcp/`. Go bridge uses `tulir/whatsmeow` | [`tulir/whatsmeow`](https://github.com/tulir/whatsmeow) (7k★, Go, multi-device API) |
|
||||
| **LinkedIn** | ✅ Connected via `mcp-server-linkedin`. Full browser profile + cookies | Browser profile: `~/.linkedin-mcp/profile/`. Active cookies: `~/.linkedin-mcp/storage-state.json`. Cookie backup: `~/.linkedin-mcp/cookies.json.bak-agent`. Script cookies: `/mnt/8TB/projects/eslider/cv/bin/creds/linkedin.cookies.json`. JobHunt session: `/mnt/8TB/projects/eslider/JobHunt/sessions/linkedin.json`. Env: `/mnt/8TB/projects/eslider/cv/.env` (`LINKEDIN_AUTH_TOKEN`). | [`swiftlysingh/lnk`](https://github.com/swiftlysingh/lnk) (Go CLI, Voyager API) |
|
||||
| **Gmail** | ✅ Already working pipeline (`bin/mail/sync.go` + `bin/mail/import`) | OAuth at `~/.gmail-mcp/` | Already in 2dph |
|
||||
| **Twitter/X.com** | ⚠️ Password only, no MCP, no session. Not needed now. | `/mnt/8TB/projects/eslider/scratches/Accounts/twitter-pass.txt` | [`benoitpetit/xsh`](https://github.com/benoitpetit/xsh) (Go CLI, cookie auth) |
|
||||
| **Google Calendar** | ❌ Not connected yet. Gmail OAuth could extend | None | Нужен Google Calendar API |
|
||||
|
||||
### Telegram session (проверен, активен)
|
||||
|
||||
- API credentials in: `/mnt/8TB/projects/eslider/mcp-servers/telegram.env`
|
||||
- `TELEGRAM_API_ID=30382285`
|
||||
- `TELEGRAM_API_HASH=fd8a8c1e987bb908b457f374b721c062`
|
||||
- `TELEGRAM_PHONE=+34643861471`
|
||||
- `TELEGRAM_SESSION_STRING` — длинная строка, active session
|
||||
- MCP server launcher: `/mnt/8TB/projects/eslider/mcp-servers/run-telegram-mcp.sh`
|
||||
- Session generator: `/mnt/8TB/projects/eslider/mcp-servers/telegram-mcp/session_string_generator.py`
|
||||
- Regenerator: `/mnt/8TB/projects/eslider/mcp-servers/regen-telegram-session.py`
|
||||
|
||||
### LinkedIn authentication (работает)
|
||||
|
||||
- Запускается через Cursor MCP: `uvx mcp-server-linkedin@latest --user-data-dir /home/ano/.linkedin-mcp/profile --no-auto-import`
|
||||
- Cookie файлы:
|
||||
- `~/.linkedin-mcp/storage-state.json` — Playwright storage state (активные cookies)
|
||||
- `~/.linkedin-mcp/cookies.json.bak-agent` — резерв
|
||||
- `~/.linkedin-mcp/cookies.json.bak-webtop-20260728T143535Z` — резерв
|
||||
- `/mnt/8TB/projects/eslider/cv/bin/creds/linkedin.cookies.json` — для скриптов
|
||||
- `/mnt/8TB/projects/eslider/JobHunt/sessions/linkedin.json` — JobHunt
|
||||
- `li_at` token: `AQEDASsyC9gF8hS2AAABnC2A_vIAAAGcdZZfTk4Ahfp9wURXwTpCAAtRIw_htVldds5UN9JDr6L_hr3lN5CcPxnKDvy-CxQo0apf7K4pmftQzQYHdqpKNNzlNakcvEcOOalpqRxPhlTfIt8jxqevkZcr`
|
||||
- Для Go подхода: можно читать cookie из `storage-state.json` и использовать LinkedIn Voyager API напрямую
|
||||
|
||||
### WhatsApp bridge (не синхронизирован)
|
||||
|
||||
- Go bridge at: `/mnt/8TB/projects/eslider/mcp-servers/whatsapp-mcp/whatsapp-bridge/`
|
||||
- Использует `tulir/whatsmeow` Go library (уже в `go.mod`)
|
||||
- SQLite store: `whatsapp-bridge/store/messages.db` — **не существует** (0 байт), нужна первая авторизация по QR
|
||||
- Launcher: `/mnt/8TB/projects/eslider/mcp-servers/run-whatsapp-bridge-lharries.sh`
|
||||
- QR images готовились: `/mnt/8TB/projects/produktor/work/data/export/whatsapp-qr.png`
|
||||
- Пароль: `Oomtr78k39` (из `.env` OnlyOffice, он же для многих сервисов)
|
||||
|
||||
### OnlyOffice доступ
|
||||
|
||||
- `.env` в корне 2dph: `ONLYOFFICE_URL=https://office.produktor.io`, `ONLYOFFICE_USER=eslider@gmail.com`, `ONLYOFFICE_PASS=Oomtr78k39`
|
||||
- `oo` CLI установлен: `/home/ano/go/bin/oo` (go-onlyoffice)
|
||||
- go-onlyoffice repo: `/mnt/8TB/projects/eslider/go-onlyoffice/` (main, public)
|
||||
- oo-workspace (каталог, private): `/mnt/8TB/projects/eslider/oo-workspace/`
|
||||
|
||||
### Gitea
|
||||
|
||||
- Instance: `https://git.produktor.io` (через `gitea-api` wrapper)
|
||||
- Credentials: `~/.config/gitea/produktor.env`
|
||||
- User: `eSlider`, admin
|
||||
- Repos: 30+ приватных репозиториев
|
||||
- **`eSlider/2dph` не существует на Gitea** — проект только на GitHub
|
||||
- Для задачи: создать issue в существующем репо (например `eSlider/JobHunt`) или создать новый репо
|
||||
|
||||
### Rambox
|
||||
|
||||
Не используется. IndexedDB база не существует.
|
||||
|
||||
---
|
||||
|
||||
## 2. Design Decisions
|
||||
|
||||
### One binary: `bin/chats`
|
||||
|
||||
```go
|
||||
// Source interface — каждая платформа реализует
|
||||
type Source interface {
|
||||
Name() string // "telegram", "whatsapp", "linkedin"
|
||||
Sync(ctx context.Context, out string) error
|
||||
}
|
||||
|
||||
// CLI subcommands
|
||||
chats sync telegram [--limit N] [--since DATE]
|
||||
chats sync whatsapp [--qr] [--limit N]
|
||||
chats sync linkedin [--limit N]
|
||||
chats import // JSONL → markdown (all sources)
|
||||
chats index // rebuild var/kb.lbug
|
||||
chats facts // extract + cross-check
|
||||
chats apply // push to OO CRM [--dry-run]
|
||||
```
|
||||
|
||||
### Directory layout (полные имена платформ)
|
||||
|
||||
```
|
||||
var/chats/
|
||||
telegram/<chat_id>/messages.jsonl
|
||||
whatsapp/<jid>/messages.jsonl
|
||||
linkedin/<conversation_id>/messages.jsonl
|
||||
md/<source>/<chat_name>/message.md
|
||||
```
|
||||
|
||||
### JSONL format (одна строка = одно сообщение)
|
||||
|
||||
```json
|
||||
{"id":"123","ts":"2026-01-15T10:30:00Z","from":"me","text":"Hello!","media":null,"platform":"telegram"}
|
||||
```
|
||||
|
||||
### MD format (YAML frontmatter)
|
||||
|
||||
```markdown
|
||||
---
|
||||
id: tg_12345
|
||||
platform: telegram
|
||||
chat_id: "-100123456"
|
||||
chat_name: "John Doe"
|
||||
participants: ["me", "John Doe"]
|
||||
message_count: 1500
|
||||
type: personal
|
||||
---
|
||||
|
||||
# Чат с John Doe
|
||||
|
||||
**2026-01-15 10:30** — me: Hello!
|
||||
```
|
||||
|
||||
### Фильтр "личный чат"
|
||||
|
||||
- ≤3 участников
|
||||
- Боты не считаются (Telegram: `user.is_bot == false`)
|
||||
- Не канал, не публичная группа
|
||||
- Telegram: `dialog.Type == "user"` или супергруппа с `participants_count <= 3` (исключая ботов)
|
||||
- WhatsApp: `@s.whatsapp.net` (не `@g.us`)
|
||||
- LinkedIn: 1:1 conversation
|
||||
|
||||
### Очередность реализации
|
||||
|
||||
1. **Telegram** — самый простой (session string + API ID/Hash активны)
|
||||
2. **Import** — конвертер JSONL → MD
|
||||
3. **Index** — интеграция с brain
|
||||
4. **WhatsApp** — после первого QR
|
||||
5. **Facts** — извлечение контактных данных
|
||||
6. **LinkedIn** — после проверки cookie
|
||||
7. **Apply** — запись в OO CRM
|
||||
|
||||
### Тестирование (TDD, system tests only)
|
||||
|
||||
- Никаких unit-тестов, никаких моков
|
||||
- Тесты используют реальные данные (слепки)
|
||||
- Workflow use-case как тест:
|
||||
1. Написать тест с реальным сценарием (sync → import → index → search)
|
||||
2. Проверить что файлы созданы
|
||||
3. Проверить что поиск возвращает ожидаемые результаты
|
||||
- A/B тест: сравнить два последовательных sync на идентичность
|
||||
- Integration: тест через `oo` CLI с `--dry-run`
|
||||
|
||||
### Cross-check (detective method)
|
||||
|
||||
Перед записью в OO CRM — минимум 2 независимых источника:
|
||||
- S1: Чат (текст сообщения, автор, дата)
|
||||
- S2: OO CRM (существующий контакт/компания/deal) через `oo persons list` / `oo persons get`
|
||||
- S3 (опционально): Corpus SoT (`knowledge-mesh-seed.yaml`) или web-search
|
||||
- Факты с ≥2 источниками → `root=facts` в brain + `approve=true` для CRM
|
||||
- Факты с 1 источником → предложение на review (catalog-паттерн)
|
||||
|
||||
### Apply в OO CRM
|
||||
|
||||
Через `oo` CLI:
|
||||
- `oo persons update <id> --about ... --job-title ...`
|
||||
- `oo contacts info-add <id> --type Phone|LinkedIn --value ...`
|
||||
- `oo opportunities create --contact-id ... --title ... --stage ...`
|
||||
- `oo persons update <id> --company-id ...` (ассоциация)
|
||||
- `oo projects contacts <project-id>` (связь с проектом)
|
||||
|
||||
---
|
||||
|
||||
## 3. Implementation Plan
|
||||
|
||||
### Phase 1: Telegram sync + import (MVP)
|
||||
|
||||
1. Create `bin/chats/` directory structure (nested Go module like `bin/kbsearch/`)
|
||||
2. Implement `Source` interface
|
||||
3. Implement `TelegramSource` using `gotd/td` (import from `iyear/tdl` or direct)
|
||||
4. Write JSONL output to `var/chats/telegram/<id>/messages.jsonl`
|
||||
5. Implement `chats import` — reads all JSONL, writes MD
|
||||
6. Write system test: real sync → import → verify output
|
||||
|
||||
### Phase 2: Brain indexing
|
||||
|
||||
1. Implement `chats index` — rebuild `var/kb.lbug` with corpus + chats
|
||||
2. Write system test: sync → import → index → `bin/kb/search "query"` → verify recall
|
||||
|
||||
### Phase 3: WhatsApp
|
||||
|
||||
1. Implement `WhatsAppSource` using `tulir/whatsmeow`
|
||||
2. First run requires QR auth (save session for later reuse)
|
||||
3. Write sync output to `var/chats/whatsapp/<jid>/messages.jsonl`
|
||||
|
||||
### Phase 4: Facts extraction + cross-check
|
||||
|
||||
1. Implement `chats facts`:
|
||||
- Phone regex: `\+?\d{7,15}`
|
||||
- LinkedIn URL regex: `linkedin\.com/in/[\w-]+`
|
||||
- Cross-check: match extracted values against OO CRM API
|
||||
2. Write facts to brain as `root=facts`
|
||||
|
||||
### Phase 5: LinkedIn + Apply
|
||||
|
||||
1. Implement `LinkedInSource` using Voyager API + cookies from `storage-state.json`
|
||||
2. Implement `chats apply`:
|
||||
- Build YAML catalog (as `oo-workspace` does)
|
||||
- Run `oo catalog match` → review → `oo catalog apply`
|
||||
|
||||
---
|
||||
|
||||
## 4. Existing code references
|
||||
|
||||
| Code | Path | Purpose |
|
||||
|------|------|---------|
|
||||
| Mail sync | `bin/mail/sync.go` + `bin/mail/sync/` | Pattern for chat sync (Go subprocess pattern) |
|
||||
| Mail import | `bin/mail/import` | Pattern for JSONL → MD conversion |
|
||||
| Mail index | `bin/mail/index_mail` | Pattern for brain rebuild with new corpus |
|
||||
| kbsearch | `bin/kbsearch/` | Go module pattern (nested, `system_ladybug` tag) |
|
||||
| kb/index | `bin/kb/index` | Python brain index (upsert_leaf, init_schema) |
|
||||
| kblib | `bin/tools/kblib.py` | Python brain core (connect, upsert_leaf, hybrid_search) |
|
||||
| facts/crm | `bin/facts/crm` | Cross-check CRM facts pattern |
|
||||
| facts/extract | `bin/facts/extract` | 2-source fact extraction pattern |
|
||||
| oo-workspace catalog | `/mnt/8TB/projects/eslider/oo-workspace/catalog/` | Catalog scan/match/apply pattern (VCF, Thunderbird, projects) |
|
||||
| WhatsApp bridge | `/mnt/8TB/projects/eslider/mcp-servers/whatsapp-mcp/whatsapp-bridge/` | Existing `whatsmeow` Go bridge |
|
||||
| go-onlyoffice | `/mnt/8TB/projects/eslider/go-onlyoffice/` | OO CRM API library |
|
||||
| oo CLI | `/home/ano/go/bin/oo` | Built go-onlyoffice CLI |
|
||||
|
||||
---
|
||||
|
||||
## 5. Critical files
|
||||
|
||||
| File | Content |
|
||||
|------|---------|
|
||||
| `~/.config/gitea/produktor.env` | Gitea URL + token |
|
||||
| `/mnt/8TB/projects/eslider/mcp-servers/telegram.env` | Telegram API ID/Hash + phone + session |
|
||||
| `/mnt/8TB/projects/eslider/mcp-servers/telegram-mcp/.env` | Telegram session string |
|
||||
| `/mnt/8TB/projects/ai/2dph/.env` | ONLYOFFICE URL/USER/PASS |
|
||||
| `/mnt/8TB/projects/eslider/cv/.env` | LinkedIn auth token |
|
||||
| `~/.linkedin-mcp/storage-state.json` | LinkedIn active cookies |
|
||||
| `/mnt/8TB/projects/eslider/go-onlyoffice/.env` | OO creds for oo CLI |
|
||||
| `/mnt/8TB/projects/eslider/go-onlyoffice/go.mod` | Deps for go-onlyoffice |
|
||||
| `/mnt/8TB/projects/eslider/oo-workspace/go.mod` | Deps for oo-workspace |
|
||||
|
||||
---
|
||||
|
||||
## 6. Agent prompt (для запуска новой сессии)
|
||||
|
||||
Скопируйте этот промпт при старте новой сессии:
|
||||
|
||||
```
|
||||
Ты — ассистент для разработки chat import pipeline в проекте 2dph (eSlider/2dph).
|
||||
|
||||
Прочитай docs/chat-import-plan.md полностью. Это план, находки и дизайн.
|
||||
|
||||
Задача: реализовать bin/chats — единый Go бинарник для sync/import/index/facts/apply
|
||||
личных чатов из Telegram, WhatsApp, LinkedIn с последующей записью фактов в OnlyOffice CRM.
|
||||
|
||||
Правила:
|
||||
1. TDD first — workflow use-case как тест. Реальные данные, без моков.
|
||||
2. Go, всё в один бинарник bin/chats.
|
||||
3. Source interface — каждая платформа плагином.
|
||||
4. var/chats/telegram/ — полные имена платформ.
|
||||
5. Начни с Telegram (session string активна).
|
||||
6. После реализации — создай/обнови тесты.
|
||||
7. Сохрани прогресс в docs/chat-import-plan.md.
|
||||
8. Запроси разрешение перед apply в OO CRM.
|
||||
|
||||
Ключевые пути:
|
||||
- /mnt/8TB/projects/ai/2dph/ — корень проекта
|
||||
- /mnt/8TB/projects/eslider/mcp-servers/telegram-mcp/ — Telegram MCP
|
||||
- /mnt/8TB/projects/eslider/mcp-servers/telegram.env — API ID/Hash + session
|
||||
- /mnt/8TB/projects/eslider/go-onlyoffice/ — go-onlyoffice библиотека
|
||||
- /home/ano/go/bin/oo — OO CLI
|
||||
|
||||
Готов начать.
|
||||
```
|
||||
=======
|
||||
---
|
||||
|
||||
## 1. Находки / Discoveries
|
||||
|
||||
### Доступные платформы и данные
|
||||
|
||||
| Platform | Status | Credentials (path, not values) | Go Library |
|
||||
|----------|--------|-------------------------------|------------|
|
||||
| **Telegram** | ✅ Active MCP server, session string exists | API creds in `telegram.env` (see §5). Session string in `telegram-mcp/.env`. | MCP JSON-RPC (через telegram-mcp server) |
|
||||
| **WhatsApp** | ✅ MCP server exists + Go bridge. **No `messages.db` yet** (empty). Needs QR re-auth + sync first | MCP: `/mnt/8TB/projects/eslider/mcp-servers/whatsapp-mcp/`. Go bridge uses `tulir/whatsmeow` | [`tulir/whatsmeow`](https://github.com/tulir/whatsmeow) (7k★, Go, multi-device API) |
|
||||
| **LinkedIn** | ✅ Connected via `mcp-server-linkedin`. Full browser profile + cookies | Browser profile: `~/.linkedin-mcp/profile/`. Active cookies: `~/.linkedin-mcp/storage-state.json`. Cookie backup: `~/.linkedin-mcp/cookies.json.bak-agent`. Script cookies: `/mnt/8TB/projects/eslider/cv/bin/creds/linkedin.cookies.json`. JobHunt session: `/mnt/8TB/projects/eslider/JobHunt/sessions/linkedin.json`. Env: `/mnt/8TB/projects/eslider/cv/.env` (`LINKEDIN_AUTH_TOKEN`). | [`swiftlysingh/lnk`](https://github.com/swiftlysingh/lnk) (Go CLI, Voyager API) |
|
||||
| **Gmail** | ✅ Already working pipeline (`bin/mail/sync.go` + `bin/mail/import`) | OAuth at `~/.gmail-mcp/` | Already in 2dph |
|
||||
| **Twitter/X.com** | ⚠️ Password only, no MCP, no session. Not needed now. | `/mnt/8TB/projects/eslider/scratches/Accounts/twitter-pass.txt` | [`benoitpetit/xsh`](https://github.com/benoitpetit/xsh) (Go CLI, cookie auth) |
|
||||
| **Google Calendar** | ❌ Not connected yet. Gmail OAuth could extend | None | Нужен Google Calendar API |
|
||||
|
||||
### Telegram session (проверен, активен)
|
||||
|
||||
- API credentials in: `/mnt/8TB/projects/eslider/mcp-servers/telegram.env`
|
||||
- `TELEGRAM_API_ID`, `TELEGRAM_API_HASH`, `TELEGRAM_PHONE` — загружаются из env
|
||||
- `TELEGRAM_SESSION_STRING` — активная Telethon session (в env)
|
||||
- MCP server launcher: `/mnt/8TB/projects/eslider/mcp-servers/run-telegram-mcp.sh`
|
||||
- Session generator: `/mnt/8TB/projects/eslider/mcp-servers/telegram-mcp/session_string_generator.py`
|
||||
- Regenerator: `/mnt/8TB/projects/eslider/mcp-servers/regen-telegram-session.py`
|
||||
|
||||
### LinkedIn authentication (работает)
|
||||
|
||||
- Запускается через Cursor MCP: `uvx mcp-server-linkedin@latest --user-data-dir ~/.linkedin-mcp/profile --no-auto-import`
|
||||
- Cookie файлы:
|
||||
- `~/.linkedin-mcp/storage-state.json` — Playwright storage state (активные cookies)
|
||||
- `~/.linkedin-mcp/cookies.json.bak-agent` — резерв
|
||||
- `~/.linkedin-mcp/cookies.json.bak-webtop-20260728T143535Z` — резерв
|
||||
- `/mnt/8TB/projects/eslider/cv/bin/creds/linkedin.cookies.json` — для скриптов
|
||||
- `/mnt/8TB/projects/eslider/JobHunt/sessions/linkedin.json` — JobHunt
|
||||
- `li_at` token: в `storage-state.json` (см. §5)
|
||||
- Для Go подхода: можно читать cookie из `storage-state.json` и использовать LinkedIn Voyager API напрямую
|
||||
|
||||
### WhatsApp bridge (не синхронизирован)
|
||||
|
||||
- Go bridge at: `/mnt/8TB/projects/eslider/mcp-servers/whatsapp-mcp/whatsapp-bridge/`
|
||||
- Использует `tulir/whatsmeow` Go library (уже в `go.mod`)
|
||||
- SQLite store: `whatsapp-bridge/store/messages.db` — **не существует** (0 байт), нужна первая авторизация по QR
|
||||
- Launcher: `/mnt/8TB/projects/eslider/mcp-servers/run-whatsapp-bridge-lharries.sh`
|
||||
- QR images готовились: `/mnt/8TB/projects/produktor/work/data/export/whatsapp-qr.png`
|
||||
- Пароль: в `.env` OnlyOffice
|
||||
|
||||
### OnlyOffice доступ
|
||||
|
||||
- `.env` в корне 2dph: `ONLYOFFICE_URL`, `ONLYOFFICE_USER`, `ONLYOFFICE_PASS`
|
||||
- `oo` CLI установлен: `/home/ano/go/bin/oo` (go-onlyoffice)
|
||||
- go-onlyoffice repo: `/mnt/8TB/projects/eslider/go-onlyoffice/` (main, public)
|
||||
- oo-workspace (каталог, private): `/mnt/8TB/projects/eslider/oo-workspace/`
|
||||
|
||||
### Gitea
|
||||
|
||||
- Instance: `https://git.produktor.io` (через `gitea-api` wrapper)
|
||||
- Credentials: `~/.config/gitea/produktor.env`
|
||||
- User: `eSlider`, admin
|
||||
- Repos: 30+ приватных репозиториев
|
||||
- **`eSlider/2dph` не существует на Gitea** — проект только на GitHub
|
||||
- Для задачи: создать issue в существующем репо (например `eSlider/JobHunt`) или создать новый репо
|
||||
|
||||
### Rambox
|
||||
|
||||
Не используется. IndexedDB база не существует.
|
||||
|
||||
---
|
||||
|
||||
## 2. Design Decisions
|
||||
|
||||
### One binary: `bin/chats`
|
||||
|
||||
```go
|
||||
// Source interface — каждая платформа реализует
|
||||
type Source interface {
|
||||
Name() string // "telegram", "whatsapp", "linkedin"
|
||||
Sync(ctx context.Context, out string) error
|
||||
}
|
||||
|
||||
// CLI subcommands
|
||||
chats sync telegram [--limit N] [--since DATE]
|
||||
chats sync whatsapp [--qr] [--limit N]
|
||||
chats sync linkedin [--limit N]
|
||||
chats import // JSONL → markdown (all sources)
|
||||
chats index // rebuild var/kb.lbug
|
||||
chats facts // extract + cross-check
|
||||
chats apply // push to OO CRM [--dry-run]
|
||||
```
|
||||
|
||||
### Directory layout (полные имена платформ)
|
||||
|
||||
```
|
||||
var/chats/
|
||||
telegram/<chat_id>/messages.jsonl
|
||||
whatsapp/<jid>/messages.jsonl
|
||||
linkedin/<conversation_id>/messages.jsonl
|
||||
md/<source>/<chat_name>/message.md
|
||||
```
|
||||
|
||||
### JSONL format (одна строка = одно сообщение)
|
||||
|
||||
```json
|
||||
{"id":"123","ts":"2026-01-15T10:30:00Z","from":"me","text":"Hello!","media":null,"platform":"telegram"}
|
||||
```
|
||||
|
||||
### MD format (YAML frontmatter)
|
||||
|
||||
```markdown
|
||||
---
|
||||
id: tg_12345
|
||||
platform: telegram
|
||||
chat_id: "-100123456"
|
||||
chat_name: "John Doe"
|
||||
participants: ["me", "John Doe"]
|
||||
message_count: 1500
|
||||
type: personal
|
||||
---
|
||||
|
||||
# Чат с John Doe
|
||||
|
||||
**2026-01-15 10:30** — me: Hello!
|
||||
```
|
||||
|
||||
### Фильтр "личный чат"
|
||||
|
||||
- ≤3 участников
|
||||
- Боты не считаются (Telegram: `user.is_bot == false`)
|
||||
- Не канал, не публичная группа
|
||||
- Telegram: `dialog.Type == "user"` или супергруппа с `participants_count <= 3` (исключая ботов)
|
||||
- WhatsApp: `@s.whatsapp.net` (не `@g.us`)
|
||||
- LinkedIn: 1:1 conversation
|
||||
|
||||
### Очередность реализации
|
||||
|
||||
1. **Telegram** — самый простой (session string + API ID/Hash активны)
|
||||
2. **Import** — конвертер JSONL → MD
|
||||
3. **Index** — интеграция с brain
|
||||
4. **WhatsApp** — после первого QR
|
||||
5. **Facts** — извлечение контактных данных
|
||||
6. **LinkedIn** — после проверки cookie
|
||||
7. **Apply** — запись в OO CRM
|
||||
|
||||
### Тестирование (TDD, system tests only)
|
||||
|
||||
- Никаких unit-тестов, никаких моков
|
||||
- Тесты используют синтетические данные (слепки)
|
||||
- Workflow use-case как тест:
|
||||
1. Написать тест с реальным сценарием (sync → import → index → search)
|
||||
2. Проверить что файлы созданы
|
||||
3. Проверить что поиск возвращает ожидаемые результаты
|
||||
- A/B тест: сравнить два последовательных sync на идентичность
|
||||
- Integration: тест через `oo` CLI с `--dry-run`
|
||||
|
||||
### Cross-check (detective method)
|
||||
|
||||
Перед записью в OO CRM — минимум 2 независимых источника:
|
||||
- S1: Чат (текст сообщения, автор, дата)
|
||||
- S2: OO CRM (существующий контакт/компания/deal) через `oo persons list` / `oo persons get`
|
||||
- S3 (опционально): Corpus SoT (`knowledge-mesh-seed.yaml`) или web-search
|
||||
- Факты с ≥2 источниками → `root=facts` в brain + `approve=true` для CRM
|
||||
- Факты с 1 источником → предложение на review (catalog-паттерн)
|
||||
|
||||
### Apply в OO CRM
|
||||
|
||||
Через `oo` CLI:
|
||||
- `oo persons update <id> --about ... --job-title ...`
|
||||
- `oo contacts info-add <id> --type Phone|LinkedIn --value ...`
|
||||
- `oo opportunities create --contact-id ... --title ... --stage ...`
|
||||
- `oo persons update <id> --company-id ...` (ассоциация)
|
||||
- `oo projects contacts <project-id>` (связь с проектом)
|
||||
|
||||
---
|
||||
|
||||
## 3. Implementation Plan
|
||||
|
||||
### Phase 1: Telegram sync + import (MVP)
|
||||
|
||||
1. Create `bin/chats/` directory structure (nested Go module like `bin/kbsearch/`)
|
||||
2. Implement `Source` interface
|
||||
3. Implement `TelegramSource` — MCP JSON-RPC клиент к telegram-mcp
|
||||
4. Write JSONL output to `var/chats/telegram/<id>/messages.jsonl`
|
||||
5. Implement `chats import` — reads all JSONL, writes MD
|
||||
6. Write system test: sync → import → verify output
|
||||
|
||||
### Phase 2: Brain indexing
|
||||
|
||||
1. Implement `chats index` — rebuild `var/kb.lbug` with corpus + chats
|
||||
2. Write system test: sync → import → index → `bin/kb/search "query"` → verify recall
|
||||
|
||||
### Phase 3: WhatsApp
|
||||
|
||||
1. Implement `WhatsAppSource` using `tulir/whatsmeow`
|
||||
2. First run requires QR auth (save session for later reuse)
|
||||
3. Write sync output to `var/chats/whatsapp/<jid>/messages.jsonl`
|
||||
|
||||
### Phase 4: Facts extraction + cross-check
|
||||
|
||||
1. Implement `chats facts`:
|
||||
- Phone regex: `\+?\d{7,15}` + validation (exclude dates/amounts/cards)
|
||||
- LinkedIn URL regex: `linkedin\.com/in/[\w-]+`
|
||||
- Cross-check: match extracted values against OO CRM API
|
||||
2. Write facts to brain as `root=facts`
|
||||
|
||||
### Phase 5: LinkedIn + Apply
|
||||
|
||||
1. Implement `LinkedInSource` using Voyager API + cookies from `storage-state.json`
|
||||
2. Implement `chats apply`:
|
||||
- Build YAML catalog (as `oo-workspace` does)
|
||||
- Run `oo catalog match` → review → `oo catalog apply`
|
||||
|
||||
---
|
||||
|
||||
## 4. Existing code references
|
||||
|
||||
| Code | Path | Purpose |
|
||||
|------|------|---------|
|
||||
| Mail sync | `bin/mail/sync.go` + `bin/mail/sync/` | Pattern for chat sync (Go subprocess pattern) |
|
||||
| Mail import | `bin/mail/import` | Pattern for JSONL → MD conversion |
|
||||
| Mail index | `bin/mail/index_mail` | Pattern for brain rebuild with new corpus |
|
||||
| kbsearch | `bin/kbsearch/` | Go module pattern (nested, `system_ladybug` tag) |
|
||||
| kb/index | `bin/kb/index` | Python brain index (upsert_leaf, init_schema) |
|
||||
| kblib | `bin/tools/kblib.py` | Python brain core (connect, upsert_leaf, hybrid_search) |
|
||||
| facts/crm | `bin/facts/crm` | Cross-check CRM facts pattern |
|
||||
| facts/extract | `bin/facts/extract` | 2-source fact extraction pattern |
|
||||
| oo-workspace catalog | `/mnt/8TB/projects/eslider/oo-workspace/catalog/` | Catalog scan/match/apply pattern (VCF, Thunderbird, projects) |
|
||||
| WhatsApp bridge | `/mnt/8TB/projects/eslider/mcp-servers/whatsapp-mcp/whatsapp-bridge/` | Existing `whatsmeow` Go bridge |
|
||||
| go-onlyoffice | `/mnt/8TB/projects/eslider/go-onlyoffice/` | OO CRM API library |
|
||||
| oo CLI | `/home/ano/go/bin/oo` | Built go-onlyoffice CLI |
|
||||
|
||||
---
|
||||
|
||||
## 5. Critical files (paths only, no secrets)
|
||||
|
||||
| File | Content |
|
||||
|------|---------|
|
||||
| `~/.config/gitea/produktor.env` | Gitea URL + token |
|
||||
| `/mnt/8TB/projects/eslider/mcp-servers/telegram.env` | Telegram API ID/Hash + phone + session |
|
||||
| `/mnt/8TB/projects/eslider/mcp-servers/telegram-mcp/.env` | Telegram session string |
|
||||
| `/mnt/8TB/projects/ai/2dph/.env` | ONLYOFFICE URL/USER/PASS |
|
||||
| `/mnt/8TB/projects/eslider/cv/.env` | LinkedIn auth token |
|
||||
| `~/.linkedin-mcp/storage-state.json` | LinkedIn active cookies |
|
||||
| `/mnt/8TB/projects/eslider/go-onlyoffice/.env` | OO creds for oo CLI |
|
||||
| `/mnt/8TB/projects/eslider/go-onlyoffice/go.mod` | Deps for go-onlyoffice |
|
||||
| `/mnt/8TB/projects/eslider/oo-workspace/go.mod` | Deps for oo-workspace |
|
||||
|
||||
---
|
||||
|
||||
## 6. Agent prompt (для запуска новой сессии)
|
||||
|
||||
Скопируйте этот промпт при старте новой сессии:
|
||||
|
||||
```
|
||||
Ты — ассистент для разработки chat import pipeline в проекте 2dph (eSlider/2dph).
|
||||
|
||||
Прочитай docs/chat-import-plan.md полностью. Это план, находки и дизайн.
|
||||
|
||||
Задача: реализовать bin/chats — единый Go бинарник для sync/import/index/facts/apply
|
||||
личных чатов из Telegram, WhatsApp, LinkedIn с последующей записью фактов в OnlyOffice CRM.
|
||||
|
||||
Правила:
|
||||
1. TDD first — workflow use-case как тест. Синтетические данные (Alice, Bob).
|
||||
2. Go, всё в один бинарник bin/chats.
|
||||
3. Source interface — каждая платформа плагином.
|
||||
4. var/chats/telegram/ — полные имена платформ.
|
||||
5. Начни с Telegram (session string активна, через MCP server).
|
||||
6. После реализации — создай/обнови тесты.
|
||||
7. Сохрани прогресс в docs/chat-import-plan.md.
|
||||
8. Запроси разрешение перед apply в OO CRM.
|
||||
9. НИКАКИХ реальных имён, телефонов, email в коммитах — только ссылки на env файлы.
|
||||
|
||||
Ключевые пути:
|
||||
- /mnt/8TB/projects/ai/2dph/ — корень проекта
|
||||
- /mnt/8TB/projects/eslider/mcp-servers/telegram-mcp/ — Telegram MCP
|
||||
- /mnt/8TB/projects/eslider/mcp-servers/telegram.env — API ID/Hash + session (не коммитить!)
|
||||
- /mnt/8TB/projects/eslider/go-onlyoffice/ — go-onlyoffice библиотека
|
||||
- /home/ano/go/bin/oo — OO CLI
|
||||
|
||||
Готов начать.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 7. Implementation Progress
|
||||
|
||||
### Session: 2026-08-12 — Phase 1 complete, sync проверен с реальными данными
|
||||
|
||||
**Результаты тестового синка (--limit=100, 31 личный чат):**
|
||||
- Подключение к Telegram MCP server: 1.5с
|
||||
- Получен 31 личный чат (отфильтровано боты + Telegram service)
|
||||
- Синк всех чатов: 7.7с (1 flood wait на 3с, обработан MCP сервером)
|
||||
- Записано 922 сообщения в JSONL
|
||||
- Import: 30 чатов в MD (1 пустой — Saved Messages)
|
||||
- Факты: 14 phone + 2 email после фильтрации
|
||||
|
||||
**Ключевое изменение: `gotd/td` → Telegram MCP Server**
|
||||
- Вместо прямого gotd/td подключения (FLOOD_WAIT из-за 5 уже запущенных MCP серверов)
|
||||
- Используется MCP JSON-RPC (JSONL, `\n`-delimited, **не** Content-Length headers)
|
||||
- Переиспользует существующий Telethon session string (без phone auth)
|
||||
- Стартует транзиентный MCP сервер на время синка, убивает после
|
||||
- Инструменты: `list_chats` + `get_history` через MCP protocol v1.8+
|
||||
|
||||
**Файлы:**
|
||||
|
||||
| Файл | Роль |
|
||||
|------|------|
|
||||
| `bin/chats/mcpclient.go` | MCP JSON-RPC client + `TelegramMCPSource` |
|
||||
| `bin/chats/sync_cmd.go` | CLI: `chats sync telegram --limit N` |
|
||||
| `bin/chats/import_cmd.go` | JSONL → MD с YAML frontmatter |
|
||||
| `bin/chats/index_cmd.go` | Делегирует `bin/kb/index --corpus` |
|
||||
| `bin/chats/facts_cmd.go` | Regex extraction (phone/email/linkedin) с валидацией |
|
||||
| `bin/chats/apply_cmd.go` | OO CRM cross-check + apply через `oo` CLI + --dry-run |
|
||||
| `bin/chat` | build+exec wrapper (как `bin/kb/search`) |
|
||||
|
||||
**Phone regex улучшен:**
|
||||
- Фильтр: исключает даты (`2026-06-14`), суммы (`2 500 000`), номера карт (16 digits), инвойсы (`25-002332001`)
|
||||
- Валидация: >=7 digits, <=15 digits, без `000` pattern, не дата
|
||||
- Результат: 37 → 16 фактов (14 phone + 2 email)
|
||||
|
||||
**Cross-check + Apply в OO CRM (без персональных данных в файлах):**
|
||||
- Найден существующий контакт через first name: добавлен phone + email
|
||||
- Создан новый контакт: phone добавлен
|
||||
- Apply через `oo contacts info-add <id> --type Phone|Email --value ...`
|
||||
|
||||
**Brain:** 132 leafs (120 info + 12 facts), чаты заиндексены
|
||||
|
||||
**Безопасность:**
|
||||
- `var/` в `.gitignore` — сырые чат-данные не коммитятся
|
||||
- credentials в env файлах, не в коде
|
||||
- Тесты используют синтетические данные (Alice, Bob, Charlie, Diana)
|
||||
- Этот документ не содержит реальных значений API ключей, токенов, паролей или персональных данных третьих лиц
|
||||
|
||||
**Что дальше (следующая сессия):**
|
||||
1. WhatsApp source (`tulir/whatsmeow`) — после первого QR
|
||||
2. LinkedIn source (Voyager API через cookies)
|
||||
3. Full history sync (без --limit) для всех чатов
|
||||
4. Telegram session reuse через MCP server (уже работает)
|
||||
5. Apply через `bin/chats apply` (уже работает, улучшен поиск контактов)
|
||||
>>>>>>> 313599b (bin/chats: Phase 1 MVP — Telegram sync/import/index/facts/apply)
|
||||
|
||||
Reference in New Issue
Block a user