bin/chats: Phase 1 MVP — Telegram sync/import/index/facts/apply

- bin/chats/ — nested Go module (как bin/kbsearch/) - sync telegram — MCP JSON-RPC клиент, 31 личный чат, 922 сообщения - import — конвертация JSONL → MD с YAML frontmatter - index — делегирует bin/kb/index --corpus (132 leafs в brain) - facts — regex extraction phone/email/linkedin с валидацией (исключены: даты, суммы, номера карт, инвойсы) - apply — oo CLI cross-check + dry-run - Source interface для будущих WhatsApp/LinkedIn - 4 system tests (import, facts, empty, roundtrip) — синтетические данные - bin/chat — build+exec wrapper - docs/chat-import-plan.md — прогресс, пути к env (без секретов) Безопасность: var/ в gitignore, credentials в env, тесты без реальных данных.
2026-08-12 23:59:29 +01:00
parent a7cb8d4c76
commit 27d9521e7f
13 changed files with 2447 additions and 0 deletions
@@ -0,0 +1,316 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+var (
+	phoneRegex    = regexp.MustCompile(`[+\d][\d\s\-()]{6,25}\d`)
+	dateRegex     = regexp.MustCompile(`^\d{2,4}[-/]\d{1,2}[-/]\d{2,4}$`)
+	rangeRegex    = regexp.MustCompile(`^\d+\s*[-–]\s*\d+$`)
+	linkedinRegex = regexp.MustCompile(`linkedin\.com/in/[\w-]+`)
+	emailRegex    = regexp.MustCompile(`[\w.+-]+@[\w-]+\.[\w.-]+`)
+	projectRegex  = regexp.MustCompile(`(?i)project\s*[:/]\s*(.+)`)
+	dealRegex     = regexp.MustCompile(`(?i)(deal|opportunity)\s*[:/]\s*(.+)`)
+	skillRegex    = regexp.MustCompile(`(?i)(works?|worked|working)\s+(at|on|with)\s+([A-Z][\w\s]+)`)
+)
+
+func isValidPhone(s string) bool {
+	s = strings.TrimSpace(s)
+	s = strings.Trim(s, "+()-\t ")
+	if len(s) < 6 || len(s) > 25 {
+		return false
+	}
+	if dateRegex.MatchString(s) || rangeRegex.MatchString(s) {
+		return false
+	}
+	if strings.ContainsAny(s, "/abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") {
+		return false
+	}
+	if strings.Contains(s, "000") || strings.Contains(s, "500 ") || strings.Contains(s, "000 ") {
+		return false
+	}
+	digits := 0
+	for _, r := range s {
+		if r >= '0' && r <= '9' {
+			digits++
+		}
+	}
+	if digits < 7 || digits > 15 {
+		return false
+	}
+	// Card number pattern: 16 digits with possible spaces
+	if digits == 16 {
+		return false
+	}
+	// Date-like: 8 digits starting with 20xx or 19xx
+	if len(s) <= 8 && digits == 8 && (strings.HasPrefix(s, "20") || strings.HasPrefix(s, "19")) {
+		return false
+	}
+	// 11+ digits starting with 2 - unlikely phone
+	if digits >= 11 && strings.HasPrefix(s, "2") && !strings.HasPrefix(s, "+") {
+		return false
+	}
+	// Must start with + or be at least 7 digits
+	if !strings.HasPrefix(s, "+") && digits < 7 {
+		return false
+	}
+	return true
+}
+
+type ExtractedFact struct {
+	ChatID    string `json:"chat_id"`
+	ChatName  string `json:"chat_name"`
+	Platform  string `json:"platform"`
+	FactType  string `json:"fact_type"`
+	Value     string `json:"value"`
+	Source    string `json:"source"`
+	MessageID string `json:"message_id"`
+}
+
+func runFacts(args []string) int {
+	fs := flag.NewFlagSet("chats facts", flag.ContinueOnError)
+	help := fs.Bool("help", false, "")
+	fs.SetOutput(os.Stderr)
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if *help {
+		fmt.Fprintln(os.Stderr, "usage: chats facts")
+		return 0
+	}
+
+	root := chatsDir()
+	telegramDir := filepath.Join(root, "telegram")
+
+	entries, err := os.ReadDir(telegramDir)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "chats facts: read %s: %v\n", telegramDir, err)
+		return 1
+	}
+
+	var allFacts []ExtractedFact
+	for _, entry := range entries {
+		if !entry.IsDir() {
+			continue
+		}
+		chatID := entry.Name()
+		jsonlPath := filepath.Join(telegramDir, chatID, "messages.jsonl")
+		info, err := os.Stat(jsonlPath)
+		if err != nil {
+			continue
+		}
+		if info.Size() == 0 {
+			continue
+		}
+
+		facts, chatName := extractFacts(jsonlPath, chatID)
+		allFacts = append(allFacts, facts...)
+		_ = chatName
+	}
+
+	if len(allFacts) == 0 {
+		fmt.Println("chats facts: no facts extracted")
+		return 0
+	}
+
+	phoneFacts := filterFacts(allFacts, "phone")
+	emailFacts := filterFacts(allFacts, "email")
+	linkedinFacts := filterFacts(allFacts, "linkedin")
+	projectFacts := filterFacts(allFacts, "project")
+	skillFacts := filterFacts(allFacts, "skill")
+
+	fmt.Printf("chats facts: extracted %d facts (%d phone, %d email, %d linkedin, %d project, %d skill)\n",
+		len(allFacts), len(phoneFacts), len(emailFacts), len(linkedinFacts), len(projectFacts), len(skillFacts))
+
+	factsDir := filepath.Join(root, "facts")
+	if err := os.MkdirAll(factsDir, 0755); err != nil {
+		fmt.Fprintf(os.Stderr, "chats facts: mkdir %s: %v\n", factsDir, err)
+		return 1
+	}
+	factsPath := filepath.Join(factsDir, "chat-facts.json")
+	data, err := json.MarshalIndent(allFacts, "", "  ")
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "chats facts: marshal: %v\n", err)
+		return 1
+	}
+	if err := os.WriteFile(factsPath, data, 0644); err != nil {
+		fmt.Fprintf(os.Stderr, "chats facts: write %s: %v\n", factsPath, err)
+		return 1
+	}
+	fmt.Printf("chats facts: saved to %s\n", factsPath)
+
+	writeFactsToBrain(root, allFacts)
+
+	return 0
+}
+
+func extractFacts(jsonlPath, chatID string) ([]ExtractedFact, string) {
+	f, err := os.Open(jsonlPath)
+	if err != nil {
+		return nil, ""
+	}
+	defer f.Close()
+
+	var facts []ExtractedFact
+	chatName := ""
+
+	scanner := bufio.NewScanner(f)
+	scanner.Buffer(make([]byte, 1<<20), 1<<20)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" {
+			continue
+		}
+		var msg Message
+		if err := json.Unmarshal([]byte(line), &msg); err != nil {
+			continue
+		}
+		if chatName == "" && msg.From != "" {
+			chatName = msg.From
+		}
+
+		text := msg.Text
+
+		phones := phoneRegex.FindAllString(text, -1)
+		for _, p := range phones {
+			p = strings.TrimSpace(p)
+			p = strings.Trim(p, "()- \t")
+			if isValidPhone(p) {
+				facts = append(facts, ExtractedFact{
+					ChatID:    chatID,
+					ChatName:  chatName,
+					Platform:  "telegram",
+					FactType:  "phone",
+					Value:     p,
+					Source:    "chat:" + msg.ID,
+					MessageID: msg.ID,
+				})
+			}
+		}
+
+		emails := emailRegex.FindAllString(text, -1)
+		for _, e := range emails {
+			facts = append(facts, ExtractedFact{
+				ChatID:    chatID,
+				ChatName:  chatName,
+				Platform:  "telegram",
+				FactType:  "email",
+				Value:     strings.ToLower(e),
+				Source:    "chat:" + msg.ID,
+				MessageID: msg.ID,
+			})
+		}
+
+		linkedins := linkedinRegex.FindAllString(text, -1)
+		for _, l := range linkedins {
+			facts = append(facts, ExtractedFact{
+				ChatID:    chatID,
+				ChatName:  chatName,
+				Platform:  "telegram",
+				FactType:  "linkedin",
+				Value:     "https://" + l,
+				Source:    "chat:" + msg.ID,
+				MessageID: msg.ID,
+			})
+		}
+
+		if matches := projectRegex.FindStringSubmatch(text); len(matches) > 1 {
+			facts = append(facts, ExtractedFact{
+				ChatID:    chatID,
+				ChatName:  chatName,
+				Platform:  "telegram",
+				FactType:  "project",
+				Value:     strings.TrimSpace(matches[1]),
+				Source:    "chat:" + msg.ID,
+				MessageID: msg.ID,
+			})
+		}
+
+		if matches := dealRegex.FindStringSubmatch(text); len(matches) > 2 {
+			facts = append(facts, ExtractedFact{
+				ChatID:    chatID,
+				ChatName:  chatName,
+				Platform:  "telegram",
+				FactType:  "deal",
+				Value:     strings.TrimSpace(matches[2]),
+				Source:    "chat:" + msg.ID,
+				MessageID: msg.ID,
+			})
+		}
+
+		if matches := skillRegex.FindStringSubmatch(text); len(matches) > 3 {
+			facts = append(facts, ExtractedFact{
+				ChatID:    chatID,
+				ChatName:  chatName,
+				Platform:  "telegram",
+				FactType:  "skill",
+				Value:     strings.TrimSpace(matches[0]),
+				Source:    "chat:" + msg.ID,
+				MessageID: msg.ID,
+			})
+		}
+	}
+	return facts, chatName
+}
+
+func filterFacts(facts []ExtractedFact, factType string) []ExtractedFact {
+	var result []ExtractedFact
+	for _, f := range facts {
+		if f.FactType == factType {
+			result = append(result, f)
+		}
+	}
+	return result
+}
+
+func writeFactsToBrain(root string, facts []ExtractedFact) {
+	indexScript := filepath.Join(root, "bin", "kb", "index")
+	if _, err := os.Stat(indexScript); os.IsNotExist(err) {
+		fmt.Fprintf(os.Stderr, "chats facts: kb/index not found, skipping brain write\n")
+		return
+	}
+
+	mdDir := filepath.Join(chatsDir(), "facts")
+	if err := os.MkdirAll(mdDir, 0755); err != nil {
+		fmt.Fprintf(os.Stderr, "chats facts: mkdir %s: %v\n", mdDir, err)
+		return
+	}
+
+	var sb strings.Builder
+	sb.WriteString("---\n")
+	sb.WriteString("root: facts\n")
+	sb.WriteString("---\n\n")
+	sb.WriteString("# Chat-Derived Facts\n\n")
+	for _, f := range facts {
+		sb.WriteString(fmt.Sprintf("- **%s**: %s (source: %s, chat: %s)\n",
+			f.FactType, f.Value, f.Source, f.ChatName))
+	}
+	sb.WriteString("\n")
+
+	factsMD := filepath.Join(mdDir, "chat-facts.md")
+	if err := os.WriteFile(factsMD, []byte(sb.String()), 0644); err != nil {
+		fmt.Fprintf(os.Stderr, "chats facts: write %s: %v\n", factsMD, err)
+		return
+	}
+
+	cmd := exec.Command(indexScript, "--corpus", mdDir, "--skip-indexes")
+	var outBuf, errBuf bytes.Buffer
+	cmd.Stdout = &outBuf
+	cmd.Stderr = &errBuf
+	cmd.Dir = root
+	if err := cmd.Run(); err != nil {
+		fmt.Fprintf(os.Stderr, "chats facts: brain index: %v\n%s", err, errBuf.String())
+		return
+	}
+	fmt.Printf("chats facts: written to brain (%s)\n", strings.TrimSpace(outBuf.String()))
+}