package main import ( "bufio" "bytes" "encoding/json" "flag" "fmt" "os" "os/exec" "path/filepath" "regexp" "strings" ) var ( phoneRegex = regexp.MustCompile(`[+\d][\d\s\-()]{6,25}\d`) dateRegex = regexp.MustCompile(`^\d{2,4}[-/]\d{1,2}[-/]\d{2,4}$`) rangeRegex = regexp.MustCompile(`^\d+\s*[-–]\s*\d+$`) linkedinRegex = regexp.MustCompile(`linkedin\.com/in/[\w-]+`) emailRegex = regexp.MustCompile(`[\w.+-]+@[\w-]+\.[\w.-]+`) projectRegex = regexp.MustCompile(`(?i)project\s*[:/]\s*(.+)`) dealRegex = regexp.MustCompile(`(?i)(deal|opportunity)\s*[:/]\s*(.+)`) skillRegex = regexp.MustCompile(`(?i)(works?|worked|working)\s+(at|on|with)\s+([A-Z][\w\s]+)`) ) func isValidPhone(s string) bool { s = strings.TrimSpace(s) s = strings.Trim(s, "+()-\t ") if len(s) < 6 || len(s) > 25 { return false } if dateRegex.MatchString(s) || rangeRegex.MatchString(s) { return false } if strings.ContainsAny(s, "/abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") { return false } if strings.Contains(s, "000") || strings.Contains(s, "500 ") || strings.Contains(s, "000 ") { return false } digits := 0 for _, r := range s { if r >= '0' && r <= '9' { digits++ } } if digits < 7 || digits > 15 { return false } // Card number pattern: 16 digits with possible spaces if digits == 16 { return false } // Date-like: 8 digits starting with 20xx or 19xx if len(s) <= 8 && digits == 8 && (strings.HasPrefix(s, "20") || strings.HasPrefix(s, "19")) { return false } // 11+ digits starting with 2 - unlikely phone if digits >= 11 && strings.HasPrefix(s, "2") && !strings.HasPrefix(s, "+") { return false } // Must start with + or be at least 7 digits if !strings.HasPrefix(s, "+") && digits < 7 { return false } return true } type ExtractedFact struct { ChatID string `json:"chat_id"` ChatName string `json:"chat_name"` Platform string `json:"platform"` FactType string `json:"fact_type"` Value string `json:"value"` Source string `json:"source"` MessageID string `json:"message_id"` } func runFacts(args []string) int { fs := flag.NewFlagSet("chats facts", flag.ContinueOnError) help := fs.Bool("help", false, "") fs.SetOutput(os.Stderr) if err := fs.Parse(args); err != nil { return 2 } if *help { fmt.Fprintln(os.Stderr, "usage: chats facts") return 0 } root := chatsDir() telegramDir := filepath.Join(root, "telegram") entries, err := os.ReadDir(telegramDir) if err != nil { fmt.Fprintf(os.Stderr, "chats facts: read %s: %v\n", telegramDir, err) return 1 } var allFacts []ExtractedFact for _, entry := range entries { if !entry.IsDir() { continue } chatID := entry.Name() jsonlPath := filepath.Join(telegramDir, chatID, "messages.jsonl") info, err := os.Stat(jsonlPath) if err != nil { continue } if info.Size() == 0 { continue } facts, chatName := extractFacts(jsonlPath, chatID) allFacts = append(allFacts, facts...) _ = chatName } if len(allFacts) == 0 { fmt.Println("chats facts: no facts extracted") return 0 } phoneFacts := filterFacts(allFacts, "phone") emailFacts := filterFacts(allFacts, "email") linkedinFacts := filterFacts(allFacts, "linkedin") projectFacts := filterFacts(allFacts, "project") skillFacts := filterFacts(allFacts, "skill") fmt.Printf("chats facts: extracted %d facts (%d phone, %d email, %d linkedin, %d project, %d skill)\n", len(allFacts), len(phoneFacts), len(emailFacts), len(linkedinFacts), len(projectFacts), len(skillFacts)) factsDir := filepath.Join(root, "facts") if err := os.MkdirAll(factsDir, 0755); err != nil { fmt.Fprintf(os.Stderr, "chats facts: mkdir %s: %v\n", factsDir, err) return 1 } factsPath := filepath.Join(factsDir, "chat-facts.json") data, err := json.MarshalIndent(allFacts, "", " ") if err != nil { fmt.Fprintf(os.Stderr, "chats facts: marshal: %v\n", err) return 1 } if err := os.WriteFile(factsPath, data, 0644); err != nil { fmt.Fprintf(os.Stderr, "chats facts: write %s: %v\n", factsPath, err) return 1 } fmt.Printf("chats facts: saved to %s\n", factsPath) writeFactsToBrain(root, allFacts) return 0 } func extractFacts(jsonlPath, chatID string) ([]ExtractedFact, string) { f, err := os.Open(jsonlPath) if err != nil { return nil, "" } defer f.Close() var facts []ExtractedFact chatName := "" scanner := bufio.NewScanner(f) scanner.Buffer(make([]byte, 1<<20), 1<<20) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if line == "" { continue } var msg Message if err := json.Unmarshal([]byte(line), &msg); err != nil { continue } if chatName == "" && msg.From != "" { chatName = msg.From } text := msg.Text phones := phoneRegex.FindAllString(text, -1) for _, p := range phones { p = strings.TrimSpace(p) p = strings.Trim(p, "()- \t") if isValidPhone(p) { facts = append(facts, ExtractedFact{ ChatID: chatID, ChatName: chatName, Platform: "telegram", FactType: "phone", Value: p, Source: "chat:" + msg.ID, MessageID: msg.ID, }) } } emails := emailRegex.FindAllString(text, -1) for _, e := range emails { facts = append(facts, ExtractedFact{ ChatID: chatID, ChatName: chatName, Platform: "telegram", FactType: "email", Value: strings.ToLower(e), Source: "chat:" + msg.ID, MessageID: msg.ID, }) } linkedins := linkedinRegex.FindAllString(text, -1) for _, l := range linkedins { facts = append(facts, ExtractedFact{ ChatID: chatID, ChatName: chatName, Platform: "telegram", FactType: "linkedin", Value: "https://" + l, Source: "chat:" + msg.ID, MessageID: msg.ID, }) } if matches := projectRegex.FindStringSubmatch(text); len(matches) > 1 { facts = append(facts, ExtractedFact{ ChatID: chatID, ChatName: chatName, Platform: "telegram", FactType: "project", Value: strings.TrimSpace(matches[1]), Source: "chat:" + msg.ID, MessageID: msg.ID, }) } if matches := dealRegex.FindStringSubmatch(text); len(matches) > 2 { facts = append(facts, ExtractedFact{ ChatID: chatID, ChatName: chatName, Platform: "telegram", FactType: "deal", Value: strings.TrimSpace(matches[2]), Source: "chat:" + msg.ID, MessageID: msg.ID, }) } if matches := skillRegex.FindStringSubmatch(text); len(matches) > 3 { facts = append(facts, ExtractedFact{ ChatID: chatID, ChatName: chatName, Platform: "telegram", FactType: "skill", Value: strings.TrimSpace(matches[0]), Source: "chat:" + msg.ID, MessageID: msg.ID, }) } } return facts, chatName } func filterFacts(facts []ExtractedFact, factType string) []ExtractedFact { var result []ExtractedFact for _, f := range facts { if f.FactType == factType { result = append(result, f) } } return result } func writeFactsToBrain(root string, facts []ExtractedFact) { indexScript := filepath.Join(root, "bin", "kb", "index") if _, err := os.Stat(indexScript); os.IsNotExist(err) { fmt.Fprintf(os.Stderr, "chats facts: kb/index not found, skipping brain write\n") return } mdDir := filepath.Join(chatsDir(), "facts") if err := os.MkdirAll(mdDir, 0755); err != nil { fmt.Fprintf(os.Stderr, "chats facts: mkdir %s: %v\n", mdDir, err) return } var sb strings.Builder sb.WriteString("---\n") sb.WriteString("root: facts\n") sb.WriteString("---\n\n") sb.WriteString("# Chat-Derived Facts\n\n") for _, f := range facts { sb.WriteString(fmt.Sprintf("- **%s**: %s (source: %s, chat: %s)\n", f.FactType, f.Value, f.Source, f.ChatName)) } sb.WriteString("\n") factsMD := filepath.Join(mdDir, "chat-facts.md") if err := os.WriteFile(factsMD, []byte(sb.String()), 0644); err != nil { fmt.Fprintf(os.Stderr, "chats facts: write %s: %v\n", factsMD, err) return } cmd := exec.Command(indexScript, "--corpus", mdDir, "--skip-indexes") var outBuf, errBuf bytes.Buffer cmd.Stdout = &outBuf cmd.Stderr = &errBuf cmd.Dir = root if err := cmd.Run(); err != nil { fmt.Fprintf(os.Stderr, "chats facts: brain index: %v\n%s", err, errBuf.String()) return } fmt.Printf("chats facts: written to brain (%s)\n", strings.TrimSpace(outBuf.String())) }