eSlider/2dph
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: chats method shebangs; drop chats index (D14). (#11)
Tests / Test (push) Failing after 5s
Details
Tests / Release (semver) (push) Skipped
Details

Browse Source 
Parsers and commands live in internal/chats. bin/chats/{sync,import,facts,apply}.go
are tagged shebang mains. Brain ingest is not a chats command.
This commit is contained in:
eSlider
2026-08-13 17:31:03 +01:00
committed by GitHub
co-authored by GitHub
parent 1c7db6d499
commit 5d4b3427a4
25 changed files with 212 additions and 223 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/chats/apply.go
Executable
+19
View File
@@ -0,0 +1,19 @@
//usr/bin/env go run -tags=chats_apply "$0" "$@"; exit
//go:build chats_apply
//
// bin/chats/apply.go - push extracted chat facts to OnlyOffice CRM.
//
// ./bin/chats/apply.go [--dry-run]
//
// NOTE: never run `gofmt -w` on this file — it breaks the shebang.
package main
import (
"os"
"github.com/eSlider/2dph/internal/chats"
)
func main() {
os.Exit(chats.RunApply(os.Args[1:]))
}
bin/chats/apply_cmd.go
-317
View File
@@ -1,317 +0,0 @@
package main
import (
"bytes"
"encoding/json"
"flag"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
)
type ooContact struct {
ID int `json:"id"`
DisplayName string `json:"displayName"`
FirstName string `json:"firstName"`
LastName string `json:"lastName"`
About string `json:"about"`
CommonData []struct {
InfoType int `json:"infoType"`
Data string `json:"data"`
Category string `json:"categoryName"`
} `json:"commonData"`
}
func runApply(args []string) int {
fs := flag.NewFlagSet("chats apply", flag.ContinueOnError)
dryRun := fs.Bool("dry-run", false, "show what would be done without writing")
help := fs.Bool("help", false, "")
fs.SetOutput(os.Stderr)
if err := fs.Parse(args); err != nil {
return 2
}
if *help {
fmt.Fprintln(os.Stderr, "usage: chats apply [--dry-run]")
return 0
}
ooCLI := findOO()
if ooCLI == "" {
fmt.Fprintln(os.Stderr, "chats apply: oo CLI not found; set OO_CLI or install go-onlyoffice")
return 1
}
facts, err := loadFacts()
if err != nil {
fmt.Fprintf(os.Stderr, "chats apply: %v\n", err)
return 1
}
if len(facts) == 0 {
fmt.Println("chats apply: no facts to process")
return 0
}
phoneFacts := filterFacts(facts, "phone")
emailFacts := filterFacts(facts, "email")
phoneFacts = dedupeFacts(phoneFacts)
emailFacts = dedupeFacts(emailFacts)
type resolvedFact struct {
Fact ExtractedFact
OoID int
OoName string
Action string // "info-add" or "persons-create"
}
var resolved []resolvedFact
for _, f := range phoneFacts {
contact, err := searchContact(ooCLI, f.ChatName)
if err != nil || contact == nil {
fmt.Printf(" ✗ %s: phone %s — not found in CRM\n", f.ChatName, f.Value)
resolved = append(resolved, resolvedFact{Fact: f, Action: "persons-create"})
continue
}
hasPhone := false
for _, d := range contact.CommonData {
if d.InfoType == 2 {
hasPhone = true
break
}
}
if hasPhone {
fmt.Printf(" ✓ %s (ID %d): phone %s — already has phone, skip\n", contact.DisplayName, contact.ID, f.Value)
continue
}
fmt.Printf(" → %s (ID %d): add phone %s\n", contact.DisplayName, contact.ID, f.Value)
resolved = append(resolved, resolvedFact{
Fact: f, OoID: contact.ID, OoName: contact.DisplayName, Action: "info-add",
})
}
for _, f := range emailFacts {
if strings.EqualFold(f.Value, envVar("ONLYOFFICE_USER", "")) ||
strings.EqualFold(f.Value, envVar("OO_USER", "")) ||
strings.EqualFold(f.Value, os.Getenv("EMAIL")) {
continue
}
contact, err := searchContact(ooCLI, f.ChatName)
if err != nil || contact == nil {
fmt.Printf(" ✗ %s: email %s — not found in CRM\n", f.ChatName, f.Value)
resolved = append(resolved, resolvedFact{Fact: f, Action: "persons-create"})
continue
}
hasEmail := false
for _, d := range contact.CommonData {
if d.InfoType == 1 && d.Data == f.Value {
hasEmail = true
break
}
}
if hasEmail {
fmt.Printf(" ✓ %s (ID %d): email %s — already exists\n", contact.DisplayName, contact.ID, f.Value)
continue
}
fmt.Printf(" → %s (ID %d): add email %s\n", contact.DisplayName, contact.ID, f.Value)
resolved = append(resolved, resolvedFact{
Fact: f, OoID: contact.ID, OoName: contact.DisplayName, Action: "info-add",
})
}
if len(resolved) == 0 {
fmt.Println("chats apply: nothing to apply")
return 0
}
fmt.Printf("\nchats apply: %d actions to apply\n", len(resolved))
if *dryRun {
for _, r := range resolved {
switch r.Action {
case "info-add":
infoType := "Phone"
if r.Fact.FactType == "email" {
infoType = "Email"
}
fmt.Printf(" [dry-run] oo contacts info-add %d --type %s --value %s\n",
r.OoID, infoType, r.Fact.Value)
case "persons-create":
fmt.Printf(" [dry-run] oo persons create --first %q --about %q\n",
r.Fact.ChatName, "Contact from Telegram chat")
}
}
return 0
}
success := 0
failed := 0
for _, r := range resolved {
switch r.Action {
case "info-add":
infoType := "Phone"
if r.Fact.FactType == "email" {
infoType = "Email"
}
if err := ooInfoAdd(ooCLI, r.OoID, infoType, r.Fact.Value); err != nil {
fmt.Fprintf(os.Stderr, " ✗ info-add %s: %v\n", r.Fact.Value, err)
failed++
} else {
fmt.Printf(" ✓ %s → %s (ID %d)\n", r.Fact.Value, r.OoName, r.OoID)
success++
}
case "persons-create":
fmt.Printf(" - create %s (skipped — needs review)\n", r.Fact.ChatName)
success++
}
}
fmt.Printf("\nchats apply: %d succeeded, %d failed\n", success, failed)
if failed > 0 {
return 1
}
return 0
}
func loadFacts() ([]ExtractedFact, error) {
factsPath := filepath.Join(chatsDir(), "facts", "chat-facts.json")
data, err := os.ReadFile(factsPath)
if err != nil {
if os.IsNotExist(err) {
return nil, fmt.Errorf("no facts at %s; run 'chats facts' first", factsPath)
}
return nil, fmt.Errorf("read facts: %w", err)
}
var facts []ExtractedFact
if err := json.Unmarshal(data, &facts); err != nil {
return nil, fmt.Errorf("parse facts: %w", err)
}
return facts, nil
}
func dedupeFacts(facts []ExtractedFact) []ExtractedFact {
seen := make(map[string]bool)
var result []ExtractedFact
for _, f := range facts {
norm := normalizePhone(f.Value)
key := f.ChatName + ":" + factTypeKey(f.FactType) + ":" + norm
if seen[key] {
continue
}
seen[key] = true
f.Value = norm
result = append(result, f)
}
return result
}
func normalizePhone(s string) string {
var digits []rune
for _, r := range s {
if r >= '0' && r <= '9' {
digits = append(digits, r)
}
}
if len(digits) > 0 {
return string(digits)
}
return s
}
func factTypeKey(t string) string {
switch t {
case "phone":
return "p"
case "email":
return "e"
default:
return t
}
}
func findOO() string {
if v := os.Getenv("OO_CLI"); v != "" {
if _, err := os.Stat(v); err == nil {
return v
}
}
candidates := []string{
filepath.Join(os.Getenv("HOME"), "go", "bin", "oo"),
}
for _, c := range candidates {
if _, err := os.Stat(c); err == nil {
return c
}
}
return ""
}
func searchContact(ooCLI, name string) (*ooContact, error) {
query := name
// Try full name first
if c, _ := searchByQuery(ooCLI, query); c != nil {
return c, nil
}
// Try first word
firstWord := strings.Fields(name)[0]
if firstWord != name {
if c, _ := searchByQuery(ooCLI, firstWord); c != nil {
return c, nil
}
}
return nil, nil
}
func searchByQuery(ooCLI, query string) (*ooContact, error) {
cmd := exec.Command(ooCLI, "persons", "list", "--search", query, "-o", "json")
var outBuf, errBuf bytes.Buffer
cmd.Stdout = &outBuf
cmd.Stderr = &errBuf
cmd.Env = os.Environ()
if err := cmd.Run(); err != nil {
return nil, fmt.Errorf("oo persons list: %w\n%s", err, errBuf.String())
}
var contacts []ooContact
if err := json.Unmarshal(outBuf.Bytes(), &contacts); err != nil {
return nil, nil
}
for _, c := range contacts {
lower := strings.ToLower(c.DisplayName)
lowerQuery := strings.ToLower(query)
if strings.EqualFold(c.DisplayName, query) ||
strings.Contains(lower, lowerQuery) ||
strings.Contains(lowerQuery, strings.ToLower(c.FirstName)) {
return &c, nil
}
for _, d := range c.CommonData {
if d.InfoType == 1 && strings.Contains(strings.ToLower(d.Data), lowerQuery) {
return &c, nil
}
}
}
if len(contacts) > 0 {
return &contacts[0], nil
}
return nil, nil
}
func ooInfoAdd(ooCLI string, contactID int, infoType, value string) error {
cmd := exec.Command(ooCLI, "contacts", "info-add",
fmt.Sprintf("%d", contactID),
"--type", infoType,
"--value", value,
"--category", "Work",
"-o", "json",
)
var outBuf, errBuf bytes.Buffer
cmd.Stdout = &outBuf
cmd.Stderr = &errBuf
cmd.Env = os.Environ()
if err := cmd.Run(); err != nil {
return fmt.Errorf("info-add: %w\n%s", err, errBuf.String())
}
return nil
}
bin/chats/chats_test.go
-202
View File
@@ -1,202 +0,0 @@
// System tests for bin/chats.
//
// These are integration tests using real data and real Telegram API (when
// credentials are available). They follow the TDD workflow pattern:
// sync → import → facts → verify.
package main
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
)
// TestChatsImport validates JSONL → MD conversion with a synthetic fixture.
func TestChatsImport(t *testing.T) {
dir := t.TempDir()
root := filepath.Join(dir, "var", "chats")
chatDir := filepath.Join(root, "telegram", "test_user_123")
if err := os.MkdirAll(chatDir, 0755); err != nil {
t.Fatal(err)
}
jsonlPath := filepath.Join(chatDir, "messages.jsonl")
f, err := os.Create(jsonlPath)
if err != nil {
t.Fatal(err)
}
defer f.Close()
enc := json.NewEncoder(f)
messages := []Message{
{ID: "tg_1", Timestamp: "2026-01-15T10:30:00Z", From: "Alice", Text: "Hello!", Platform: "telegram"},
{ID: "tg_2", Timestamp: "2026-01-15T10:31:00Z", From: "Bob", Text: "Hi Alice, my phone is +34 612 345 678", Platform: "telegram"},
{ID: "tg_3", Timestamp: "2026-01-15T10:32:00Z", From: "Alice", Text: "Check my LinkedIn: https://linkedin.com/in/alice-test", Platform: "telegram"},
{ID: "tg_4", Timestamp: "2026-01-15T10:33:00Z", From: "Bob", Text: "My email is bob@example.com, working on Project X", Platform: "telegram"},
}
for _, m := range messages {
if err := enc.Encode(m); err != nil {
t.Fatal(err)
}
}
f.Close()
cwd, _ := os.Getwd()
os.Chdir(dir)
t.Cleanup(func() { os.Chdir(cwd) })
t.Setenv("KB_ROOT", dir)
exitCode := runImport([]string{})
if exitCode != 0 {
t.Fatalf("import exit code %d", exitCode)
}
mdGlob := filepath.Join(root, "md", "telegram", "*", "messages.md")
matches, err := filepath.Glob(mdGlob)
if err != nil {
t.Fatal(err)
}
if len(matches) == 0 {
t.Fatal("no markdown files created by import")
}
mdData, err := os.ReadFile(matches[0])
if err != nil {
t.Fatal(err)
}
content := string(mdData)
if !strings.Contains(content, "Alice") {
t.Error("markdown missing sender name 'Alice'")
}
if !strings.Contains(content, "2026-01-15") {
t.Error("markdown missing date")
}
if !strings.Contains(content, "---") {
t.Error("markdown missing YAML frontmatter")
}
}
// TestChatsFacts validates fact extraction from JSONL fixture.
func TestChatsFacts(t *testing.T) {
dir := t.TempDir()
root := filepath.Join(dir, "var", "chats")
chatDir := filepath.Join(root, "telegram", "test_user_facts")
if err := os.MkdirAll(chatDir, 0755); err != nil {
t.Fatal(err)
}
jsonlPath := filepath.Join(chatDir, "messages.jsonl")
f, err := os.Create(jsonlPath)
if err != nil {
t.Fatal(err)
}
defer f.Close()
enc := json.NewEncoder(f)
messages := []Message{
{ID: "tg_10", Timestamp: "2026-06-01T12:00:00Z", From: "Charlie", Text: "Call me at +1 555 123 4567", Platform: "telegram"},
{ID: "tg_11", Timestamp: "2026-06-01T12:01:00Z", From: "Charlie", Text: "My LinkedIn is linkedin.com/in/charlie-dev", Platform: "telegram"},
{ID: "tg_12", Timestamp: "2026-06-01T12:02:00Z", From: "Charlie", Text: "Email: charlie@dev.com", Platform: "telegram"},
{ID: "tg_13", Timestamp: "2026-06-01T12:03:00Z", From: "Charlie", Text: "I work at Acme Corp on Project Mercury", Platform: "telegram"},
}
for _, m := range messages {
if err := enc.Encode(m); err != nil {
t.Fatal(err)
}
}
f.Close()
facts, _ := extractFacts(jsonlPath, "test_user_facts")
if len(facts) == 0 {
t.Fatal("expected facts, got none")
}
types := make(map[string]int)
for _, f := range facts {
types[f.FactType]++
}
if types["phone"] < 1 {
t.Errorf("expected >=1 phone fact, got %d", types["phone"])
}
if types["email"] < 1 {
t.Errorf("expected >=1 email fact, got %d", types["email"])
}
if types["linkedin"] < 1 {
t.Errorf("expected >=1 linkedin fact, got %d", types["linkedin"])
}
if types["skill"] < 1 {
t.Errorf("expected >=1 skill fact, got %d", types["skill"])
}
}
// TestChatsImportEmptyDir tests that import handles no JSONL gracefully.
func TestChatsImportEmpty(t *testing.T) {
dir := t.TempDir()
cwd, _ := os.Getwd()
os.Chdir(dir)
t.Cleanup(func() { os.Chdir(cwd) })
t.Setenv("KB_ROOT", dir)
exitCode := runImport([]string{})
if exitCode == 0 {
t.Fatal("expected non-zero exit for empty data dir")
}
}
// TestChatsRoundTrip creates a synthetic JSONL, imports it, then verifies
// the markdown structure is parseable and contains YAML frontmatter.
func TestChatsRoundTrip(t *testing.T) {
dir := t.TempDir()
root := filepath.Join(dir, "var", "chats")
chatDir := filepath.Join(root, "telegram", "rt_user")
if err := os.MkdirAll(chatDir, 0755); err != nil {
t.Fatal(err)
}
jsonlPath := filepath.Join(chatDir, "messages.jsonl")
f, err := os.Create(jsonlPath)
if err != nil {
t.Fatal(err)
}
enc := json.NewEncoder(f)
enc.Encode(Message{ID: "tg_100", Timestamp: "2026-07-01T08:00:00Z", From: "Diana", Text: "Hey", Platform: "telegram"})
enc.Encode(Message{ID: "tg_101", Timestamp: "2026-07-01T08:01:00Z", From: "Diana", Text: "How are you?", Platform: "telegram"})
f.Close()
cwd, _ := os.Getwd()
os.Chdir(dir)
t.Cleanup(func() { os.Chdir(cwd) })
t.Setenv("KB_ROOT", dir)
if code := runImport([]string{}); code != 0 {
t.Fatalf("import exit %d", code)
}
mdGlob := filepath.Join(root, "md", "telegram", "*", "messages.md")
matches, _ := filepath.Glob(mdGlob)
if len(matches) == 0 {
t.Fatal("no markdown produced")
}
data, err := os.ReadFile(matches[0])
if err != nil {
t.Fatal(err)
}
content := string(data)
if !strings.HasPrefix(content, "---") {
t.Error("markdown should start with YAML frontmatter delimiter")
}
if !strings.Contains(content, "platform: telegram") {
t.Error("markdown should contain platform field")
}
if !strings.Contains(content, "message_count: 2") {
t.Error("markdown should contain correct message count")
}
if !strings.Contains(content, "Diana") {
t.Error("markdown should contain participants")
}
}
bin/chats/doc.go
+4
View File
@@ -0,0 +1,4 @@
// Commands in this directory are shebang mains (sync.go, import.go, facts.go,
// apply.go), each behind an exclusive build tag so `go build ./bin/chats`
// does not see two mains. Shared code lives in internal/chats.
package main
bin/chats/facts.go
Executable
+20
View File
@@ -0,0 +1,20 @@
//usr/bin/env go run -tags=chats_facts "$0" "$@"; exit
//go:build chats_facts
//
// bin/chats/facts.go - extract phone/email/linkedin facts from JSONL.
//
// ./bin/chats/facts.go
//
// Writes var/chats/facts/. Does not index the brain.
// NOTE: never run `gofmt -w` on this file — it breaks the shebang.
package main
import (
"os"
"github.com/eSlider/2dph/internal/chats"
)
func main() {
os.Exit(chats.RunFacts(os.Args[1:]))
}
bin/chats/facts_cmd.go
-316
View File
@@ -1,316 +0,0 @@
package main
import (
"bufio"
"bytes"
"encoding/json"
"flag"
"fmt"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
)
var (
phoneRegex = regexp.MustCompile(`[+\d][\d\s\-()]{6,25}\d`)
dateRegex = regexp.MustCompile(`^\d{2,4}[-/]\d{1,2}[-/]\d{2,4}$`)
rangeRegex = regexp.MustCompile(`^\d+\s*[-–]\s*\d+$`)
linkedinRegex = regexp.MustCompile(`linkedin\.com/in/[\w-]+`)
emailRegex = regexp.MustCompile(`[\w.+-]+@[\w-]+\.[\w.-]+`)
projectRegex = regexp.MustCompile(`(?i)project\s*[:/]\s*(.+)`)
dealRegex = regexp.MustCompile(`(?i)(deal|opportunity)\s*[:/]\s*(.+)`)
skillRegex = regexp.MustCompile(`(?i)(works?|worked|working)\s+(at|on|with)\s+([A-Z][\w\s]+)`)
)
func isValidPhone(s string) bool {
s = strings.TrimSpace(s)
s = strings.Trim(s, "+()-\t ")
if len(s) < 6 || len(s) > 25 {
return false
}
if dateRegex.MatchString(s) || rangeRegex.MatchString(s) {
return false
}
if strings.ContainsAny(s, "/abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") {
return false
}
if strings.Contains(s, "000") || strings.Contains(s, "500 ") || strings.Contains(s, "000 ") {
return false
}
digits := 0
for _, r := range s {
if r >= '0' && r <= '9' {
digits++
}
}
if digits < 7 || digits > 15 {
return false
}
// Card number pattern: 16 digits with possible spaces
if digits == 16 {
return false
}
// Date-like: 8 digits starting with 20xx or 19xx
if len(s) <= 8 && digits == 8 && (strings.HasPrefix(s, "20") || strings.HasPrefix(s, "19")) {
return false
}
// 11+ digits starting with 2 - unlikely phone
if digits >= 11 && strings.HasPrefix(s, "2") && !strings.HasPrefix(s, "+") {
return false
}
// Must start with + or be at least 7 digits
if !strings.HasPrefix(s, "+") && digits < 7 {
return false
}
return true
}
type ExtractedFact struct {
ChatID string `json:"chat_id"`
ChatName string `json:"chat_name"`
Platform string `json:"platform"`
FactType string `json:"fact_type"`
Value string `json:"value"`
Source string `json:"source"`
MessageID string `json:"message_id"`
}
func runFacts(args []string) int {
fs := flag.NewFlagSet("chats facts", flag.ContinueOnError)
help := fs.Bool("help", false, "")
fs.SetOutput(os.Stderr)
if err := fs.Parse(args); err != nil {
return 2
}
if *help {
fmt.Fprintln(os.Stderr, "usage: chats facts")
return 0
}
root := chatsDir()
telegramDir := filepath.Join(root, "telegram")
entries, err := os.ReadDir(telegramDir)
if err != nil {
fmt.Fprintf(os.Stderr, "chats facts: read %s: %v\n", telegramDir, err)
return 1
}
var allFacts []ExtractedFact
for _, entry := range entries {
if !entry.IsDir() {
continue
}
chatID := entry.Name()
jsonlPath := filepath.Join(telegramDir, chatID, "messages.jsonl")
info, err := os.Stat(jsonlPath)
if err != nil {
continue
}
if info.Size() == 0 {
continue
}
facts, chatName := extractFacts(jsonlPath, chatID)
allFacts = append(allFacts, facts...)
_ = chatName
}
if len(allFacts) == 0 {
fmt.Println("chats facts: no facts extracted")
return 0
}
phoneFacts := filterFacts(allFacts, "phone")
emailFacts := filterFacts(allFacts, "email")
linkedinFacts := filterFacts(allFacts, "linkedin")
projectFacts := filterFacts(allFacts, "project")
skillFacts := filterFacts(allFacts, "skill")
fmt.Printf("chats facts: extracted %d facts (%d phone, %d email, %d linkedin, %d project, %d skill)\n",
len(allFacts), len(phoneFacts), len(emailFacts), len(linkedinFacts), len(projectFacts), len(skillFacts))
factsDir := filepath.Join(root, "facts")
if err := os.MkdirAll(factsDir, 0755); err != nil {
fmt.Fprintf(os.Stderr, "chats facts: mkdir %s: %v\n", factsDir, err)
return 1
}
factsPath := filepath.Join(factsDir, "chat-facts.json")
data, err := json.MarshalIndent(allFacts, "", " ")
if err != nil {
fmt.Fprintf(os.Stderr, "chats facts: marshal: %v\n", err)
return 1
}
if err := os.WriteFile(factsPath, data, 0644); err != nil {
fmt.Fprintf(os.Stderr, "chats facts: write %s: %v\n", factsPath, err)
return 1
}
fmt.Printf("chats facts: saved to %s\n", factsPath)
writeFactsToBrain(root, allFacts)
return 0
}
func extractFacts(jsonlPath, chatID string) ([]ExtractedFact, string) {
f, err := os.Open(jsonlPath)
if err != nil {
return nil, ""
}
defer f.Close()
var facts []ExtractedFact
chatName := ""
scanner := bufio.NewScanner(f)
scanner.Buffer(make([]byte, 1<<20), 1<<20)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
var msg Message
if err := json.Unmarshal([]byte(line), &msg); err != nil {
continue
}
if chatName == "" && msg.From != "" {
chatName = msg.From
}
text := msg.Text
phones := phoneRegex.FindAllString(text, -1)
for _, p := range phones {
p = strings.TrimSpace(p)
p = strings.Trim(p, "()- \t")
if isValidPhone(p) {
facts = append(facts, ExtractedFact{
ChatID: chatID,
ChatName: chatName,
Platform: "telegram",
FactType: "phone",
Value: p,
Source: "chat:" + msg.ID,
MessageID: msg.ID,
})
}
}
emails := emailRegex.FindAllString(text, -1)
for _, e := range emails {
facts = append(facts, ExtractedFact{
ChatID: chatID,
ChatName: chatName,
Platform: "telegram",
FactType: "email",
Value: strings.ToLower(e),
Source: "chat:" + msg.ID,
MessageID: msg.ID,
})
}
linkedins := linkedinRegex.FindAllString(text, -1)
for _, l := range linkedins {
facts = append(facts, ExtractedFact{
ChatID: chatID,
ChatName: chatName,
Platform: "telegram",
FactType: "linkedin",
Value: "https://" + l,
Source: "chat:" + msg.ID,
MessageID: msg.ID,
})
}
if matches := projectRegex.FindStringSubmatch(text); len(matches) > 1 {
facts = append(facts, ExtractedFact{
ChatID: chatID,
ChatName: chatName,
Platform: "telegram",
FactType: "project",
Value: strings.TrimSpace(matches[1]),
Source: "chat:" + msg.ID,
MessageID: msg.ID,
})
}
if matches := dealRegex.FindStringSubmatch(text); len(matches) > 2 {
facts = append(facts, ExtractedFact{
ChatID: chatID,
ChatName: chatName,
Platform: "telegram",
FactType: "deal",
Value: strings.TrimSpace(matches[2]),
Source: "chat:" + msg.ID,
MessageID: msg.ID,
})
}
if matches := skillRegex.FindStringSubmatch(text); len(matches) > 3 {
facts = append(facts, ExtractedFact{
ChatID: chatID,
ChatName: chatName,
Platform: "telegram",
FactType: "skill",
Value: strings.TrimSpace(matches[0]),
Source: "chat:" + msg.ID,
MessageID: msg.ID,
})
}
}
return facts, chatName
}
func filterFacts(facts []ExtractedFact, factType string) []ExtractedFact {
var result []ExtractedFact
for _, f := range facts {
if f.FactType == factType {
result = append(result, f)
}
}
return result
}
func writeFactsToBrain(root string, facts []ExtractedFact) {
indexScript := filepath.Join(root, "bin", "kb", "index")
if _, err := os.Stat(indexScript); os.IsNotExist(err) {
fmt.Fprintf(os.Stderr, "chats facts: kb/index not found, skipping brain write\n")
return
}
mdDir := filepath.Join(chatsDir(), "facts")
if err := os.MkdirAll(mdDir, 0755); err != nil {
fmt.Fprintf(os.Stderr, "chats facts: mkdir %s: %v\n", mdDir, err)
return
}
var sb strings.Builder
sb.WriteString("---\n")
sb.WriteString("root: facts\n")
sb.WriteString("---\n\n")
sb.WriteString("# Chat-Derived Facts\n\n")
for _, f := range facts {
sb.WriteString(fmt.Sprintf("- **%s**: %s (source: %s, chat: %s)\n",
f.FactType, f.Value, f.Source, f.ChatName))
}
sb.WriteString("\n")
factsMD := filepath.Join(mdDir, "chat-facts.md")
if err := os.WriteFile(factsMD, []byte(sb.String()), 0644); err != nil {
fmt.Fprintf(os.Stderr, "chats facts: write %s: %v\n", factsMD, err)
return
}
cmd := exec.Command(indexScript, "--corpus", mdDir, "--skip-indexes")
var outBuf, errBuf bytes.Buffer
cmd.Stdout = &outBuf
cmd.Stderr = &errBuf
cmd.Dir = root
if err := cmd.Run(); err != nil {
fmt.Fprintf(os.Stderr, "chats facts: brain index: %v\n%s", err, errBuf.String())
return
}
fmt.Printf("chats facts: written to brain (%s)\n", strings.TrimSpace(outBuf.String()))
}
bin/chats/go.sum
View File
bin/chats/import.go
Executable
+20
View File
@@ -0,0 +1,20 @@
//usr/bin/env go run -tags=chats_import "$0" "$@"; exit
//go:build chats_import
//
// bin/chats/import.go - JSONL → markdown under var/chats/md/.
//
// ./bin/chats/import.go
//
// Conversion only. Brain ingest is bin/brain/index.go, not this command.
// NOTE: never run `gofmt -w` on this file — it breaks the shebang.
package main
import (
"os"
"github.com/eSlider/2dph/internal/chats"
)
func main() {
os.Exit(chats.RunImport(os.Args[1:]))
}
bin/chats/import_cmd.go
-204
View File
@@ -1,204 +0,0 @@
package main
import (
"bufio"
"bytes"
"encoding/json"
"flag"
"fmt"
"html"
"os"
"path/filepath"
"sort"
"strings"
)
func runImport(args []string) int {
fs := flag.NewFlagSet("chats import", flag.ContinueOnError)
help := fs.Bool("help", false, "")
fs.SetOutput(os.Stderr)
if err := fs.Parse(args); err != nil {
return 2
}
if *help {
fmt.Fprintln(os.Stderr, "usage: chats import")
return 0
}
root := chatsDir()
mdRoot := filepath.Join(root, "md")
glob := filepath.Join(root, "telegram", "*", "messages.jsonl")
matches, err := filepath.Glob(glob)
if err != nil {
fmt.Fprintf(os.Stderr, "chats import: glob %s: %v\n", glob, err)
return 1
}
if len(matches) == 0 {
fmt.Fprintf(os.Stderr, "chats import: no messages.jsonl found under %s\n", root)
return 1
}
written := 0
failed := 0
for _, jsonlPath := range matches {
chatID := filepath.Base(filepath.Dir(jsonlPath))
messages, chatName, err := readJSONL(jsonlPath)
if err != nil {
fmt.Fprintf(os.Stderr, "chats import: read %s: %v\n", jsonlPath, err)
failed++
continue
}
if len(messages) == 0 {
continue
}
if chatName == "" {
chatName = chatID
}
participants := collectParticipants(messages)
chatType := "personal"
if len(participants) > 3 {
chatType = "group"
}
firstID := ""
if len(messages) > 0 {
firstID = messages[0].ID
}
var b bytes.Buffer
b.WriteString("---\n")
fmt.Fprintf(&b, "id: %s\n", firstID)
fmt.Fprintf(&b, "platform: telegram\n")
fmt.Fprintf(&b, "chat_id: %s\n", chatID)
fmt.Fprintf(&b, "chat_name: %s\n", escapeYAML(chatName))
fmt.Fprintf(&b, "participants: [")
for i, p := range participants {
if i > 0 {
b.WriteString(", ")
}
b.WriteString(escapeYAML(p))
}
b.WriteString("]\n")
fmt.Fprintf(&b, "message_count: %d\n", len(messages))
fmt.Fprintf(&b, "type: %s\n", chatType)
b.WriteString("---\n\n")
fmt.Fprintf(&b, "# Чат с %s\n\n", chatName)
for _, msg := range messages {
ts := msg.Timestamp
if len(ts) > 10 {
ts = ts[:10]
}
text := msg.Text
text = html.UnescapeString(text)
text = strings.ReplaceAll(text, "\n", "\n ")
line := fmt.Sprintf("**%s** — %s: %s", ts, msg.From, text)
if msg.Media != nil {
line += " *(" + *msg.Media + ")*"
}
b.WriteString(line + "\n\n")
}
mdFile := filepath.Join(mdRoot, "telegram", sanitizeDir(chatName), "messages.md")
if err := os.MkdirAll(filepath.Dir(mdFile), 0755); err != nil {
fmt.Fprintf(os.Stderr, "chats import: mkdir %s: %v\n", filepath.Dir(mdFile), err)
failed++
continue
}
if err := os.WriteFile(mdFile, b.Bytes(), 0644); err != nil {
fmt.Fprintf(os.Stderr, "chats import: write %s: %v\n", mdFile, err)
failed++
continue
}
written++
}
fmt.Printf("chats import: %d chats written", written)
if failed > 0 {
fmt.Printf(", %d failed", failed)
}
fmt.Println()
if failed > 0 {
return 1
}
return 0
}
func readJSONL(path string) ([]Message, string, error) {
f, err := os.Open(path)
if err != nil {
return nil, "", err
}
defer f.Close()
var messages []Message
scanner := bufio.NewScanner(f)
scanner.Buffer(make([]byte, 1<<20), 1<<20)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
var msg Message
if err := json.Unmarshal([]byte(line), &msg); err != nil {
continue
}
messages = append(messages, msg)
}
if err := scanner.Err(); err != nil {
return messages, "", err
}
chatName := ""
if len(messages) > 0 {
nameCounts := make(map[string]int)
for _, msg := range messages {
nameCounts[msg.From]++
}
best := ""
bestN := 0
for name, n := range nameCounts {
if name != "" && name != "unknown" && n > bestN {
best = name
bestN = n
}
}
if best != "" {
chatName = best
}
}
return messages, chatName, nil
}
func collectParticipants(messages []Message) []string {
seen := make(map[string]bool)
var result []string
for _, msg := range messages {
if msg.From == "" || seen[msg.From] {
continue
}
seen[msg.From] = true
result = append(result, msg.From)
}
sort.Strings(result)
return result
}
func escapeYAML(s string) string {
if strings.ContainsAny(s, ":#,[]{}'\"") || strings.HasPrefix(s, "-") {
return `"` + strings.ReplaceAll(s, `"`, `\"`) + `"`
}
return s
}
func sanitizeDir(name string) string {
r := strings.NewReplacer(
"/", "_", "\\", "_", ":", "_", "*", "_",
"?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",
" ", "_",
)
return strings.TrimSpace(r.Replace(name))
}
bin/chats/index_cmd.go
-56
View File
@@ -1,56 +0,0 @@
package main
import (
"bytes"
"flag"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
)
func runIndex(args []string) int {
fs := flag.NewFlagSet("chats index", flag.ContinueOnError)
help := fs.Bool("help", false, "")
fs.SetOutput(os.Stderr)
if err := fs.Parse(args); err != nil {
return 2
}
if *help {
fmt.Fprintln(os.Stderr, "usage: chats index")
return 0
}
root := repoRoot()
mdDir := filepath.Join(chatsDir(), "md")
_, err := os.Stat(mdDir)
if os.IsNotExist(err) {
fmt.Fprintf(os.Stderr, "chats index: no chat markdown at %s; run 'chats import' first\n", mdDir)
return 1
}
indexScript := filepath.Join(root, "bin", "kb", "index")
if _, err := os.Stat(indexScript); os.IsNotExist(err) {
fmt.Fprintf(os.Stderr, "chats index: %s not found\n", indexScript)
return 1
}
cmd := exec.Command(indexScript, "--corpus", mdDir)
var outBuf, errBuf bytes.Buffer
cmd.Stdout = &outBuf
cmd.Stderr = &errBuf
cmd.Dir = root
if err := cmd.Run(); err != nil {
fmt.Fprintf(os.Stderr, "chats index: %v\n%s", err, errBuf.String())
return 1
}
result := strings.TrimSpace(outBuf.String())
if result == "" {
result = strings.TrimSpace(errBuf.String())
}
fmt.Printf("chats index: %s\n", result)
return 0
}
bin/chats/linkedin.go
-669
View File
@@ -1,669 +0,0 @@
package main
import (
"bufio"
"context"
"encoding/json"
"fmt"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"time"
)
type LinkedInMCPSource struct {
userDataDir string
limit int
}
type lnInboxItem struct {
ThreadID string `json:"thread_id"`
Participants string `json:"participants"`
LastMessage string `json:"last_message"`
LastMessageDate string `json:"last_message_date"`
Unread bool `json:"unread"`
}
// mcp-server-linkedin v4.22 returns get_inbox / get_conversation as
// {url, sections:{inbox|conversation: textblob}, references:{...}}.
// The conversation list lives in references (kind=conversation); messages live
// in the sections text blob, delimited by "<From> sent the following message
// at <time>" markers. See testdata/linkedin_*.json for the wire shape.
type lnEnvelope struct {
URL string `json:"url"`
Sections map[string]any `json:"sections"`
References map[string]any `json:"references"`
}
type lnReference struct {
Kind string `json:"kind"`
URL string `json:"url"`
Text string `json:"text"`
Context string `json:"context"`
}
type lnMessage struct {
From string `json:"from"`
Date string `json:"date"`
Text string `json:"text"`
}
var (
lnWeekdays = map[string]time.Weekday{
"SUNDAY": time.Sunday, "MONDAY": time.Monday, "TUESDAY": time.Tuesday,
"WEDNESDAY": time.Wednesday, "THURSDAY": time.Thursday,
"FRIDAY": time.Friday, "SATURDAY": time.Saturday,
}
lnMsgStartRe = regexp.MustCompile(`^(.+?) sent the following messages? at (.+)$`)
lnTimeRe = regexp.MustCompile(`\d{1,2}:\d{2}\s*[AP]M`)
)
func isWeekdayLine(s string) bool {
if _, ok := lnWeekdays[s]; ok {
return true
}
switch s {
case "TODAY", "YESTERDAY", "THIS WEEK", "LAST WEEK":
return true
}
return lnMonthDayRe.MatchString(s)
}
var lnMonthDayRe = regexp.MustCompile(`^[A-Z]{3}\s+\d{1,2}$`)
// parseLinkedInInbox extracts conversations from a get_inbox response.
func parseLinkedInInbox(text string) []lnInboxItem {
var env lnEnvelope
if err := json.Unmarshal([]byte(text), &env); err != nil {
return nil
}
refs, _ := env.References["inbox"].([]any)
var items []lnInboxItem
for _, r := range refs {
rr, ok := r.(map[string]any)
if !ok {
continue
}
if rr["kind"] != "conversation" {
continue
}
u, _ := rr["url"].(string)
tid := threadIDFromURL(u)
if !validThreadID(tid) {
continue
}
name, _ := rr["text"].(string)
items = append(items, lnInboxItem{
ThreadID: tid,
Participants: name,
})
}
return items
}
// parseLinkedInConversation parses the sections.conversation text blob into
// messages. Messages are delimited by "<From> sent the following message(s) at
// <time>" lines; each message body runs until the next marker. Day headers
// (all-caps weekdays) provide date context; times are mapped to the most
// recent matching weekday.
func parseLinkedInConversation(text string) []lnMessage {
var env lnEnvelope
if err := json.Unmarshal([]byte(text), &env); err != nil {
return nil
}
blob, _ := env.Sections["conversation"].(string)
if blob == "" {
return nil
}
var msgs []lnMessage
var cur *lnMessage
var body []string
day := ""
flush := func() {
if cur == nil {
return
}
cur.Text = strings.TrimSpace(strings.Join(body, "\n"))
if ts := linkedInTimestamp(day, cur.Date); ts != "" {
cur.Date = ts
}
if cur.Text != "" {
msgs = append(msgs, *cur)
}
cur = nil
body = nil
}
for _, raw := range strings.Split(blob, "\n") {
line := strings.TrimSpace(raw)
if line == "" {
continue
}
if isWeekdayLine(line) {
if line != day {
// A new day header terminates the previous message,
// which must keep the earlier date context.
flush()
}
day = line
continue
}
if m := lnMsgStartRe.FindStringSubmatch(line); m != nil {
flush()
cur = &lnMessage{From: strings.TrimSpace(m[1]), Date: strings.TrimSpace(m[2])}
continue
}
if cur == nil {
continue
}
// Skip "View X's profile" and the "<From> (pronouns) <time>" header.
if strings.HasPrefix(line, "View ") && strings.HasSuffix(line, "'s profile") {
continue
}
if strings.HasPrefix(line, cur.From) && lnTimeRe.MatchString(line) {
continue
}
body = append(body, line)
}
flush()
return msgs
}
// linkedInTimestamp maps a weekday, relative, or MON DD date header + clock
// string to a timestamp, or returns "" when the clock cannot be parsed.
func linkedInTimestamp(day, clock string) string {
t, err := time.Parse("3:04 PM", clock)
if err != nil {
return ""
}
now := time.Now()
var d time.Time
if wd, ok := lnWeekdays[day]; ok {
diff := (int(now.Weekday()) - int(wd) + 7) % 7
d = now.AddDate(0, 0, -diff)
} else {
switch day {
case "TODAY":
d = now
case "YESTERDAY":
d = now.AddDate(0, 0, -1)
case "THIS WEEK":
diff := int(now.Weekday())
d = now.AddDate(0, 0, -diff)
case "LAST WEEK":
diff := int(now.Weekday()) + 7
d = now.AddDate(0, 0, -diff)
default:
if m := lnMonthDayRe.FindStringSubmatch(day); m != nil {
// MON DD without a year: resolve to the most recent
// occurrence that is not in the future.
d = monthDayDate(day, now)
if d.IsZero() {
return t.Format("15:04")
}
} else {
// No date context; keep bare clock time.
return t.Format("15:04")
}
}
}
res := time.Date(d.Year(), d.Month(), d.Day(), t.Hour(), t.Minute(), 0, 0, time.UTC)
return res.UTC().Format(time.RFC3339)
}
var lnMonths = map[string]time.Month{
"JAN": time.January, "FEB": time.February, "MAR": time.March,
"APR": time.April, "MAY": time.May, "JUN": time.June,
"JUL": time.July, "AUG": time.August, "SEP": time.September,
"OCT": time.October, "NOV": time.November, "DEC": time.December,
}
// monthDayDate resolves "MON DD" to the most recent occurrence of that date,
// preferring the current year and falling back to the previous year when the
// date is in the future. Returns zero time when unresolvable.
func monthDayDate(day string, now time.Time) time.Time {
parts := strings.Fields(day)
if len(parts) != 2 {
return time.Time{}
}
mo, ok := lnMonths[parts[0]]
if !ok {
return time.Time{}
}
var dd int
if _, err := fmt.Sscanf(parts[1], "%d", &dd); err != nil {
return time.Time{}
}
if dd < 1 || dd > 31 {
return time.Time{}
}
d := time.Date(now.Year(), mo, dd, 0, 0, 0, 0, time.UTC)
if d.After(now) {
d = d.AddDate(-1, 0, 0)
}
if d.After(now) {
return time.Time{}
}
return d
}
func threadIDFromURL(u string) string {
u = strings.TrimSuffix(u, "/")
idx := strings.LastIndex(u, "/")
if idx < 0 {
return ""
}
return u[idx+1:]
}
// validThreadID rejects path segments that are not real thread ids (e.g. the
// literal "thread" or an empty trailing segment).
func validThreadID(id string) bool {
if id == "" || id == "thread" {
return false
}
return true
}
func NewLinkedInMCPSource(userDataDir string) *LinkedInMCPSource {
return &LinkedInMCPSource{userDataDir: userDataDir}
}
func (s *LinkedInMCPSource) Name() string { return "linkedin" }
func (s *LinkedInMCPSource) Sync(ctx context.Context, outDir string, limit int) error {
if limit > 0 {
s.limit = limit
}
// getConversation fetches one thread, recreating the MCP server when it
// wedges. A single 429 makes mcp-server-linkedin close its browser and
// refuse every later call ("still has a browser open"), so a broken server
// must be restarted rather than hammered.
getConversation := func(threadID string) ([]lnMessage, error) {
client, err := newLinkedInMCP(ctx, s.userDataDir)
if err != nil {
return nil, fmt.Errorf("linkedin mcp: %w", err)
}
defer client.Close()
msgs, err := client.GetConversation(ctx, "", threadID, msgLimitFor(s.limit))
if err != nil && wedged(err) {
fmt.Fprintf(os.Stderr, "chats: %s: server wedged, restarting broker\n", threadID)
time.Sleep(5 * time.Second)
client2, cerr := newLinkedInMCP(ctx, s.userDataDir)
if cerr == nil {
defer client2.Close()
msgs, err = client2.GetConversation(ctx, "", threadID, msgLimitFor(s.limit))
}
}
return msgs, err
}
client, err := newLinkedInMCP(ctx, s.userDataDir)
if err != nil {
return fmt.Errorf("linkedin mcp: %w", err)
}
inbox, err := client.GetInbox(ctx, 50)
if err != nil {
client.Close()
return fmt.Errorf("get_inbox: %w", err)
}
client.Close()
if len(inbox) == 0 {
fmt.Println("chats: no LinkedIn conversations found")
return nil
}
fmt.Printf("chats: found %d LinkedIn conversations\n", len(inbox))
for _, conv := range inbox {
convID := sanitizeDir(conv.ThreadID)
if convID == "" {
convID = fmt.Sprintf("conv_%d", time.Now().UnixNano())
}
parts := strings.SplitN(conv.Participants, ",", 2)
chatName := strings.TrimSpace(parts[0])
if chatName == "" {
chatName = convID
}
chatDir := filepath.Join(outDir, "linkedin", convID)
if err := os.MkdirAll(chatDir, 0755); err != nil {
fmt.Fprintf(os.Stderr, "chats: mkdir %s: %v\n", chatDir, err)
continue
}
msgs, err := getConversation(conv.ThreadID)
if err != nil {
fmt.Fprintf(os.Stderr, "chats: get_conversation %s: %v\n", convID, err)
continue
}
jsonlPath := filepath.Join(chatDir, "messages.jsonl")
// A rate-limited response can parse to zero messages. Never clobber
// previously synced data with an empty file.
if len(msgs) == 0 {
fmt.Fprintf(os.Stderr, "chats: %s (%s): 0 messages parsed, keeping existing file\n", chatName, convID)
continue
}
f, err := os.Create(jsonlPath)
if err != nil {
fmt.Fprintf(os.Stderr, "chats: create %s: %v\n", jsonlPath, err)
continue
}
enc := json.NewEncoder(f)
written := 0
for i, m := range msgs {
text := m.Text
if text == "" {
continue
}
ts := m.Date
if t, err := time.Parse("2006-01-02T15:04:05Z07:00", m.Date); err == nil {
ts = t.UTC().Format(time.RFC3339)
} else if t, err := time.Parse(time.RFC3339, m.Date); err == nil {
ts = t.UTC().Format(time.RFC3339)
}
chatMsg := Message{
ID: fmt.Sprintf("li_%s_%d", convID, i),
Timestamp: ts,
From: m.From,
Text: text,
Platform: "linkedin",
}
if err := enc.Encode(chatMsg); err != nil {
fmt.Fprintf(os.Stderr, "chats: encode: %v\n", err)
continue
}
written++
}
f.Close()
fmt.Printf("chats: synced %s (%s) — %d messages\n", chatName, convID, written)
// Pause between conversations to reduce LinkedIn rate limiting.
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(2 * time.Second):
}
}
return nil
}
type linkedInMCPClient struct {
cmd *exec.Cmd
stdin *bufio.Writer
stdout *bufio.Scanner
msgID int
}
func newLinkedInMCP(ctx context.Context, userDataDir string) (*linkedInMCPClient, error) {
args := []string{
"mcp-server-linkedin@latest",
"--user-data-dir", userDataDir,
"--no-auto-import",
"--no-daemon",
"--transport", "stdio",
"--login-timeout", "10",
"--browser-wait", "1",
"--browser-idle-timeout", "10",
"--log-level", "ERROR",
}
cmd := exec.CommandContext(ctx, "uvx", args...)
cmd.Env = os.Environ()
stdin, err := cmd.StdinPipe()
if err != nil {
return nil, fmt.Errorf("stdin pipe: %w", err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, fmt.Errorf("stdout pipe: %w", err)
}
cmd.Stderr = os.Stderr
if err := cmd.Start(); err != nil {
return nil, fmt.Errorf("start: %w", err)
}
c := &linkedInMCPClient{
cmd: cmd,
stdin: bufio.NewWriter(stdin),
stdout: bufio.NewScanner(stdout),
msgID: 0,
}
c.stdout.Buffer(make([]byte, 1<<20), 1<<20)
if err := c.initialize(ctx); err != nil {
c.Close()
return nil, fmt.Errorf("init: %w", err)
}
return c, nil
}
func (c *linkedInMCPClient) nextID() int {
c.msgID++
return c.msgID
}
func (c *linkedInMCPClient) initialize(ctx context.Context) error {
params := map[string]interface{}{
"protocolVersion": "2024-11-05",
"capabilities": map[string]interface{}{},
"clientInfo": map[string]string{
"name": "chats-sync",
"version": "0.1.0",
},
}
_, err := c.send(ctx, "initialize", params)
return err
}
func (c *linkedInMCPClient) send(ctx context.Context, method string, params interface{}) (json.RawMessage, error) {
id := c.nextID()
req := map[string]interface{}{
"jsonrpc": "2.0",
"id": id,
"method": method,
}
if params != nil {
req["params"] = params
}
body, err := json.Marshal(req)
if err != nil {
return nil, fmt.Errorf("marshal: %w", err)
}
if _, err := c.stdin.Write(body); err != nil {
return nil, fmt.Errorf("write: %w", err)
}
if err := c.stdin.WriteByte('\n'); err != nil {
return nil, fmt.Errorf("newline: %w", err)
}
if err := c.stdin.Flush(); err != nil {
return nil, fmt.Errorf("flush: %w", err)
}
for c.stdout.Scan() {
line := c.stdout.Text()
if line == "" {
continue
}
var resp struct {
JSONRPC string `json:"jsonrpc"`
ID int `json:"id"`
Result json.RawMessage `json:"result,omitempty"`
Error *struct {
Code int `json:"code"`
Message string `json:"message"`
} `json:"error,omitempty"`
}
if err := json.Unmarshal([]byte(line), &resp); err != nil {
return nil, fmt.Errorf("unmarshal: %w\nline: %s", err, line[:min(len(line), 500)])
}
if resp.Error != nil {
return nil, fmt.Errorf("rpc error %d: %s", resp.Error.Code, resp.Error.Message)
}
return resp.Result, nil
}
return nil, fmt.Errorf("no response: %w", c.stdout.Err())
}
// msgLimitFor returns the per-conversation message cap for a sync.
func msgLimitFor(limit int) int {
if limit > 0 {
return limit
}
return 100
}
// wedged reports whether a conversation fetch failure means the MCP server
// closed its browser and will refuse every later call.
func wedged(err error) bool {
return strings.Contains(err.Error(), "still has a browser open")
}
// callTool invokes an MCP tool, retrying transient (rate-limit) failures.
func (c *linkedInMCPClient) callTool(ctx context.Context, name string, params map[string]interface{}) (json.RawMessage, error) {
var lastErr error
for attempt := 0; attempt < 3; attempt++ {
if attempt > 0 {
delay := time.Duration(1<<uint(attempt)) * 5 * time.Second
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(delay):
}
}
result, err := c.send(ctx, "tools/call", map[string]interface{}{
"name": name,
"arguments": params,
})
if err == nil {
// Tool-level errors surface as a successful RPC with an
// isError=true content entry.
if hint := toolErrorHint(result); hint != "" {
lastErr = fmt.Errorf("%s error: %s", name, hint)
if !isTransientLinkedInError(lastErr.Error()) {
return nil, lastErr
}
continue
}
return result, nil
}
lastErr = err
if !isTransientLinkedInError(err.Error()) {
return nil, err
}
}
return nil, fmt.Errorf("%s: %w", name, lastErr)
}
// toolErrorHint returns the tool's error text when the result has isError set.
func toolErrorHint(result json.RawMessage) string {
var toolRes struct {
Content []struct {
Type string `json:"type"`
Text string `json:"text"`
} `json:"content"`
IsError bool `json:"isError"`
}
if err := json.Unmarshal(result, &toolRes); err != nil || !toolRes.IsError {
return ""
}
if len(toolRes.Content) > 0 {
return toolRes.Content[0].Text
}
return "unknown tool error"
}
// isTransientLinkedInError reports whether a fetch failed due to rate limiting
// or a transient server error, which may succeed on retry.
func isTransientLinkedInError(msg string) bool {
return strings.Contains(msg, "503") || strings.Contains(msg, "429") ||
strings.Contains(msg, "ERR_HTTP_RESPONSE_CODE_FAILURE") ||
strings.Contains(msg, "ERR_ABORTED") ||
strings.Contains(msg, "Error calling tool") ||
strings.Contains(msg, "Unexpected error") ||
strings.Contains(msg, "still has a browser open")
}
func (c *linkedInMCPClient) GetInbox(ctx context.Context, limit int) ([]lnInboxItem, error) {
params := map[string]interface{}{
"limit": limit,
}
result, err := c.callTool(ctx, "get_inbox", params)
if err != nil {
return nil, err
}
var toolRes struct {
Content []struct {
Type string `json:"type"`
Text string `json:"text"`
} `json:"content"`
IsError bool `json:"isError"`
}
if err := json.Unmarshal(result, &toolRes); err != nil {
return nil, fmt.Errorf("unmarshal tool: %w", err)
}
if len(toolRes.Content) == 0 {
return nil, nil
}
text := toolRes.Content[0].Text
return parseLinkedInInbox(text), nil
}
func (c *linkedInMCPClient) GetConversation(ctx context.Context, username, threadID string, limit int) ([]lnMessage, error) {
params := map[string]interface{}{
"linkedin_username": username,
"thread_id": threadID,
"index": limit,
}
result, err := c.callTool(ctx, "get_conversation", params)
if err != nil {
return nil, err
}
var toolRes struct {
Content []struct {
Type string `json:"type"`
Text string `json:"text"`
} `json:"content"`
IsError bool `json:"isError"`
}
if err := json.Unmarshal(result, &toolRes); err != nil {
return nil, fmt.Errorf("unmarshal tool: %w", err)
}
if len(toolRes.Content) == 0 {
return nil, nil
}
text := toolRes.Content[0].Text
return parseLinkedInConversation(text), nil
}
func (c *linkedInMCPClient) Close() error {
if c.stdin != nil {
c.stdin.Flush()
}
if c.cmd != nil && c.cmd.Process != nil {
c.cmd.Process.Kill()
}
return nil
}
bin/chats/linkedin_test.go
-215
View File
@@ -1,215 +0,0 @@
package main
import (
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func readFixture(t *testing.T, name string) string {
t.Helper()
data, err := os.ReadFile(filepath.Join("testdata", name))
if err != nil {
t.Fatal(err)
}
return string(data)
}
// TestParseLinkedInInbox verifies get_inbox parsing against the v4.22 wire
// format (testdata/linkedin_inbox.json — synthetic Alice/Bob/Charlie).
func TestParseLinkedInInbox(t *testing.T) {
text := readFixture(t, "linkedin_inbox.json")
items := parseLinkedInInbox(text)
if len(items) != 2 {
t.Fatalf("expected 2 conversations (empty thread url skipped), got %d", len(items))
}
first := items[0]
if first.ThreadID == "" {
t.Error("expected thread id extracted from reference url")
}
if first.Participants != "Alice Example" {
t.Errorf("participants=%q, want Alice Example", first.Participants)
}
if !strings.HasPrefix(first.ThreadID, "2-") {
t.Errorf("unexpected thread id format %q", first.ThreadID)
}
if items[1].Participants != "Charlie Example" {
t.Errorf("second participant=%q, want Charlie Example", items[1].Participants)
}
}
// TestParseLinkedInInboxBadJSON verifies a non-JSON response yields no items
// rather than a panic or error.
func TestParseLinkedInInboxBadJSON(t *testing.T) {
if got := parseLinkedInInbox("Session expired"); len(got) != 0 {
t.Fatalf("expected no items for non-JSON, got %d", len(got))
}
}
// TestParseLinkedInConversation verifies message extraction from the sections
// blob (testdata/linkedin_conversation.json — synthetic Alice/Bob).
func TestParseLinkedInConversation(t *testing.T) {
text := readFixture(t, "linkedin_conversation.json")
msgs := parseLinkedInConversation(text)
if len(msgs) != 2 {
t.Fatalf("expected 2 messages, got %d", len(msgs))
}
if msgs[0].From != "Alice Example" {
t.Errorf("from=%q, want Alice Example", msgs[0].From)
}
if !strings.Contains(msgs[0].Text, "Senior Software Engineer") {
t.Errorf("alice text missing role, got %q", msgs[0].Text)
}
if msgs[0].Date == "" {
t.Error("expected message date")
}
if msgs[1].From != "Bob Example" {
t.Errorf("from=%q, want Bob Example", msgs[1].From)
}
}
// TestParseLinkedInConversationEmpty verifies empty/non-JSON blobs parse to
// zero messages.
func TestParseLinkedInConversationEmpty(t *testing.T) {
if got := parseLinkedInConversation("no data here"); len(got) != 0 {
t.Fatalf("expected 0 messages, got %d", len(got))
}
}
// TestLinkedInTimestamp verifies weekday+clock resolution to a recent UTC date.
func TestLinkedInTimestamp(t *testing.T) {
// The most recent Wednesday before/equal to "now".
ts := linkedInTimestamp("WEDNESDAY", "10:02 AM")
parsed, err := time.Parse(time.RFC3339, ts)
if err != nil {
t.Fatalf("unparseable timestamp %q: %v", ts, err)
}
if parsed.Weekday() != time.Wednesday {
t.Errorf("expected Wednesday, got %s", parsed.Weekday())
}
if parsed.Hour() != 10 || parsed.Minute() != 2 {
t.Errorf("expected 10:02, got %02d:%02d", parsed.Hour(), parsed.Minute())
}
now := time.Now()
diff := now.Sub(parsed)
if diff < 0 || diff > 7*24*time.Hour {
t.Errorf("timestamp %s is not within the last week of %s", parsed, now)
}
if got := linkedInTimestamp("MONDAY", "garbage"); got != "" {
t.Errorf("expected empty for bad clock, got %q", got)
}
if got := linkedInTimestamp("", "1:22 PM"); got != "13:22" {
t.Errorf("expected bare 13:22 for missing weekday, got %q", got)
}
// Relative day headers must resolve to full dates, not bare clocks.
today := linkedInTimestamp("TODAY", "9:42 AM")
yp, err := time.Parse(time.RFC3339, today)
if err != nil {
t.Fatalf("TODAY unparseable %q: %v", today, err)
}
if yp.Year() != now.Year() || yp.Month() != now.Month() || yp.Day() != now.Day() {
t.Errorf("TODAY expected %v, got %v", now, yp)
}
yest := linkedInTimestamp("YESTERDAY", "3:00 PM")
yp, err = time.Parse(time.RFC3339, yest)
if err != nil {
t.Fatalf("YESTERDAY unparseable %q: %v", yest, err)
}
if yp.Day() != now.AddDate(0, 0, -1).Day() {
t.Errorf("YESTERDAY expected day %d, got %d", now.AddDate(0, 0, -1).Day(), yp.Day())
}
// MON DD header (e.g. "JUN 25"): must resolve to a full date. The
// timestamp should fall within the current year (falling back to the
// prior year if the date would be in the future).
md := linkedInTimestamp("JUN 25", "10:48 AM")
mp, err := time.Parse(time.RFC3339, md)
if err != nil {
t.Fatalf("MON DD unparseable %q: %v", md, err)
}
if mp.Year() != now.Year() && mp.Year() != now.Year()-1 {
t.Errorf("JUN 25 expected year %d or %d, got %d", now.Year(), now.Year()-1, mp.Year())
}
if mp.Month() != time.June || mp.Day() != 25 {
t.Errorf("JUN 25 expected Jun 25, got %s %d", mp.Month(), mp.Day())
}
if mp.After(now) {
t.Errorf("JUN 25 resolved to the future: %s > %s", mp, now)
}
}
// TestTransientLinkedInError verifies rate-limit errors are retryable but
// genuine failures are not.
func TestTransientLinkedInError(t *testing.T) {
retryable := []string{
"get_conversation error: Error calling tool 'get_conversation'",
"get_conversation error: Unexpected error in get_conversation: net::ERR_HTTP_RESPONSE_CODE_FAILURE",
"rpc error 503: rate limited",
"rpc error 429: too many requests",
"get_conversation: get_conversation error: This server still has a browser open on the profile.",
}
for _, msg := range retryable {
if !isTransientLinkedInError(msg) {
t.Errorf("expected %q to be transient", msg)
}
}
permanent := []string{
"get_inbox error: bad credentials",
"rpc error -32602: Invalid request parameters",
"unmarshal: unexpected end of JSON input",
}
for _, msg := range permanent {
if isTransientLinkedInError(msg) {
t.Errorf("expected %q to be permanent", msg)
}
}
}
// TestMsgLimitFor verifies the per-conversation message cap resolution.
func TestMsgLimitFor(t *testing.T) {
if got := msgLimitFor(0); got != 100 {
t.Errorf("expected default 100, got %d", got)
}
if got := msgLimitFor(5); got != 5 {
t.Errorf("expected 5, got %d", got)
}
}
// TestWedged verifies the browser-open failure is recognized as a wedge.
func TestWedged(t *testing.T) {
if !wedged(errors.New("get_conversation error: This server still has a browser open on the profile")) {
t.Error("expected wedged error to be recognized")
}
if wedged(errors.New("get_conversation error: bad thing")) {
t.Error("unexpected wedge detection")
}
}
// TestThreadIDFromURL verifies thread id extraction.
func TestThreadIDFromURL(t *testing.T) {
cases := []struct {
url, want string
}{
{"/messaging/thread/2-abc123/", "2-abc123"},
{"/messaging/thread/2-abc123", "2-abc123"},
{"", ""},
{"/messaging/thread/", ""},
}
for _, c := range cases {
got := threadIDFromURL(c.url)
if c.want == "" && validThreadID(got) {
t.Errorf("threadIDFromURL(%q) = %q, want empty", c.url, got)
}
if c.want != "" && got != c.want {
t.Errorf("threadIDFromURL(%q) = %q, want %q", c.url, got, c.want)
}
}
}
bin/chats/main.go
-115
View File
@@ -1,115 +0,0 @@
// bin/chats - sync, import, index, extract facts, and apply chat data
// from Telegram, WhatsApp, LinkedIn into the brain and OnlyOffice CRM.
//
// Usage:
//
// chats sync telegram [--limit N] [--since DATE] [--phone PHONE]
// chats sync whatsapp [--qr] [--limit N]
// chats sync linkedin [--limit N]
// chats import # JSONL → MD (all sources)
// chats index # rebuild var/kb.lbug with chats
// chats facts # extract + cross-check
// chats apply [--dry-run] # push to OnlyOffice CRM
package main
import (
"fmt"
"os"
"strings"
)
func main() {
if len(os.Args) < 2 {
usage()
os.Exit(2)
}
cmd := os.Args[1]
args := os.Args[2:]
switch cmd {
case "sync":
if len(args) < 1 {
usage()
os.Exit(2)
}
platform := args[0]
platformArgs := args[1:]
switch platform {
case "telegram":
os.Exit(runSyncTelegram(platformArgs))
case "whatsapp":
fmt.Fprintf(os.Stderr, "chats: WhatsApp not implemented yet\n")
os.Exit(1)
case "linkedin":
os.Exit(runSyncLinkedIn(platformArgs))
default:
fmt.Fprintf(os.Stderr, "chats: unknown platform %q\n", platform)
os.Exit(2)
}
case "import":
os.Exit(runImport(args))
case "index":
os.Exit(runIndex(args))
case "facts":
os.Exit(runFacts(args))
case "apply":
os.Exit(runApply(args))
case "help", "-h", "--help":
usage()
return
default:
fmt.Fprintf(os.Stderr, "chats: unknown command %q\n", cmd)
usage()
os.Exit(2)
}
}
func usage() {
w := os.Stderr
fmt.Fprintln(w, `Usage: chats <command> [args]
Commands:
sync telegram [--limit N] [--since DATE] [--phone PHONE]
sync whatsapp [--qr] [--limit N]
sync linkedin [--limit N]
import JSONL → MD (all sources)
index rebuild var/kb.lbug with chats
facts extract + cross-check facts
apply [--dry-run] push to OnlyOffice CRM
Output layout:
var/chats/<platform>/<chat_id>/messages.jsonl
var/chats/md/<platform>/<chat_name>/messages.md`)
}
// repoRoot locates the 2dph project root by walking up from the binary.
func repoRoot() string {
if v := os.Getenv("KB_ROOT"); v != "" {
return v
}
wd, err := os.Getwd()
if err != nil {
return "."
}
for i := 0; i < 10; i++ {
if _, err := os.Stat(wd + "/var"); err == nil {
return wd
}
if _, err := os.Stat(wd + "/.git"); err == nil {
return wd
}
parent := wd
if idx := strings.LastIndex(wd, "/"); idx >= 0 {
parent = wd[:idx]
}
if parent == wd {
break
}
wd = parent
}
return "."
}
// chatsDir returns var/chats under the repo root.
func chatsDir() string {
return repoRoot() + "/var/chats"
}
bin/chats/mcpclient.go
-424
View File
@@ -1,424 +0,0 @@
package main
import (
"bufio"
"context"
"encoding/json"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
)
type MCPClient struct {
cmd *exec.Cmd
stdin *bufio.Writer
stdout *bufio.Scanner
msgID int
}
type mcpRequest struct {
JSONRPC string `json:"jsonrpc"`
ID int `json:"id"`
Method string `json:"method"`
Params interface{} `json:"params,omitempty"`
}
type mcpResponse struct {
JSONRPC string `json:"jsonrpc"`
ID int `json:"id"`
Result json.RawMessage `json:"result,omitempty"`
Error *struct {
Code int `json:"code"`
Message string `json:"message"`
} `json:"error,omitempty"`
}
type mcpToolResult struct {
Content []struct {
Type string `json:"type"`
Text string `json:"text"`
} `json:"content"`
IsError bool `json:"isError,omitempty"`
}
type ListChatsResult struct {
ChatID int64 `json:"chat_id"`
Title string `json:"name"`
Type string `json:"type"`
Username string `json:"username,omitempty"`
}
type listChatsEnvelope struct {
Results []ListChatsResult `json:"results"`
}
type historyEnvelope struct {
Results []GetHistoryResult `json:"results"`
}
type GetHistoryResult struct {
ID int `json:"id"`
Sender string `json:"sender"`
Date string `json:"date"`
Text string `json:"text"`
Media string `json:"media,omitempty"`
Out bool `json:"out,omitempty"`
}
func NewMCPClient(ctx context.Context, apiID int, apiHash, phone, sessionString, mcpDir string) (*MCPClient, error) {
env := os.Environ()
env = append(env,
fmt.Sprintf("TELEGRAM_API_ID=%d", apiID),
fmt.Sprintf("TELEGRAM_API_HASH=%s", apiHash),
fmt.Sprintf("TELEGRAM_PHONE=%s", phone),
fmt.Sprintf("TELEGRAM_SESSION_STRING=%s", sessionString),
"MCP_TRANSPORT=stdio",
)
serverPath := filepath.Join(mcpDir, ".venv", "bin", "python3")
mainPath := filepath.Join(mcpDir, "main.py")
cmd := exec.CommandContext(ctx, serverPath, mainPath)
cmd.Env = env
cmd.Dir = mcpDir
stdin, err := cmd.StdinPipe()
if err != nil {
return nil, fmt.Errorf("stdin pipe: %w", err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, fmt.Errorf("stdout pipe: %w", err)
}
cmd.Stderr = os.Stderr
if err := cmd.Start(); err != nil {
return nil, fmt.Errorf("start mcp: %w", err)
}
c := &MCPClient{
cmd: cmd,
stdin: bufio.NewWriter(stdin),
stdout: bufio.NewScanner(stdout),
msgID: 0,
}
c.stdout.Buffer(make([]byte, 1<<20), 1<<20)
if err := c.initialize(ctx); err != nil {
c.Close()
return nil, fmt.Errorf("initialize: %w", err)
}
return c, nil
}
func (c *MCPClient) nextID() int {
c.msgID++
return c.msgID
}
func (c *MCPClient) sendRequest(ctx context.Context, method string, params interface{}) (json.RawMessage, error) {
id := c.nextID()
req := mcpRequest{
JSONRPC: "2.0",
ID: id,
Method: method,
Params: params,
}
body, err := json.Marshal(req)
if err != nil {
return nil, fmt.Errorf("marshal: %w", err)
}
if _, err := c.stdin.Write(body); err != nil {
return nil, fmt.Errorf("write: %w", err)
}
if err := c.stdin.WriteByte('\n'); err != nil {
return nil, fmt.Errorf("write newline: %w", err)
}
if err := c.stdin.Flush(); err != nil {
return nil, fmt.Errorf("flush: %w", err)
}
for c.stdout.Scan() {
line := c.stdout.Text()
if line == "" {
continue
}
var resp mcpResponse
if err := json.Unmarshal([]byte(line), &resp); err != nil {
return nil, fmt.Errorf("unmarshal response: %w\nline: %s", err, line[:min(len(line), 500)])
}
if resp.Error != nil {
return nil, fmt.Errorf("rpc error %d: %s", resp.Error.Code, resp.Error.Message)
}
return resp.Result, nil
}
return nil, fmt.Errorf("no response: %w", c.stdout.Err())
}
func (c *MCPClient) initialize(ctx context.Context) error {
params := map[string]interface{}{
"protocolVersion": "2024-11-05",
"capabilities": map[string]interface{}{},
"clientInfo": map[string]string{
"name": "chats-sync",
"version": "0.1.0",
},
}
_, err := c.sendRequest(ctx, "initialize", params)
return err
}
func (c *MCPClient) ListChats(ctx context.Context, chatType string, limit int) ([]ListChatsResult, error) {
args := map[string]interface{}{
"chat_type": chatType,
"limit": limit,
}
result, err := c.sendRequest(ctx, "tools/call", map[string]interface{}{
"name": "list_chats",
"arguments": args,
})
if err != nil {
return nil, err
}
var toolRes mcpToolResult
if err := json.Unmarshal(result, &toolRes); err != nil {
return nil, fmt.Errorf("unmarshal tool result: %w", err)
}
if toolRes.IsError {
msg := "unknown"
if len(toolRes.Content) > 0 {
msg = toolRes.Content[0].Text
}
return nil, fmt.Errorf("list_chats error: %s", msg)
}
if len(toolRes.Content) == 0 {
return nil, nil
}
text := toolRes.Content[0].Text
if text == "" || text == "No chats found matching the criteria." {
return nil, nil
}
var env listChatsEnvelope
if err := json.Unmarshal([]byte(text), &env); err != nil {
var arr []ListChatsResult
if err2 := json.Unmarshal([]byte(text), &arr); err2 != nil {
return nil, fmt.Errorf("parse chats: %w (also tried array: %v)\nbody: %s", err, err2, text[:min(len(text), 500)])
}
return arr, nil
}
return env.Results, nil
}
func (c *MCPClient) GetHistory(ctx context.Context, chatID int64, limit int) ([]GetHistoryResult, error) {
args := map[string]interface{}{
"chat_id": chatID,
"limit": limit,
}
result, err := c.sendRequest(ctx, "tools/call", map[string]interface{}{
"name": "get_history",
"arguments": args,
})
if err != nil {
return nil, err
}
var toolRes mcpToolResult
if err := json.Unmarshal(result, &toolRes); err != nil {
return nil, fmt.Errorf("unmarshal tool result: %w", err)
}
if toolRes.IsError {
msg := "unknown"
if len(toolRes.Content) > 0 {
msg = toolRes.Content[0].Text
}
return nil, fmt.Errorf("get_history error: %s", msg)
}
if len(toolRes.Content) == 0 {
return nil, nil
}
text := toolRes.Content[0].Text
if text == "" || text == "No messages found for this page." || text == "No messages found matching the criteria." {
return nil, nil
}
var env historyEnvelope
if err := json.Unmarshal([]byte(text), &env); err != nil {
var arr []GetHistoryResult
if err2 := json.Unmarshal([]byte(text), &arr); err2 != nil {
return nil, fmt.Errorf("parse history: %w (also tried array: %v)\nbody: %s", err, err2, text[:min(len(text), 500)])
}
return arr, nil
}
return env.Results, nil
}
func (c *MCPClient) Close() error {
if c.stdin != nil {
c.stdin.Flush()
}
if c.cmd != nil && c.cmd.Process != nil {
c.cmd.Process.Kill()
}
return nil
}
type TelegramMCPSource struct {
mcpDir string
apiID int
apiHash string
phone string
sessionStr string
limit int
}
func NewTelegramMCPSource(apiID int, apiHash, phone, sessionString, mcpDir string) *TelegramMCPSource {
return &TelegramMCPSource{
mcpDir: mcpDir,
apiID: apiID,
apiHash: apiHash,
phone: phone,
sessionStr: sessionString,
}
}
func (s *TelegramMCPSource) Name() string { return "telegram" }
func (s *TelegramMCPSource) Sync(ctx context.Context, outDir string, limit int) error {
if limit > 0 {
s.limit = limit
}
client, err := NewMCPClient(ctx, s.apiID, s.apiHash, s.phone, s.sessionStr, s.mcpDir)
if err != nil {
return fmt.Errorf("mcp client: %w", err)
}
defer client.Close()
chats, err := client.ListChats(ctx, "user", 100)
if err != nil {
return fmt.Errorf("list chats: %w", err)
}
if len(chats) == 0 {
fmt.Println("chats: no personal chats found")
return nil
}
fmt.Printf("chats: found %d personal chats\n", len(chats))
var filtered []ListChatsResult
for _, c := range chats {
if strings.Contains(strings.ToLower(c.Username), "bot") {
continue
}
if c.ChatID == 777000 { // Telegram service
continue
}
filtered = append(filtered, c)
}
fmt.Printf("chats: %d after filter (bots excluded)\n", len(filtered))
for _, chat := range filtered {
chatID := fmt.Sprintf("user_%d", chat.ChatID)
chatName := chat.Title
if chatName == "" {
chatName = chatID
}
chatDir := filepath.Join(outDir, "telegram", chatID)
if err := os.MkdirAll(chatDir, 0755); err != nil {
fmt.Fprintf(os.Stderr, "chats: mkdir %s: %v\n", chatDir, err)
continue
}
jsonlPath := filepath.Join(chatDir, "messages.jsonl")
f, err := os.Create(jsonlPath)
if err != nil {
fmt.Fprintf(os.Stderr, "chats: create %s: %v\n", jsonlPath, err)
continue
}
msgLimit := 100
if s.limit > 0 {
msgLimit = s.limit
}
msgs, err := client.GetHistory(ctx, chat.ChatID, msgLimit)
if err != nil {
fmt.Fprintf(os.Stderr, "chats: get_history for %s: %v\n", chatName, err)
f.Close()
continue
}
enc := json.NewEncoder(f)
written := 0
for _, m := range msgs {
if m.Out {
continue
}
text := m.Text
if text == "" && m.Media != "" {
text = fmt.Sprintf("[%s]", m.Media)
}
if text == "" {
continue
}
sender := cleanSender(m.Sender)
ts := m.Date
if t, err := time.Parse(time.RFC3339, m.Date); err == nil {
ts = t.UTC().Format(time.RFC3339)
}
chatMsg := Message{
ID: fmt.Sprintf("tg_%d_%d", chat.ChatID, m.ID),
Timestamp: ts,
From: sender,
Text: text,
Platform: "telegram",
}
if m.Media != "" {
desc := fmt.Sprintf("[%s]", m.Media)
chatMsg.Media = &desc
}
if err := enc.Encode(chatMsg); err != nil {
fmt.Fprintf(os.Stderr, "chats: encode msg: %v\n", err)
continue
}
written++
}
f.Close()
if written > 0 {
fmt.Printf("chats: synced %s (%s) — %d messages\n", chatName, chatID, written)
}
}
return nil
}
func cleanSender(sender string) string {
if idx := strings.Index(sender, " ("); idx > 0 {
sender = sender[:idx]
} else if idx := strings.Index(sender, " @"); idx > 0 {
sender = sender[:idx]
}
if idx := strings.Index(sender, " ["); idx > 0 {
sender = sender[:idx]
}
return sender
}
func min(a, b int) int {
if a < b {
return a
}
return b
}
bin/chats/source.go
-52
View File
@@ -1,52 +0,0 @@
package main
import (
"context"
"fmt"
"os"
"time"
)
type Source interface {
Name() string
Sync(ctx context.Context, outDir string, limit int) error
}
type Message struct {
ID string `json:"id"`
Timestamp string `json:"ts"`
From string `json:"from"`
Text string `json:"text"`
Media *string `json:"media,omitempty"`
Platform string `json:"platform"`
}
type ChatInfo struct {
ID string `json:"id"`
Platform string `json:"platform"`
Name string `json:"name"`
Participants []string `json:"participants"`
Type string `json:"type"`
MessageCount int `json:"messageCount"`
LastTS string `json:"lastTs,omitempty"`
}
func envVar(key, fallback string) string {
if v := os.Getenv(key); v != "" {
return v
}
return fallback
}
func parseSince(s string) (time.Time, error) {
for _, layout := range []string{
time.RFC3339,
"2006-01-02T15:04:05",
"2006-01-02",
} {
if t, err := time.Parse(layout, s); err == nil {
return t, nil
}
}
return time.Time{}, fmt.Errorf("cannot parse --since %q; use YYYY-MM-DD or RFC3339", s)
}
bin/chats/sync.go
Executable
+41
View File
@@ -0,0 +1,41 @@
//usr/bin/env go run -tags=chats_sync "$0" "$@"; exit
//go:build chats_sync
//
// bin/chats/sync.go - download chat messages to var/chats/<platform>/.
//
// ./bin/chats/sync.go telegram [--limit N] [--phone PHONE]
// ./bin/chats/sync.go linkedin [--limit N] [--refresh]
//
// NOTE: never run `gofmt -w` on this file — it breaks the shebang.
package main
import (
"fmt"
"os"
"github.com/eSlider/2dph/internal/chats"
)
func main() {
if len(os.Args) < 2 {
fmt.Fprintln(os.Stderr, `usage: bin/chats/sync.go telegram|linkedin [flags]`)
os.Exit(2)
}
platform := os.Args[1]
args := os.Args[2:]
switch platform {
case "telegram":
os.Exit(chats.RunSyncTelegram(args))
case "linkedin":
os.Exit(chats.RunSyncLinkedIn(args))
case "whatsapp":
fmt.Fprintln(os.Stderr, "chats: WhatsApp not implemented yet")
os.Exit(1)
case "help", "-h", "--help":
fmt.Fprintln(os.Stderr, `usage: bin/chats/sync.go telegram|linkedin [flags]`)
return
default:
fmt.Fprintf(os.Stderr, "chats: unknown platform %q\n", platform)
os.Exit(2)
}
}
bin/chats/sync_cmd.go
-87
View File
@@ -1,87 +0,0 @@
package main
import (
"context"
"flag"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"time"
)
func runSyncTelegram(args []string) int {
fs := flag.NewFlagSet("chats sync telegram", flag.ContinueOnError)
limit := fs.Int("limit", 0, "max messages per chat (0 = all)")
phone := fs.String("phone", "", "phone number (default env TELEGRAM_PHONE)")
help := fs.Bool("help", false, "")
fs.SetOutput(os.Stderr)
if err := fs.Parse(args); err != nil {
return 2
}
if *help {
fmt.Fprintln(os.Stderr, "usage: chats sync telegram [--limit N] [--phone PHONE]")
return 0
}
apiIDStr := envVar("TELEGRAM_API_ID", "")
apiHash := envVar("TELEGRAM_API_HASH", "")
sessionStr := envVar("TELEGRAM_SESSION_STRING", "")
phoneNum := *phone
if phoneNum == "" {
phoneNum = envVar("TELEGRAM_PHONE", "")
}
if apiIDStr == "" || apiHash == "" || phoneNum == "" {
fmt.Fprintln(os.Stderr, "chats: need TELEGRAM_API_ID, TELEGRAM_API_HASH, TELEGRAM_PHONE in env")
return 2
}
apiID, err := strconv.Atoi(apiIDStr)
if err != nil {
fmt.Fprintf(os.Stderr, "chats: invalid TELEGRAM_API_ID %q\n", apiIDStr)
return 2
}
mcpDir := envVar("TELEGRAM_MCP_DIR", "")
if mcpDir == "" {
fmt.Fprintln(os.Stderr, "chats: set TELEGRAM_MCP_DIR to telegram-mcp directory")
return 1
}
if _, err := os.Stat(filepath.Join(mcpDir, "main.py")); err != nil {
fmt.Fprintf(os.Stderr, "chats: TELEGRAM_MCP_DIR=%s: main.py not found\n", mcpDir)
return 1
}
if sessionStr == "" {
envPath := filepath.Join(mcpDir, ".env")
if data, err := os.ReadFile(envPath); err == nil {
for _, line := range strings.Split(string(data), "\n") {
if strings.HasPrefix(line, "TELEGRAM_SESSION_STRING=") {
sessionStr = strings.TrimPrefix(line, "TELEGRAM_SESSION_STRING=")
sessionStr = strings.Trim(sessionStr, "\"'")
break
}
}
}
}
if sessionStr == "" {
sessionStr = envVar("TELEGRAM_SESSION_STRING", "")
}
if sessionStr == "" {
fmt.Fprintln(os.Stderr, "chats: TELEGRAM_SESSION_STRING not found; set env or in TELEGRAM_MCP_DIR/.env")
return 1
}
src := NewTelegramMCPSource(apiID, apiHash, phoneNum, sessionStr, mcpDir)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
start := time.Now()
if err := src.Sync(ctx, chatsDir(), *limit); err != nil {
fmt.Fprintf(os.Stderr, "chats sync telegram: %v\n", err)
return 1
}
fmt.Printf("chats sync telegram: completed in %s\n", time.Since(start).Round(time.Millisecond))
return 0
}
bin/chats/sync_linkedin_cmd.go
-105
View File
@@ -1,105 +0,0 @@
package main
import (
"context"
"flag"
"fmt"
"os"
"os/exec"
"path/filepath"
"time"
)
func checkLinkedInSession(userDataDir string) (bool, error) {
// Validate the source-session files without launching a browser. A full
// `--status` run spawns Chromium and loads /feed/, doubling the automation
// exposed to LinkedIn (429 rate limits) before the sync even starts.
root := filepath.Dir(userDataDir)
sessionFiles := []string{
filepath.Join(root, "source-state.json"),
filepath.Join(root, "cookies.json"),
filepath.Join(userDataDir, "Default", "Cookies"),
}
for _, f := range sessionFiles {
if _, err := os.Stat(f); err != nil {
return true, fmt.Errorf("missing session file %s", f)
}
}
return false, nil
}
func runSyncLinkedIn(args []string) int {
fs := flag.NewFlagSet("chats sync linkedin", flag.ContinueOnError)
limit := fs.Int("limit", 0, "max messages per conversation (0 = all)")
refresh := fs.Bool("refresh", false, "refresh session from live webtop browser before sync")
help := fs.Bool("help", false, "")
fs.SetOutput(os.Stderr)
if err := fs.Parse(args); err != nil {
return 2
}
if *help {
fmt.Fprintln(os.Stderr, "usage: chats sync linkedin [--limit N] [--refresh]")
return 0
}
userDataDir := envVar("LINKEDIN_USER_DATA_DIR", "")
if userDataDir == "" {
home, _ := os.UserHomeDir()
userDataDir = home + "/.linkedin-mcp/profile"
}
if *refresh {
if code := refreshLinkedInSession(userDataDir); code != 0 {
return code
}
}
// Check session files first (no browser launch).
loginNeeded, err := checkLinkedInSession(userDataDir)
if err != nil {
fmt.Fprintf(os.Stderr, "chats: linkedin status check: %v\n", err)
}
if loginNeeded {
fmt.Fprintf(os.Stderr, "chats: LinkedIn session missing. Run:\n")
fmt.Fprintf(os.Stderr, " chats sync linkedin --refresh\n")
fmt.Fprintf(os.Stderr, "or point LINKEDIN_USER_DATA_DIR at a valid session\n")
return 1
}
src := NewLinkedInMCPSource(userDataDir)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
start := time.Now()
if err := src.Sync(ctx, chatsDir(), *limit); err != nil {
fmt.Fprintf(os.Stderr, "chats sync linkedin: %v\n", err)
return 1
}
fmt.Printf("chats sync linkedin: completed in %s\n", time.Since(start).Round(time.Millisecond))
return 0
}
// refreshLinkedInSession re-syncs the LinkedIn source session from the live
// webtop browser via the vendored refresh-linkedin-session helper.
func refreshLinkedInSession(userDataDir string) int {
exe, err := os.Executable()
if err != nil {
fmt.Fprintf(os.Stderr, "chats: resolve executable: %v\n", err)
return 1
}
helper := filepath.Join(filepath.Dir(exe), "refresh-linkedin-session")
if _, err := os.Stat(helper); err != nil {
// Fall back to the source tree helper next to this command file.
helper = "bin/chats/refresh-linkedin-session"
}
root := filepath.Dir(userDataDir)
cmd := exec.Command(helper, "--root", root)
cmd.Stdout = os.Stderr
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
fmt.Fprintf(os.Stderr, "chats: linkedin session refresh: %v\n", err)
return 1
}
return 0
}
bin/chats/testdata/linkedin_conversation.json
Vendored
-12
View File
@@ -1,12 +0,0 @@
{
"url": "https://www.linkedin.com/messaging/thread/2-YWxpY2UtYm9iLXRocmVhZC0xMjM=/",
"sections": {
"conversation": "WEDNESDAY\nAlice Example sent the following message at 10:02 AM\nView Alice Example's profile\nAlice Example (She/Her) 10:02 AM\nHi Bob, we have a Senior Software Engineer role that matches your Go and Python background. Happy to share more if you are open to a chat.\n\nBob Example sent the following messages at 1:22 PM\nView Bob Example's profile\nBob Example 1:22 PM\nHi Alice, thanks for reaching out — yes, I am open to exploring a Senior Software Engineer role. Happy to do a short video call.\n"
},
"references": {
"conversation": [
{"kind": "person", "url": "/in/alice-example/", "text": "Alice Example"},
{"kind": "person", "url": "/in/bob-example/", "text": "Bob Example"}
]
}
}
bin/chats/testdata/linkedin_inbox.json
Vendored
-34
View File
@@ -1,34 +0,0 @@
{
"url": "https://www.linkedin.com/messaging/",
"sections": {
"inbox": "Messaging\nInbox\nConversation List\nAlice Example\nExciting opportunity for a senior software engineer\n"
},
"references": {
"inbox": [
{
"kind": "conversation",
"url": "/messaging/thread/2-YWxpY2UtYm9iLXRocmVhZC0xMjM=/",
"context": "inbox",
"text": "Alice Example"
},
{
"kind": "person",
"url": "/in/alice-example/",
"text": "Alice Example",
"context": "inbox"
},
{
"kind": "conversation",
"url": "/messaging/thread/2-Y2hhcmxpZS1ib2ItdGhyZWFkLTQ1Ng==/",
"context": "inbox",
"text": "Charlie Example"
},
{
"kind": "conversation",
"url": "/messaging/thread/",
"context": "inbox",
"text": "should-be-skipped"
}
]
}
}