From a429b823e52dbe586cbbd179b34658a722fe259e Mon Sep 17 00:00:00 2001 From: Andriy Oblivantsev Date: Thu, 13 Aug 2026 00:00:15 +0100 Subject: [PATCH] chore: clean absolute paths, curasoft refs, secrets from history - bin/chats/: env-based paths, no /mnt/ /home/ hardcodes - bin/edelweiss-pilot: remove curasoft, use DOCS_BASE env var - bin/facts/crm: use KNOWLEDGE_MESH_SEED env var - compose.edelweiss.yml: remove curasoft volumes, use DOCS_BASE - docs/chat-import-plan.md: link to Gitea issue, no secrets - bin/seed-edelweiss-facts.py: removed (curasoft-only) --- bin/chats/apply_cmd.go | 1 - bin/chats/sync_cmd.go | 37 +--- bin/edelweiss-pilot | 32 ++- bin/facts/crm | 4 +- bin/seed-edelweiss-facts.py | 187 ------------------ compose.edelweiss.yml | 24 +-- docs/chat-import-plan.md | 380 +++++++++++++++++++++++++++++++++++- 7 files changed, 403 insertions(+), 262 deletions(-) delete mode 100755 bin/seed-edelweiss-facts.py diff --git a/bin/chats/apply_cmd.go b/bin/chats/apply_cmd.go index a846cf2..1b17127 100644 --- a/bin/chats/apply_cmd.go +++ b/bin/chats/apply_cmd.go @@ -239,7 +239,6 @@ func findOO() string { } candidates := []string{ filepath.Join(os.Getenv("HOME"), "go", "bin", "oo"), - "/home/ano/go/bin/oo", } for _, c := range candidates { if _, err := os.Stat(c); err == nil { diff --git a/bin/chats/sync_cmd.go b/bin/chats/sync_cmd.go index 37f3252..50823d5 100644 --- a/bin/chats/sync_cmd.go +++ b/bin/chats/sync_cmd.go @@ -42,28 +42,17 @@ func runSyncTelegram(args []string) int { return 2 } - // Locate telegram-mcp directory - mcpDirs := []string{ - envVar("TELEGRAM_MCP_DIR", ""), - "/mnt/8TB/projects/eslider/mcp-servers/telegram-mcp", - filepath.Join(os.Getenv("HOME"), "projects", "eSlider", "mcp-servers", "telegram-mcp"), - } - var mcpDir string - for _, d := range mcpDirs { - if d != "" { - if _, err := os.Stat(filepath.Join(d, "main.py")); err == nil { - mcpDir = d - break - } - } - } + mcpDir := envVar("TELEGRAM_MCP_DIR", "") if mcpDir == "" { - fmt.Fprintln(os.Stderr, "chats: telegram-mcp not found; set TELEGRAM_MCP_DIR") + fmt.Fprintln(os.Stderr, "chats: set TELEGRAM_MCP_DIR to telegram-mcp directory") + return 1 + } + if _, err := os.Stat(filepath.Join(mcpDir, "main.py")); err != nil { + fmt.Fprintf(os.Stderr, "chats: TELEGRAM_MCP_DIR=%s: main.py not found\n", mcpDir) return 1 } if sessionStr == "" { - // Fallback: try to read from telegram-mcp .env envPath := filepath.Join(mcpDir, ".env") if data, err := os.ReadFile(envPath); err == nil { for _, line := range strings.Split(string(data), "\n") { @@ -76,20 +65,10 @@ func runSyncTelegram(args []string) int { } } if sessionStr == "" { - // Also try the env file at /mnt/8TB/projects/eslider/mcp-servers/telegram.env - envPath := "/mnt/8TB/projects/eslider/mcp-servers/telegram.env" - if data, err := os.ReadFile(envPath); err == nil { - for _, line := range strings.Split(string(data), "\n") { - if strings.HasPrefix(line, "TELEGRAM_SESSION_STRING=") { - sessionStr = strings.TrimPrefix(line, "TELEGRAM_SESSION_STRING=") - sessionStr = strings.Trim(sessionStr, "\"'") - break - } - } - } + sessionStr = envVar("TELEGRAM_SESSION_STRING", "") } if sessionStr == "" { - fmt.Fprintln(os.Stderr, "chats: TELEGRAM_SESSION_STRING not found in env or .env files") + fmt.Fprintln(os.Stderr, "chats: TELEGRAM_SESSION_STRING not found; set env or in TELEGRAM_MCP_DIR/.env") return 1 } diff --git a/bin/edelweiss-pilot b/bin/edelweiss-pilot index 63d6840..4e9dff8 100755 --- a/bin/edelweiss-pilot +++ b/bin/edelweiss-pilot @@ -6,32 +6,26 @@ cd "$ROOT" export HF_HOME="${HF_HOME:-$ROOT/var/hf}" export PATH="$ROOT/.venv/bin:$PATH" mkdir -p "$HF_HOME" "$ROOT/var" -# Corpus policy (Andriy 2026-08-12): -# 1 cs-lexicon, 2 vendor kbs, 3 reports only (not raw STT), -# 4/5/6 seeded via pairing (seed-edelweiss-facts.py), -# 7 SCHEMA/ARCHITECTURE churn — do not promote to facts, -# 9 glossary+subjects via docs/docs -DOCS=( - --corpus /home/devops/projects/docs/docs - --corpus /home/devops/projects/edelweiss-curasoft/docs - --corpus /home/devops/projects/edelweiss-ui/docs - --corpus /home/devops/projects/edelweiss-curasoft/docs/lexicon - --corpus /home/devops/projects/curasoft/docs/lexicon - --corpus /home/devops/projects/curasoft/docs/curasoft-de/kbs - --corpus /home/devops/projects/docs/docs/reports -) + +# Corpus paths set via env; fallback to PROJECTS_ROOT convention. +PROJECTS="${PROJECTS_ROOT:-/home/devops/projects}" +DOCS_BASE="${DOCS_BASE:-$PROJECTS}" +DOCS=() +for d in \ + "$DOCS_BASE/docs/docs" \ + "$DOCS_BASE/edelweiss-ui/docs" \ + "$DOCS_BASE/docs/docs/reports"; do + [ -d "$d" ] && DOCS+=(--corpus "$d") +done + cmd="${1:-stats}"; shift || true case "$cmd" in - # Write leafs without indexes → seed facts → ensure_indexes (FTS+HNSW). - # Never upsert under live FTS; never DROP INDEX (ghost catalog). index) bin/kb/index --rebuild --skip-indexes "${DOCS[@]}" "$@" - .venv/bin/python bin/seed-edelweiss-facts.py ;; search) exec bin/edelweiss-search "$@" ;; search-go) exec bin/kb/search "$@" ;; stats) exec bin/kb/stats "$@" ;; audit) exec bin/facts/audit "${1:-db}" "${@:2}" ;; - seed) exec .venv/bin/python bin/seed-edelweiss-facts.py "$@" ;; - *) echo "usage: $0 index|search|search-go|stats|audit|seed ..." >&2; exit 2 ;; + *) echo "usage: $0 index|search|search-go|stats|audit ..." >&2; exit 2 ;; esac diff --git a/bin/facts/crm b/bin/facts/crm index 2a0ddbf..964c1b1 100755 --- a/bin/facts/crm +++ b/bin/facts/crm @@ -19,6 +19,7 @@ Usage: from __future__ import annotations import json +import os import sys from pathlib import Path @@ -27,7 +28,8 @@ sys.path.insert(0, str(ROOT / "bin" / "tools")) from kblib import upsert_leaf, connect, leaf_id # noqa: E402 -CORPUS_MESH = Path("/mnt/8TB/projects/eslider/cv/projects/knowledge-mesh-seed.yaml") +MESH_ENV = os.environ.get("KNOWLEDGE_MESH_SEED", "") +CORPUS_MESH = Path(MESH_ENV) if MESH_ENV else ROOT / "../knowledge-mesh-seed.yaml" def corpus_orgs(raw: str) -> dict[str, dict]: diff --git a/bin/seed-edelweiss-facts.py b/bin/seed-edelweiss-facts.py deleted file mode 100755 index b5d17ee..0000000 --- a/bin/seed-edelweiss-facts.py +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env python3 -"""Seed confirmed Edelweiss facts via pairing (a x b). Never DROP INDEX.""" -from __future__ import annotations - -import sys -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "bin" / "tools")) - -import yaml -from kblib import ( # noqa: E402 - CONF_CONFIRMED, - ROOT_FACTS, - connect, - ensure_indexes, - leaf_index_names, - stats, - upsert_leaf, -) -from model2vec import StaticModel # noqa: E402 - -LEX = Path("/home/devops/projects/edelweiss-curasoft/docs/lexicon/edelweiss-lexicon.yml") -CS = Path("/home/devops/projects/curasoft/docs/lexicon/cs-lexicon.yml") - - -def fact(text: str, source: str, how: str, loc: str) -> tuple[str, str, str, str]: - return text, source, how, loc - - -def from_edelweiss_lex(lex: dict) -> list[tuple[str, str, str, str]]: - rows: list[tuple[str, str, str, str]] = [] - gl = lex["hosts"]["gl"] - rows.append(fact( - "CuraSoft GL server VM IP is %s on Fritz LAN (%s)." % (gl["ip"], gl["role"]), - gl["evidence"], "lexicon hosts.gl", "edelweiss-lexicon.yml", - )) - for key in ("pdl", "pdl2", "spdl", "gf"): - h = lex["hosts"][key] - rows.append(fact( - "Edelweiss host %s IP %s role=%s." % (key.upper(), h["ip"], h["role"]), - h["evidence"], "lexicon hosts.%s" % key, "edelweiss-lexicon.yml", - )) - pg, ms = lex["ports"]["postgres"], lex["ports"]["mssql"] - rows.append(fact( - "GL LAN DB: PostgreSQL port %s open=%s; MSSQL %s reachable=%s." - % (pg["port"], pg["reachable_from_lan"], ms["port"], ms["reachable_from_lan"]), - pg["evidence"], "lexicon ports", "edelweiss-lexicon.yml", - )) - b = lex["brain_2dph"] - rows.append(fact( - "Edelweiss 2dph pilot brain at %s; facts need >=2 sources. CLI: %s." - % (b["path"], b["cli"]), - b["evidence"], "lexicon brain_2dph", "edelweiss-lexicon.yml", - )) - v = lex["password_vault"] - rows.append(fact( - "Password vault pilot: %s; OO task #%s deadline %s." - % (v["choice"], v["oo_task"], v["deadline"]), - v["evidence"], "lexicon password_vault", "edelweiss-lexicon.yml", - )) - a = lex["assistant_pilot"] - rows.append(fact( - "Artem consent %s for local Pflege/CuraSoft assistant; scope: %s." - % (a["consent_date"], a["scope"]), - a["evidence"], "lexicon assistant_pilot", "edelweiss-lexicon.yml", - )) - return rows - - -def from_cs_lex(cs: dict) -> list[tuple[str, str, str, str]]: - """Item 1: cs-lexicon → facts only with paired evidence.""" - rows: list[tuple[str, str, str, str]] = [] - src = cs.get("sources") or {} - default_pair = None - if isinstance(src, dict) and len(src) >= 2: - keys = list(src.keys())[:2] - default_pair = "%s x %s" % (keys[0], keys[1]) - ep = cs.get("ep_typ") or {} - for code, meta in ep.items(): - if not isinstance(meta, dict): - continue - ev = (meta.get("evidence") or "").replace("×", " x ") - if " x " not in ev: - if not default_pair: - continue - ev = default_pair - meaning = meta.get("meaning") or meta.get("name") or "" - rows.append(fact( - "CuraSoft ep_typ %s = %s (%s)." % (code, meta.get("name"), meaning), - ev, "cs-lexicon ep_typ", "cs-lexicon.yml", - )) - pair = "cs-lexicon.yml x curasoft-de/kbs" - besuch = cs.get("besuch") or {} - for code, label in besuch.items(): - if code == "note" or not isinstance(label, str): - continue - rows.append(fact( - "CuraSoft Besuch slot %s = %s." % (code, label), - pair, "cs-lexicon besuch", "cs-lexicon.yml", - )) - bm = cs.get("binary_map") or {} - if isinstance(bm, dict) and bm: - rows.append(fact( - "CuraSoft binary_map documents RE code/status mappings for detective (not live PHI).", - "cs-lexicon.yml x curasoft-detective skill", - "cs-lexicon binary_map", "cs-lexicon.yml", - )) - return rows - - -def from_oo_and_interview() -> list[tuple[str, str, str, str]]: - """Items 4+5+6: OO, QEMU/GL, interview bullets via pairing.""" - return [ - fact( - "OO Edelweiss Remote Work #24: Vaultwarden deploy task #431 deadline 2026-08-19; companion #432 deploy DL 2026-08-15.", - "OO#431 x 07-interview-2026-08-12.md", - "OO calendar/tasks pair", "office.produktor.io + reports/07", - ), - fact( - "Fahrplan pilot kickoff agreed for Tuesday after 2026-08-12 interview; OO task #434 deadline 2026-08-18.", - "OO#434 x 07-interview-2026-08-12.md", - "interview x OO", "reports/07 + OO", - ), - fact( - "Local Pflege/CuraSoft assistant pilot: email+docs sync consented; phone/STT ticket flow is phase 2.", - "07-interview-2026-08-12.md x edelweiss-lexicon.yml", - "interview structured bullets", "reports/07", - ), - fact( - "Meta/CuraSoft Wiener Hostel event tentatively 2026-08-18 ~09:30 CEST — confirm name/place with Artem.", - "07-interview-2026-08-12.md x OO calendar event 157", - "interview x calendar", "reports/07 + OO", - ), - fact( - "QEMU/GL safe ops: never HMP quit/reset unless asked; live disk always qemu-img -U; GL VM via dockur acronis-boot.", - "gl-vm-access.md x qemu-monitor-safe rule", - "QEMU/GL access pair", "edelweiss-curasoft/docs + .cursor/rules", - ), - fact( - "2dph indexes interview reports under docs/docs/reports; raw STT docs/stt is not corpus (reports only).", - "2dph-edelweiss-pilot.md x 00-summary.md", - "STT policy pair", "pilot doc + reports", - ), - ] - - -def main() -> int: - lex = yaml.safe_load(LEX.read_text()) - cs = yaml.safe_load(CS.read_text()) if CS.exists() else {} - model = StaticModel.from_pretrained("minishlab/potion-multilingual-128M") - db, conn = connect(read_only=False) - - rows: list[tuple[str, str, str, str]] = [] - rows.extend(from_edelweiss_lex(lex)) - rows.extend(from_cs_lex(cs)) - rows.extend(from_oo_and_interview()) - - for text, source, how, loc in rows: - if " x " not in source: - print("skip (no pair)", how, text[:60]) - continue - emb = model.encode([text])[0].astype(float).tolist() - upsert_leaf( - conn, - text=text, - root=ROOT_FACTS, - confidence=CONF_CONFIRMED, - source=source, - source_rev="2026-08-12", - how=how, - loc=loc, - type_="fact", - embedding=emb, - ) - print("ok", how) - - ensure_indexes(conn) - print("INDEXES", sorted(leaf_index_names(conn))) - print(stats(conn)) - conn.close() - db.close() - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/compose.edelweiss.yml b/compose.edelweiss.yml index eb7aff4..a690727 100644 --- a/compose.edelweiss.yml +++ b/compose.edelweiss.yml @@ -1,14 +1,11 @@ -# 2dph Edelweiss pilot — knowledge brain for Pflegedienst facts/info only. +# 2dph Edelweiss pilot — knowledge brain. # # docker compose -f compose.edelweiss.yml build # docker compose -f compose.edelweiss.yml run --rm brain index --rebuild \ -# --corpus /corpus/docs --corpus /corpus/curasoft-docs \ -# --corpus /corpus/ui-docs --corpus /corpus/vendor-kbs \ -# --corpus /corpus/reports -# docker compose -f compose.edelweiss.yml run --rm brain search "GL IP" +# --corpus /corpus/docs --corpus /corpus/ui-docs --corpus /corpus/reports +# docker compose -f compose.edelweiss.yml run --rm brain search "query" # -# HNSW note: never DROP INDEX Leaf.Leaf_vec on a live DB (ghost catalog). -# Fresh indexes = delete volume/db + --rebuild. See kblib.create_fts_and_vector. +# Host corpus dirs set via env: DOCS_BASE (default: /home/devops/projects) name: 2dph-edelweiss services: @@ -25,18 +22,15 @@ services: KB_WORKERS: "2" KB_PY: python3 KB_ROOT: /app + DOCS_BASE: ${DOCS_BASE:-/home/devops/projects} volumes: - kb-model:/data/hf - kb-var:/data - ./:/app - # Edelweiss host corpus mounts (override paths on other hosts) - - /home/devops/projects/docs/docs:/corpus/docs:ro - - /home/devops/projects/edelweiss-curasoft/docs:/corpus/curasoft-docs:ro - - /home/devops/projects/edelweiss-ui/docs:/corpus/ui-docs:ro - - /home/devops/projects/curasoft/docs/curasoft-de/kbs:/corpus/vendor-kbs:ro - - /home/devops/projects/docs/docs/reports:/corpus/reports:ro - # STT raw mounted read-only for humans — do NOT pass as --corpus - - /home/devops/projects/docs/stt:/corpus/stt:ro + - ${DOCS_BASE:-/home/devops/projects}/docs/docs:/corpus/docs:ro + - ${DOCS_BASE:-/home/devops/projects}/edelweiss-ui/docs:/corpus/ui-docs:ro + - ${DOCS_BASE:-/home/devops/projects}/docs/docs/reports:/corpus/reports:ro + - ${DOCS_BASE:-/home/devops/projects}/docs/stt:/corpus/stt:ro command: ["brain", "search", "help"] read_only: false tmpfs: diff --git a/docs/chat-import-plan.md b/docs/chat-import-plan.md index b37a241..6f9d498 100644 --- a/docs/chat-import-plan.md +++ b/docs/chat-import-plan.md @@ -1,5 +1,6 @@ # Chat Import Pipeline +<<<<<<< Updated upstream Plan and implementation details: https://git.produktor.io/eSlider/brain-chats-import/issues/1 Progress tracked in that issue. This file kept as a local reference for agent sessions. @@ -91,9 +92,17 @@ OO_CLI path to oo binary (default: $HOME/go/bin/oo) ### Rambox Не используется. IndexedDB база не существует. +||||||| Stash base +Created: 2026-08-12 +Author: Andriy Oblivantsev +Purpose: One-shot handoff for a new agent session to implement `bin/chats` +======= +Plan and implementation details: https://git.produktor.io/eSlider/brain-chats-import/issues/1 +>>>>>>> Stashed changes ---- +Progress tracked in that issue. This file kept as a local reference for agent sessions. +<<<<<<< Updated upstream ## 2. Design Decisions ### One binary: `bin/chats` @@ -394,20 +403,135 @@ chats apply // push to OO CRM [--dry-run] ``` ### Directory layout (полные имена платформ) +||||||| Stash base +## What this document is for -``` -var/chats/ - telegram//messages.jsonl - whatsapp//messages.jsonl - linkedin//messages.jsonl - md///message.md +This file captures the full context, research findings, design decisions, and +execution plan for implementing a single Go binary `bin/chats` that syncs +personal 1:1/group chats from Telegram, WhatsApp, and LinkedIn, converts them +to markdown, indexes them into the brain (`var/kb.lbug`), extracts contact +facts (phones, social links, projects, deals, associations), cross-checks +against ≥2 independent sources (detective method), and applies verified facts +to OnlyOffice CRM (update persons, contacts, opportunities, projects). + +Save this file in context for the next agent session. + +--- + +## 1. Находки / Discoveries + +### Доступные платформы и данные + +| Platform | Status | Credentials (path, not values) | Go Library | +|----------|--------|-------------------------------|------------| +| **Telegram** | ✅ Active MCP server, session string exists | API creds in `telegram.env` (see §5). Session string in `telegram-mcp/.env`. | MCP JSON-RPC (через telegram-mcp server) | +| **WhatsApp** | ✅ MCP server exists + Go bridge. **No `messages.db` yet** (empty). Needs QR re-auth + sync first | MCP: `/mnt/8TB/projects/eslider/mcp-servers/whatsapp-mcp/`. Go bridge uses `tulir/whatsmeow` | [`tulir/whatsmeow`](https://github.com/tulir/whatsmeow) (7k★, Go, multi-device API) | +| **LinkedIn** | ✅ Connected via `mcp-server-linkedin`. Full browser profile + cookies | Browser profile: `~/.linkedin-mcp/profile/`. Active cookies: `~/.linkedin-mcp/storage-state.json`. Cookie backup: `~/.linkedin-mcp/cookies.json.bak-agent`. Script cookies: `/mnt/8TB/projects/eslider/cv/bin/creds/linkedin.cookies.json`. JobHunt session: `/mnt/8TB/projects/eslider/JobHunt/sessions/linkedin.json`. Env: `/mnt/8TB/projects/eslider/cv/.env` (`LINKEDIN_AUTH_TOKEN`). | [`swiftlysingh/lnk`](https://github.com/swiftlysingh/lnk) (Go CLI, Voyager API) | +| **Gmail** | ✅ Already working pipeline (`bin/mail/sync.go` + `bin/mail/import`) | OAuth at `~/.gmail-mcp/` | Already in 2dph | +| **Twitter/X.com** | ⚠️ Password only, no MCP, no session. Not needed now. | `/mnt/8TB/projects/eslider/scratches/Accounts/twitter-pass.txt` | [`benoitpetit/xsh`](https://github.com/benoitpetit/xsh) (Go CLI, cookie auth) | +| **Google Calendar** | ❌ Not connected yet. Gmail OAuth could extend | None | Нужен Google Calendar API | + +### Telegram session (проверен, активен) + +- API credentials in: `/mnt/8TB/projects/eslider/mcp-servers/telegram.env` + - `TELEGRAM_API_ID`, `TELEGRAM_API_HASH`, `TELEGRAM_PHONE` — загружаются из env + - `TELEGRAM_SESSION_STRING` — активная Telethon session (в env) +- MCP server launcher: `/mnt/8TB/projects/eslider/mcp-servers/run-telegram-mcp.sh` +- Session generator: `/mnt/8TB/projects/eslider/mcp-servers/telegram-mcp/session_string_generator.py` +- Regenerator: `/mnt/8TB/projects/eslider/mcp-servers/regen-telegram-session.py` + +### LinkedIn authentication (работает) + +- Запускается через Cursor MCP: `uvx mcp-server-linkedin@latest --user-data-dir ~/.linkedin-mcp/profile --no-auto-import` +- Cookie файлы: + - `~/.linkedin-mcp/storage-state.json` — Playwright storage state (активные cookies) + - `~/.linkedin-mcp/cookies.json.bak-agent` — резерв + - `~/.linkedin-mcp/cookies.json.bak-webtop-20260728T143535Z` — резерв + - `/mnt/8TB/projects/eslider/cv/bin/creds/linkedin.cookies.json` — для скриптов + - `/mnt/8TB/projects/eslider/JobHunt/sessions/linkedin.json` — JobHunt +- `li_at` token: в `storage-state.json` (см. §5) +- Для Go подхода: можно читать cookie из `storage-state.json` и использовать LinkedIn Voyager API напрямую + +### WhatsApp bridge (не синхронизирован) + +- Go bridge at: `/mnt/8TB/projects/eslider/mcp-servers/whatsapp-mcp/whatsapp-bridge/` +- Использует `tulir/whatsmeow` Go library (уже в `go.mod`) +- SQLite store: `whatsapp-bridge/store/messages.db` — **не существует** (0 байт), нужна первая авторизация по QR +- Launcher: `/mnt/8TB/projects/eslider/mcp-servers/run-whatsapp-bridge-lharries.sh` +- QR images готовились: `/mnt/8TB/projects/produktor/work/data/export/whatsapp-qr.png` +- Пароль: в `.env` OnlyOffice + +### OnlyOffice доступ + +- `.env` в корне 2dph: `ONLYOFFICE_URL`, `ONLYOFFICE_USER`, `ONLYOFFICE_PASS` +- `oo` CLI установлен: `/home/ano/go/bin/oo` (go-onlyoffice) +- go-onlyoffice repo: `/mnt/8TB/projects/eslider/go-onlyoffice/` (main, public) +- oo-workspace (каталог, private): `/mnt/8TB/projects/eslider/oo-workspace/` + +### Gitea + +- Instance: `https://git.produktor.io` (через `gitea-api` wrapper) +- Credentials: `~/.config/gitea/produktor.env` +- User: `eSlider`, admin +- Repos: 30+ приватных репозиториев +- **`eSlider/2dph` не существует на Gitea** — проект только на GitHub +- Для задачи: создать issue в существующем репо (например `eSlider/JobHunt`) или создать новый репо + +### Rambox + +Не используется. IndexedDB база не существует. + +--- + +## 2. Design Decisions + +### One binary: `bin/chats` + +```go +// Source interface — каждая платформа реализует +type Source interface { + Name() string // "telegram", "whatsapp", "linkedin" + Sync(ctx context.Context, out string) error +} + +// CLI subcommands +chats sync telegram [--limit N] [--since DATE] +chats sync whatsapp [--qr] [--limit N] +chats sync linkedin [--limit N] +chats import // JSONL → markdown (all sources) +chats index // rebuild var/kb.lbug +chats facts // extract + cross-check +chats apply // push to OO CRM [--dry-run] ``` -### JSONL format (одна строка = одно сообщение) +### Directory layout (полные имена платформ) +======= +## Key env vars (set in shell, never committed) +>>>>>>> Stashed changes -```json -{"id":"123","ts":"2026-01-15T10:30:00Z","from":"me","text":"Hello!","media":null,"platform":"telegram"} ``` +TELEGRAM_MCP_DIR telegram-mcp directory +TELEGRAM_API_ID from telegram.env +TELEGRAM_API_HASH from telegram.env +TELEGRAM_PHONE from telegram.env +TELEGRAM_SESSION_STRING from telegram-mcp/.env +ONLYOFFICE_URL from .env +ONLYOFFICE_USER from .env +ONLYOFFICE_PASS from .env +KNOWLEDGE_MESH_SEED path to knowledge-mesh-seed.yaml (for facts/crm) +OO_CLI path to oo binary (default: $HOME/go/bin/oo) +``` + +## Quick reference + +```bash +./bin/chat sync telegram --limit 100 +./bin/chat import +./bin/chat index # delegates to bin/kb/index +./bin/chat facts +./bin/chat apply --dry-run +``` +<<<<<<< Updated upstream ### MD format (YAML frontmatter) @@ -642,3 +766,239 @@ type: personal 4. Telegram session reuse через MCP server (уже работает) 5. Apply через `bin/chats apply` (уже работает, улучшен поиск контактов) >>>>>>> 313599b (bin/chats: Phase 1 MVP — Telegram sync/import/index/facts/apply) +||||||| Stash base + +### MD format (YAML frontmatter) + +```markdown +--- +id: tg_12345 +platform: telegram +chat_id: "-100123456" +chat_name: "John Doe" +participants: ["me", "John Doe"] +message_count: 1500 +type: personal +--- + +# Чат с John Doe + +**2026-01-15 10:30** — me: Hello! +``` + +### Фильтр "личный чат" + +- ≤3 участников +- Боты не считаются (Telegram: `user.is_bot == false`) +- Не канал, не публичная группа +- Telegram: `dialog.Type == "user"` или супергруппа с `participants_count <= 3` (исключая ботов) +- WhatsApp: `@s.whatsapp.net` (не `@g.us`) +- LinkedIn: 1:1 conversation + +### Очередность реализации + +1. **Telegram** — самый простой (session string + API ID/Hash активны) +2. **Import** — конвертер JSONL → MD +3. **Index** — интеграция с brain +4. **WhatsApp** — после первого QR +5. **Facts** — извлечение контактных данных +6. **LinkedIn** — после проверки cookie +7. **Apply** — запись в OO CRM + +### Тестирование (TDD, system tests only) + +- Никаких unit-тестов, никаких моков +- Тесты используют синтетические данные (слепки) +- Workflow use-case как тест: + 1. Написать тест с реальным сценарием (sync → import → index → search) + 2. Проверить что файлы созданы + 3. Проверить что поиск возвращает ожидаемые результаты +- A/B тест: сравнить два последовательных sync на идентичность +- Integration: тест через `oo` CLI с `--dry-run` + +### Cross-check (detective method) + +Перед записью в OO CRM — минимум 2 независимых источника: +- S1: Чат (текст сообщения, автор, дата) +- S2: OO CRM (существующий контакт/компания/deal) через `oo persons list` / `oo persons get` +- S3 (опционально): Corpus SoT (`knowledge-mesh-seed.yaml`) или web-search +- Факты с ≥2 источниками → `root=facts` в brain + `approve=true` для CRM +- Факты с 1 источником → предложение на review (catalog-паттерн) + +### Apply в OO CRM + +Через `oo` CLI: +- `oo persons update --about ... --job-title ...` +- `oo contacts info-add --type Phone|LinkedIn --value ...` +- `oo opportunities create --contact-id ... --title ... --stage ...` +- `oo persons update --company-id ...` (ассоциация) +- `oo projects contacts ` (связь с проектом) + +--- + +## 3. Implementation Plan + +### Phase 1: Telegram sync + import (MVP) + +1. Create `bin/chats/` directory structure (nested Go module like `bin/kbsearch/`) +2. Implement `Source` interface +3. Implement `TelegramSource` — MCP JSON-RPC клиент к telegram-mcp +4. Write JSONL output to `var/chats/telegram//messages.jsonl` +5. Implement `chats import` — reads all JSONL, writes MD +6. Write system test: sync → import → verify output + +### Phase 2: Brain indexing + +1. Implement `chats index` — rebuild `var/kb.lbug` with corpus + chats +2. Write system test: sync → import → index → `bin/kb/search "query"` → verify recall + +### Phase 3: WhatsApp + +1. Implement `WhatsAppSource` using `tulir/whatsmeow` +2. First run requires QR auth (save session for later reuse) +3. Write sync output to `var/chats/whatsapp//messages.jsonl` + +### Phase 4: Facts extraction + cross-check + +1. Implement `chats facts`: + - Phone regex: `\+?\d{7,15}` + validation (exclude dates/amounts/cards) + - LinkedIn URL regex: `linkedin\.com/in/[\w-]+` + - Cross-check: match extracted values against OO CRM API +2. Write facts to brain as `root=facts` + +### Phase 5: LinkedIn + Apply + +1. Implement `LinkedInSource` using Voyager API + cookies from `storage-state.json` +2. Implement `chats apply`: + - Build YAML catalog (as `oo-workspace` does) + - Run `oo catalog match` → review → `oo catalog apply` + +--- + +## 4. Existing code references + +| Code | Path | Purpose | +|------|------|---------| +| Mail sync | `bin/mail/sync.go` + `bin/mail/sync/` | Pattern for chat sync (Go subprocess pattern) | +| Mail import | `bin/mail/import` | Pattern for JSONL → MD conversion | +| Mail index | `bin/mail/index_mail` | Pattern for brain rebuild with new corpus | +| kbsearch | `bin/kbsearch/` | Go module pattern (nested, `system_ladybug` tag) | +| kb/index | `bin/kb/index` | Python brain index (upsert_leaf, init_schema) | +| kblib | `bin/tools/kblib.py` | Python brain core (connect, upsert_leaf, hybrid_search) | +| facts/crm | `bin/facts/crm` | Cross-check CRM facts pattern | +| facts/extract | `bin/facts/extract` | 2-source fact extraction pattern | +| oo-workspace catalog | `/mnt/8TB/projects/eslider/oo-workspace/catalog/` | Catalog scan/match/apply pattern (VCF, Thunderbird, projects) | +| WhatsApp bridge | `/mnt/8TB/projects/eslider/mcp-servers/whatsapp-mcp/whatsapp-bridge/` | Existing `whatsmeow` Go bridge | +| go-onlyoffice | `/mnt/8TB/projects/eslider/go-onlyoffice/` | OO CRM API library | +| oo CLI | `/home/ano/go/bin/oo` | Built go-onlyoffice CLI | + +--- + +## 5. Critical files (paths only, no secrets) + +| File | Content | +|------|---------| +| `~/.config/gitea/produktor.env` | Gitea URL + token | +| `/mnt/8TB/projects/eslider/mcp-servers/telegram.env` | Telegram API ID/Hash + phone + session | +| `/mnt/8TB/projects/eslider/mcp-servers/telegram-mcp/.env` | Telegram session string | +| `/mnt/8TB/projects/ai/2dph/.env` | ONLYOFFICE URL/USER/PASS | +| `/mnt/8TB/projects/eslider/cv/.env` | LinkedIn auth token | +| `~/.linkedin-mcp/storage-state.json` | LinkedIn active cookies | +| `/mnt/8TB/projects/eslider/go-onlyoffice/.env` | OO creds for oo CLI | +| `/mnt/8TB/projects/eslider/go-onlyoffice/go.mod` | Deps for go-onlyoffice | +| `/mnt/8TB/projects/eslider/oo-workspace/go.mod` | Deps for oo-workspace | + +--- + +## 6. Agent prompt (для запуска новой сессии) + +Скопируйте этот промпт при старте новой сессии: + +``` +Ты — ассистент для разработки chat import pipeline в проекте 2dph (eSlider/2dph). + +Прочитай docs/chat-import-plan.md полностью. Это план, находки и дизайн. + +Задача: реализовать bin/chats — единый Go бинарник для sync/import/index/facts/apply +личных чатов из Telegram, WhatsApp, LinkedIn с последующей записью фактов в OnlyOffice CRM. + +Правила: +1. TDD first — workflow use-case как тест. Синтетические данные (Alice, Bob). +2. Go, всё в один бинарник bin/chats. +3. Source interface — каждая платформа плагином. +4. var/chats/telegram/ — полные имена платформ. +5. Начни с Telegram (session string активна, через MCP server). +6. После реализации — создай/обнови тесты. +7. Сохрани прогресс в docs/chat-import-plan.md. +8. Запроси разрешение перед apply в OO CRM. +9. НИКАКИХ реальных имён, телефонов, email в коммитах — только ссылки на env файлы. + +Ключевые пути: +- /mnt/8TB/projects/ai/2dph/ — корень проекта +- /mnt/8TB/projects/eslider/mcp-servers/telegram-mcp/ — Telegram MCP +- /mnt/8TB/projects/eslider/mcp-servers/telegram.env — API ID/Hash + session (не коммитить!) +- /mnt/8TB/projects/eslider/go-onlyoffice/ — go-onlyoffice библиотека +- /home/ano/go/bin/oo — OO CLI + +Готов начать. +``` + +--- + +## 7. Implementation Progress + +### Session: 2026-08-12 — Phase 1 complete, sync проверен с реальными данными + +**Результаты тестового синка (--limit=100, 31 личный чат):** +- Подключение к Telegram MCP server: 1.5с +- Получен 31 личный чат (отфильтровано боты + Telegram service) +- Синк всех чатов: 7.7с (1 flood wait на 3с, обработан MCP сервером) +- Записано 922 сообщения в JSONL +- Import: 30 чатов в MD (1 пустой — Saved Messages) +- Факты: 14 phone + 2 email после фильтрации + +**Ключевое изменение: `gotd/td` → Telegram MCP Server** +- Вместо прямого gotd/td подключения (FLOOD_WAIT из-за 5 уже запущенных MCP серверов) +- Используется MCP JSON-RPC (JSONL, `\n`-delimited, **не** Content-Length headers) +- Переиспользует существующий Telethon session string (без phone auth) +- Стартует транзиентный MCP сервер на время синка, убивает после +- Инструменты: `list_chats` + `get_history` через MCP protocol v1.8+ + +**Файлы:** + +| Файл | Роль | +|------|------| +| `bin/chats/mcpclient.go` | MCP JSON-RPC client + `TelegramMCPSource` | +| `bin/chats/sync_cmd.go` | CLI: `chats sync telegram --limit N` | +| `bin/chats/import_cmd.go` | JSONL → MD с YAML frontmatter | +| `bin/chats/index_cmd.go` | Делегирует `bin/kb/index --corpus` | +| `bin/chats/facts_cmd.go` | Regex extraction (phone/email/linkedin) с валидацией | +| `bin/chats/apply_cmd.go` | OO CRM cross-check + apply через `oo` CLI + --dry-run | +| `bin/chat` | build+exec wrapper (как `bin/kb/search`) | + +**Phone regex улучшен:** +- Фильтр: исключает даты (`2026-06-14`), суммы (`2 500 000`), номера карт (16 digits), инвойсы (`25-002332001`) +- Валидация: >=7 digits, <=15 digits, без `000` pattern, не дата +- Результат: 37 → 16 фактов (14 phone + 2 email) + +**Cross-check + Apply в OO CRM (без персональных данных в файлах):** +- Найден существующий контакт через first name: добавлен phone + email +- Создан новый контакт: phone добавлен +- Apply через `oo contacts info-add --type Phone|Email --value ...` + +**Brain:** 132 leafs (120 info + 12 facts), чаты заиндексены + +**Безопасность:** +- `var/` в `.gitignore` — сырые чат-данные не коммитятся +- credentials в env файлах, не в коде +- Тесты используют синтетические данные (Alice, Bob, Charlie, Diana) +- Этот документ не содержит реальных значений API ключей, токенов, паролей или персональных данных третьих лиц + +**Что дальше (следующая сессия):** +1. WhatsApp source (`tulir/whatsmeow`) — после первого QR +2. LinkedIn source (Voyager API через cookies) +3. Full history sync (без --limit) для всех чатов +4. Telegram session reuse через MCP server (уже работает) +5. Apply через `bin/chats apply` (уже работает, улучшен поиск контактов) +======= +>>>>>>> Stashed changes