#!/usr/bin/env python3
"""kb/index - build the 2dph brain from markdown + factual leafs.

  bin/kb/index [--corpus DIR] [--rebuild] [--limit N]
  bin/kb/index --rebuild --with-facts --with-chats
  bin/kb/index --json                 # emit stats as JSON

Reads every .md under the corpus (default: repo root docs, skills, READMEs)
as `info` leafs, embeds them with model2vec (potion-multilingual-128M), and
writes them into var/kb.lbug with FTS + HNSW indexes. `facts` leafs come
from bin/facts/extract (docker × compose × ssh-config pairing) when
`--with-facts` is set. `--with-chats` indexes markdown under var/chats/md
(or a given dir) as info. WhatsApp sync stays out of v1.

--rebuild drops the database file and indexes from scratch. Without it a run
is idempotent (MERGE by (source,text) id).
"""
from __future__ import annotations

import json
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "bin" / "tools"))

from kblib import (  # noqa: E402
    add_leafs, connect, ensure_indexes, init_schema, upsert_leaf, link_from_file,
    open_readonly, stats,
)
from mdleaves import read_markdown, to_all, walk_markdown  # noqa: E402
from mailleafs import from_mail_root  # noqa: E402

CORPUS_DEFAULTS = ["README.md", "PLAN.md", "AGENTS.md", "docs", "skills"]


def load_corpus(root: Path) -> list[dict]:
    files: list[Path] = []
    for entry in CORPUS_DEFAULTS:
        p = root / entry
        if p.is_file():
            files.append(p)
        elif p.is_dir():
            files.extend(walk_markdown(p))
    leafs: list[dict] = []
    for path in files:
        try:
            leafs.extend(to_all(read_markdown(path), path, repo="eSlider/2dph"))
        except OSError as e:
            print(f"kb/index: skip {path}: {e}", file=sys.stderr)
    return leafs


def load_corpus_glob(source: str) -> list[dict]:
    """Add arbitrary markdown dirs/files as corpus roots (repo=dirname)."""
    root = Path(source)
    if not root.exists():
        print(f"kb/index: skip missing corpus {source}", file=sys.stderr)
        return []
    files = [root] if root.is_file() else walk_markdown(root)
    repo = root.name if root.is_dir() else root.parent.name
    leafs: list[dict] = []
    for path in files:
        try:
            leafs.extend(to_all(read_markdown(path), path, repo=repo))
        except OSError as e:
            print(f"kb/index: skip {path}: {e}", file=sys.stderr)
    # yaml seeds (knowledge-mesh, workspace catalogs) as plain info leafs
    if root.is_dir():
        for path in sorted(root.rglob("*.y*ml")):
            try:
                leafs.append({
                    "source": str(path), "repo": repo, "heading": path.stem,
                    "text": path.read_text(encoding="utf-8", errors="replace")[:20000],
                    "type": "seed", "status": "current", "related": "",
                })
            except OSError:
                continue
    return leafs


def index_leafs(conn, leafs: list[dict], embed_fn, limit: int) -> tuple[int, int]:
    count = 0
    for lf in leafs[:limit] if limit else leafs:
        query = f"{lf['heading']}\n\n{lf['text']}"
        emb = embed_fn(lf["text"]) if lf["text"] else None
        lid = upsert_leaf(conn, text=query, root="info", confidence="confirmed",
                    source=lf["source"], source_rev="working-tree",
                    how="kb/index", loc=lf["source"], type_=lf.get("type", "reference"),
                    embedding=emb)
        link_from_file(conn, lid, lf["source"], repo=str(lf.get("repo") or ""))
        count += 1
    return count, len(leafs)


def embedder():
    from model2vec import StaticModel
    model = StaticModel.from_pretrained("minishlab/potion-multilingual-128M")
    return lambda text: model.encode([text])[0].astype(float).tolist()


def index_fact_dicts(conn, facts: list[dict], embed_fn) -> int:
    """Write extract-shaped dicts as root=facts leafs (2-source source field)."""
    leafs = []
    for f in facts:
        text = str(f.get("text") or "")
        source = str(f.get("source") or "")
        if not text or not source:
            continue
        leafs.append({
            "text": text,
            "root": "facts",
            "confidence": "confirmed",
            "source": source,
            "source_rev": f.get("source_rev") or "working-tree",
            "how": f.get("how") or "facts/extract",
            "loc": f.get("loc") or source,
            "type": "fact",
            "embedding": embed_fn(text) if text else None,
        })
    return len(add_leafs(conn, leafs))


def facts_from_extract() -> list[dict]:
    import subprocess
    proc = subprocess.run(
        [sys.executable, str(ROOT / "bin" / "facts" / "extract"), "--json", "--dry-run"],
        cwd=ROOT,
        capture_output=True,
        text=True,
        check=False,
    )
    if proc.returncode != 0:
        print(f"kb/index: facts/extract failed: {proc.stderr}", file=sys.stderr)
        return []
    try:
        payload = json.loads(proc.stdout)
    except json.JSONDecodeError:
        print("kb/index: facts/extract produced non-JSON", file=sys.stderr)
        return []
    return list(payload.get("facts") or [])


def main(argv: list[str]) -> int:
    import argparse
    p = argparse.ArgumentParser(description="build the 2dph brain index")
    p.add_argument("--corpus", action="append", help="extra markdown dir/file to index (may repeat)")
    p.add_argument("--rebuild", action="store_true", help="fresh db + indexes")
    p.add_argument("--db", default="", help="path to kb.lbug (default var/kb.lbug)")
    p.add_argument("--no-defaults", action="store_true", help="do not index repo README/docs/skills")
    p.add_argument("--with-mail", action="store_true", help="include var/mail message.md leafs")
    p.add_argument("--with-facts", action="store_true", help="run facts/extract into root=facts")
    p.add_argument("--facts-json", default="", help="JSON list (or {facts:[...]}) of fact dicts")
    p.add_argument(
        "--with-chats",
        nargs="?",
        const=str(ROOT / "var" / "chats" / "md"),
        default="",
        help="index chat markdown as info (default var/chats/md)",
    )
    p.add_argument("--since", default="", help="with --with-mail, only messages dated >= YYYY-MM-DD")
    p.add_argument("--dry-run", action="store_true", help="count leafs, write nothing")
    p.add_argument(
        "--skip-indexes",
        action="store_true",
        help="write leafs only; caller runs ensure_indexes after seeding facts",
    )
    p.add_argument("--limit", type=int, default=0, help="max leafs to embed")
    p.add_argument("--json", action="store_true")
    a = p.parse_args(argv)

    from kblib import DB_PATH, VAR

    dbpath = Path(a.db) if a.db else DB_PATH
    leafs: list[dict] = [] if a.no_defaults else load_corpus(ROOT)
    if a.corpus:
        for source in a.corpus:
            leafs.extend(load_corpus_glob(source))
    chat_n = 0
    if a.with_chats:
        chats = load_corpus_glob(a.with_chats)
        chat_n = len(chats)
        leafs.extend(chats)
    mail_n = 0
    if a.with_mail:
        mail = from_mail_root(ROOT / "var" / "mail", since=a.since)
        mail_n = len(mail)
        leafs.extend(mail)

    facts: list[dict] = []
    if a.facts_json:
        raw = Path(a.facts_json).read_text(encoding="utf-8")
        payload = json.loads(raw)
        facts = list(payload.get("facts") if isinstance(payload, dict) else payload)
    if a.with_facts:
        facts.extend(facts_from_extract())

    if a.dry_run:
        msg = {
            "indexed": 0,
            "corpus_total": len(leafs),
            "mail_leafs": mail_n,
            "chat_leafs": chat_n,
            "facts_leafs": len(facts),
            "dry_run": True,
        }
        print(json.dumps(msg, indent=2) if a.json else
              f"brain/index: {len(leafs)} info + {len(facts)} facts would be indexed")
        return 0

    VAR.mkdir(exist_ok=True)
    dbpath.parent.mkdir(parents=True, exist_ok=True)
    if a.rebuild and dbpath.exists():
        dbpath.unlink()

    db, conn = connect(dbpath, read_only=False)
    init_schema(conn)

    embed = embedder()
    done, total = index_leafs(conn, leafs, embed, a.limit)
    fact_n = index_fact_dicts(conn, facts, embed) if facts else 0
    if not a.skip_indexes:
        ensure_indexes(conn)
    s = stats(conn)
    conn.close()
    db.close()

    result = {
        "indexed": done,
        "corpus_total": total,
        "facts_leafs": fact_n,
        "chat_leafs": chat_n,
        **{k: v for k, v in s.items() if k in ("total", "by_root")},
    }
    if a.skip_indexes:
        result["indexes"] = "skipped"
    print(json.dumps(result, indent=2) if a.json else
          f"indexed {done}/{total} info + {fact_n} facts; db total {s['total']}")
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))