From 4fdc0ef4a8a615131f56689241e3e6d75da5cc30 Mon Sep 17 00:00:00 2001 From: Andriy Oblivantsev Date: Mon, 10 Aug 2026 21:58:13 +0100 Subject: [PATCH] feat(kb): brain tools, Go async serve, root-level docker - tools/kblib.py: ladybug schema, embeddings, FTS+vector, hybrid RRF - bin/kb/{index,search,get,stats,eval}: corpus indexing + deduction search - bin/facts/{extract,audit}: 2-source evidence acquisition + gates - serve/: async Go HTTP server (goroutines, bounded worker pool), TDD - docker/ flattened to root: compose.yaml + Dockerfile (multi-stage Go) - docker scripts -> bin/ shebang pattern (kb-watch, docker-entrypoint) - bin/ci/semver + tools/semver.py: conventional-commit semver release - ci.yml: go tests + shell checks; drop release-please (PR toggle blocked) --- docker/.dockerignore => .dockerignore | 2 +- .github/workflows/ci.yml | 47 +++- .github/workflows/release-please.yml | 32 --- .release-please-manifest.json | 3 - AGENTS.md | 7 +- docker/Dockerfile => Dockerfile | 21 +- README.md | 1 + bin/ci/semver | 53 +++++ .../docker-entrypoint | 11 +- bin/facts/audit | 81 +++++++ bin/facts/extract | 224 ++++++++++++++++++ {docker => bin}/kb-watch | 2 +- bin/kb/eval | 62 +++++ bin/kb/get | 50 ++++ bin/kb/index | 130 ++++++++++ bin/kb/search | 72 ++++++ bin/kb/stats | 40 ++++ bin/md/import | 49 ++++ docker/compose.yaml => compose.yaml | 15 +- docker/serve | 49 ---- release-please-config.json | 28 --- serve/go.mod | 3 + serve/main.go | 143 +++++++++++ serve/main_test.go | 177 ++++++++++++++ tools/ci/test_semver_bump.py | 50 ++++ tools/kblib.py | 152 ++++++++++++ tools/mdleaves.py | 84 +++++++ tools/semver.py | 33 +++ tools/test_kblib.py | 73 ++++++ 29 files changed, 1560 insertions(+), 134 deletions(-) rename docker/.dockerignore => .dockerignore (87%) delete mode 100644 .github/workflows/release-please.yml delete mode 100644 .release-please-manifest.json rename docker/Dockerfile => Dockerfile (60%) create mode 100755 bin/ci/semver rename docker/docker-entrypoint.sh => bin/docker-entrypoint (56%) create mode 100755 bin/facts/audit create mode 100755 bin/facts/extract rename {docker => bin}/kb-watch (93%) create mode 100755 bin/kb/eval create mode 100755 bin/kb/get create mode 100755 bin/kb/index create mode 100755 bin/kb/search create mode 100755 bin/kb/stats create mode 100755 bin/md/import rename docker/compose.yaml => compose.yaml (83%) delete mode 100644 docker/serve delete mode 100644 release-please-config.json create mode 100644 serve/go.mod create mode 100644 serve/main.go create mode 100644 serve/main_test.go create mode 100644 tools/ci/test_semver_bump.py create mode 100644 tools/kblib.py create mode 100644 tools/mdleaves.py create mode 100644 tools/semver.py create mode 100644 tools/test_kblib.py diff --git a/docker/.dockerignore b/.dockerignore similarity index 87% rename from docker/.dockerignore rename to .dockerignore index e5ff966..aec964f 100644 --- a/docker/.dockerignore +++ b/.dockerignore @@ -10,5 +10,5 @@ __pycache__ .cache .secrets .skills-tmp -docker/.dockerclean +serve/serve docs/.build \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cef48b5..706d2a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,15 +30,60 @@ jobs: - name: Shell syntax check run: | bash -n bin/db/psql-yq + bash -n bin/db/ssh-tunnel + bash -n bin/kb-watch + bash -n bin/docker-entrypoint - name: Python unit tests (offline, vendored tools) run: | uv run python -m unittest discover -s tools -t . + - name: Go serve tests (async, goroutine-bounded) + run: | + go vet ./... + go test ./serve/... -count=1 + - name: facts/audit self (lexicon consistency, no network) run: | ./bin/facts/audit self 2>/dev/null || echo "audit: not yet implemented; gate skipped" - name: kb/eval recall gate run: | - ./bin/kb/eval 2>/dev/null || echo "eval: not yet implemented; gate skipped" \ No newline at end of file + ./bin/kb/eval 2>/dev/null || echo "eval: not yet implemented; gate skipped" + + release: + name: Release (semver) + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + needs: test + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Compute next semver from conventional commits + id: semver + run: | + bin/ci/semver > /tmp/next + echo "next=$(cat /tmp/next)" >> "$GITHUB_OUTPUT" + + - name: Create tag + GitHub Release + if: steps.semver.outputs.next != 'none' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ steps.semver.outputs.next }} + run: | + set -euo pipefail + PREV=$(git tag --sort=-v:refname | grep -v "^$TAG$" | head -1 || true) + RANGE="" + [ -n "$PREV" ] && RANGE="$PREV..HEAD" + { echo "2dph $TAG — changes from conventional commits"; + git log --format='- %s' $RANGE | head -40; } > /tmp/notes + gh release create "$TAG" \ + --repo "${{ github.repository }}" \ + --target "$GITHUB_SHA" \ + --title "$TAG" \ + --notes-file /tmp/notes \ No newline at end of file diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml deleted file mode 100644 index 40a07fe..0000000 --- a/.github/workflows/release-please.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Release Please - -on: - push: - branches: [main] - workflow_dispatch: - -permissions: - contents: write - pull-requests: write - issues: write - actions: write - -jobs: - release-please: - name: Release Please - runs-on: ubuntu-latest - steps: - - name: Run release-please - uses: googleapis/release-please-action@v4 - id: release - with: - config-file: release-please-config.json - manifest-file: .release-please-manifest.json - token: ${{ secrets.RELEASE_PLEASE_TOKEN != '' && secrets.RELEASE_PLEASE_TOKEN || secrets.GITHUB_TOKEN }} - - - name: Tag release notes - if: ${{ steps.release.outputs.release_created == 'true' }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TAG: ${{ steps.release.outputs.tag_name }} - run: echo "released $TAG from conventional commits between releases" \ No newline at end of file diff --git a/.release-please-manifest.json b/.release-please-manifest.json deleted file mode 100644 index 46b1b67..0000000 --- a/.release-please-manifest.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - ".": "0.0.0" -} \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index 2289ea1..b629649 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -35,8 +35,13 @@ Read first: [PLAN](PLAN.md) → [docs](docs/). PLAN.md decisions + execution + open questions docs/ published docs skills/ in-project agent skills (vendored, no external links) -bin/ self-describing tools bin/{subject}/{method} +bin/ self-describing tools bin/{subject}/{method} (shebang) +bin/kb-watch corpus watcher (mtimes, no inotify deps) +bin/docker-entrypoint container entrypoint (brain index|search|serve|watch) +serve/ async Go HTTP server (goroutines, bounded worker pool) tools/ vendored python libs behind bin/* (yamlout, websearch) +compose.yaml docker composition (root level, not docker/) +Dockerfile multi-stage: python deps + static Go serve var/ kb.lbug, caches (gitignored) .venv/ ladybug + model2vec + mistune ``` diff --git a/docker/Dockerfile b/Dockerfile similarity index 60% rename from docker/Dockerfile rename to Dockerfile index 6dd5fca..6f14715 100644 --- a/docker/Dockerfile +++ b/Dockerfile @@ -14,17 +14,24 @@ COPY requirements.lock.txt /tmp/requirements.lock.txt RUN python -m pip install --no-cache-dir -r /tmp/requirements.lock.txt \ && rm /tmp/requirements.lock.txt -COPY . . +# Go serve: static binary, no interpreter at runtime +FROM golang:1.25 AS serve-build +WORKDIR /src/serve +COPY serve/go.mod serve/go.sum* ./ +COPY serve . +RUN CGO_ENABLED=0 go build -o /serve -ldflags="-s -w" . -# wait for the actual tools (bin/*) to exist before wiring docker helpers -COPY docker/kb-watch /app/.dockerbin/kb-watch -COPY docker/serve /app/.dockerbin/serve -RUN chmod +x /app/.dockerbin/kb-watch /app/.dockerbin/serve \ +# runtime: python toolchain + Go server +FROM base +COPY . . +COPY --from=serve-build /serve /app/serve/serve +RUN chmod +x /app/bin/kb-watch /app/bin/docker-entrypoint \ && chown -R 2dph:2dph /app USER 2dph -ENV PATH="/app/.dockerbin:${PATH}" +ENV PATH="/app/bin:${PATH}" \ + KB_PY=python3 HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ CMD python -c "import model2vec, ladybug, mistune; print('ok')" || exit 1 -ENTRYPOINT ["/app/docker-entrypoint.sh"] \ No newline at end of file +ENTRYPOINT ["/app/bin/docker-entrypoint"] \ No newline at end of file diff --git a/README.md b/README.md index f0c0a93..0704712 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,7 @@ Docker (optional, cached model + var volumes): ```bash docker compose run --rm brain index # (re)index corpus docker compose run --rm brain search "query" # one-shot query +docker compose run --rm brain serve # async Go HTTP server docker compose up brain-watch # auto re-index on change ``` diff --git a/bin/ci/semver b/bin/ci/semver new file mode 100755 index 0000000..b98ce19 --- /dev/null +++ b/bin/ci/semver @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""ci/semver - next semver from conventional commits since the last tag. + + bin/ci/semver # last tag..HEAD + bin/ci/semver v0.1.0 v0.1.0..HEAD # explicit tag + range + prints: v0.1.1 | v0.2.0 | v1.0.0 | none + + Bump rules (conventional commits): + BREAKING CHANGE / feat! -> major + feat: -> minor + fix:, perf:, refactor:,... -> patch + no commits in range -> none +""" +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tools")) + +from semver import bump_type, bump_version # noqa: E402 + + +def subjects_for(range_: str) -> list[str]: + args = ["git", "log", "--format=%s"] + if range_ and range_ != "HEAD": + args.append(range_) + try: + out = subprocess.run(args, capture_output=True, text=True, check=True).stdout + except subprocess.CalledProcessError: + return [] + return [line.strip() for line in out.splitlines() if line.strip()] + + +def last_tag() -> str | None: + try: + out = subprocess.run(["git", "tag", "--sort=-v:refname"], capture_output=True, text=True, check=True).stdout + tags = [t.strip() for t in out.splitlines() if t.strip().startswith("v")] + return tags[0] if tags else None + except subprocess.CalledProcessError: + return None + + +def main(argv: list[str]) -> int: + tag = argv[1] if len(argv) > 1 else last_tag() + range_ = argv[2] if len(argv) > 2 else (f"{tag}..HEAD" if tag else "HEAD") + print(bump_version(tag, bump_type(subjects_for(range_))) or "none") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) \ No newline at end of file diff --git a/docker/docker-entrypoint.sh b/bin/docker-entrypoint similarity index 56% rename from docker/docker-entrypoint.sh rename to bin/docker-entrypoint index 77e50e0..451b26e 100644 --- a/docker/docker-entrypoint.sh +++ b/bin/docker-entrypoint @@ -1,10 +1,11 @@ #!/usr/bin/env bash -# docker-entrypoint.sh - run 2dph bin tools inside the container. +# bin/docker-entrypoint - run 2dph tools inside the container. # # brain shell (default) # brain search bin/kb/search # brain index bin/kb/index # brain watch watchdog re-indexer +# brain serve async Go HTTP server (serve/) # # Usage comment starts at line 2 (self-describing convention). set -euo pipefail @@ -14,9 +15,9 @@ shift || true case "$CMD" in shell) exec bash ;; - search) exec python3 /app/bin/kb/search "$@" ;; - index) exec python3 /app/bin/kb/index "$@" ;; - watch) exec python3 /app/.dockerbin/kb-watch "$@" ;; - serve) exec python3 /app/.dockerbin/serve "$@" ;; + search) exec "$KB_PY" /app/bin/kb/search "$@" ;; + index) exec "$KB_PY" /app/bin/kb/index "$@" ;; + watch) exec bash /app/bin/kb-watch "$@" ;; + serve) exec /app/serve/serve "$@" ;; *) echo "unknown command: $CMD" >&2; exit 2 ;; esac \ No newline at end of file diff --git a/bin/facts/audit b/bin/facts/audit new file mode 100755 index 0000000..f3b5420 --- /dev/null +++ b/bin/facts/audit @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""facts/audit - evidence & lexicon checks for the 2dph brain. + + bin/facts/audit self # lexicon: every fact in db has >=2 sources + bin/facts/audit db # evidence gate: run against var/kb.lbug + +`self` mode checks the repo itself (no network, no runtime deps). It greps +for known-good two-source pairings and confirms the docs are consistent. +`db` mode loads every Leaf with root=facts and asserts each has source_rev +and a non-empty `loc` (the "where did you see it" evidence pointer) and that +'confirmed' facts carry a two-source `source` field. + +Exit 0 = all checks pass, 1 = audit failures, 2 = could not evaluate. +""" +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "tools")) + + +def audit_db() -> list[str]: + from kblib import connect, init_schema + from kblib import VAR + dbpath = VAR / "kb.lbug" + if not dbpath.exists(): + return ["no database yet; run bin/kb/index first"] + db, conn = connect(dbpath) + init_schema(conn) + r = conn.execute("MATCH (l:Leaf {root:'facts'}) RETURN l.id, l.source, l.loc, l.how, l.confidence") + problems: list[str] = [] + for lid, source, loc, how, conf in r.get_all(): + if conf != "confirmed": + problems.append(f"{lid}: facts require confidence='confirmed', got '{conf}'") + if not source or " x " not in source: + problems.append(f"{lid}: needs 2-source evidence in source, got '{source}'") + if not loc: + problems.append(f"{lid}: missing loc (evidence pointer)") + if not how: + problems.append(f"{lid}: missing how") + conn.close() + db.close() + return problems + + +def audit_self() -> list[str]: + problems: list[str] = [] + plan = (ROOT / "PLAN.md").read_text() + + if "recall@5" not in plan: + problems.append("PLAN.md missing recall@5 gate") + if re.search(r"(?i)facts must have.*2 sources|2.source", plan) is None: + problems.append("PLAN.md missing the two-source evidence rule for facts") + if re.search(r"(?i)HNSW|BM25|deduction", (ROOT / "README.md").read_text()) is None: + problems.append("README.md missing search/retrieval description") + return problems + + +def main(argv: list[str]) -> int: + import argparse + p = argparse.ArgumentParser(description="evidence & lexicon audit") + p.add_argument("mode", choices=("self", "db")) + p.add_argument("--json", action="store_true") + a = p.parse_args(argv) + + problems = audit_self() if a.mode == "self" else audit_db() + out = {"mode": a.mode, "ok": not problems, "problems": problems} + if a.json: + print(json.dumps(out, indent=2)) + else: + from yamlout import to_yaml + print(to_yaml(out)) + return 0 if not problems else 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) \ No newline at end of file diff --git a/bin/facts/extract b/bin/facts/extract new file mode 100755 index 0000000..47d5f66 --- /dev/null +++ b/bin/facts/extract @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +"""facts/extract - acquire confirmed facts from the ops stack (2-source each). + + bin/facts/extract [--json] [--dry-run] [--ssh PATH] [--compose PATH] + +The deduction rule: a fact is only stored under root=facts if it is backed by +>=2 independent sources. Sources here: + + S1 runtime : docker ps (running containers) or ~/.ssh/config (hosts) + S2 declared : docker-compose files or PLAN.md/README.md mentions + +Extracted facts are written into var/kb.lbug (root=facts, confidence=confirmed). +--dry-run prints the proposed facts without touching the database. +""" +from __future__ import annotations + +import json +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "tools")) + +COMPOSE_FILES = [ROOT / "docker" / "compose.yaml", ROOT / "compose.yaml"] +DOC_MARKERS = ["README.md", "PLAN.md", "AGENTS.md"] +SSH_CONFIG = Path.home() / ".ssh" / "config" +REPO = "eSlider/2dph" + + +def read_docker_ps() -> list[str]: + try: + out = subprocess.run( + ["docker", "ps", "--format", "{{.Names}}"], + capture_output=True, text=True, timeout=10, + ) + if out.returncode != 0: + return [] + return [n.strip() for n in out.stdout.splitlines() if n.strip()] + except (FileNotFoundError, subprocess.TimeoutExpired): + return [] + + +def compose_files_in(dirname: Path) -> list[Path]: + out = [] + for name in ("compose.yaml", "compose.yml", "docker-compose.yml", "docker-compose.yaml"): + p = dirname / name + if p.exists(): + out.append(p) + return out + + +def container_compose_dir(name: str) -> Path | None: + """Find the compose project dir a running container belongs to.""" + try: + out = subprocess.run( + ["docker", "inspect", name, + "--format", "{{ index .Config.Labels \"com.docker.compose.project.working_dir\"}}"], + capture_output=True, text=True, timeout=10, + ) + if out.returncode != 0: + return None + path = out.stdout.strip() + return Path(path) if path and path != "" else None + except (FileNotFoundError, subprocess.TimeoutExpired): + return None + + +def read_compose_services(compose: Path) -> list[str]: + if not compose.exists(): + return [] + try: + import yaml + return list(yaml.safe_load(compose.read_text()).get("services", {}).keys()) + except Exception: + return [] + + +def read_ssh_hosts(path: Path) -> list[str]: + if not path.exists(): + return [] + hosts = [] + for line in path.read_text().splitlines(): + m = re.match(r"^\s*Host\s+(.+)$", line) + if m: + hosts.extend(h for h in m.group(1).split() if h not in ("*",)) + return hosts + + +def mentions(term: str, files: list[Path]) -> bool: + for path in files: + if not path.exists(): + continue + try: + if re.search(rf"\b{re.escape(term)}\b", path.read_text(), re.I): + return True + except OSError: + continue + return False + + +def build_facts() -> list[dict]: + facts: list[dict] = [] + doc_files = [ROOT / m for m in DOC_MARKERS] + + running = read_docker_ps() + # Pair each running container against its own compose file (2 independent + # sources: runtime state docker ps × declared state compose). + runtime_facts = 0 + for name in running: + cdir = container_compose_dir(name) + for cfile in compose_files_in(cdir) if cdir else []: + if name in read_compose_services(cfile): + facts.append({ + "text": f"container '{name}' is running and declared in {cfile.name}", + "source": f"docker ps x compose:{cfile.name}", + "loc": f"{cfile}:{name}", + "how": "facts/extract", + }) + runtime_facts += 1 + break + if runtime_facts: + print(f"facts/extract: paired {runtime_facts}/{len(running)} running containers to compose", file=sys.stderr) + + compose = [Path(p) for p in COMPOSE_FILES] + compose_services = set() + for c in compose: + compose_services.update(read_compose_services(c)) + if compose_services and running: + overlap = sorted(compose_services & set(running)) + for name in overlap: + facts.append({ + "text": f"container '{name}' is running and declared in compose", + "source": f"docker ps x {compose[0].name}", + "loc": "docker ps; docker compose config", + "how": "facts/extract", + }) + + hosts = read_ssh_hosts(SSH_CONFIG) + for host in hosts: + if mentions(host, doc_files): + facts.append({ + "text": f"host '{host}' is configured in ~/.ssh/config and referenced in this repo", + "source": f"ssh config x docs({', '.join(DOC_MARKERS)})", + "loc": f"~/.ssh/config:{host}", + "how": "facts/extract", + }) + + # Single-docker-container facts still need 2 sources: running + hostname hint + for name in running: + known_hosts = set(hosts) + if not known_hosts: + break + # a running container name that also matches a configured host + if name in known_hosts: + facts.append({ + "text": f"container '{name}' is running and matches configured host '{name}'", + "source": "docker ps x ssh config", + "loc": f"docker ps:{name}; ~/.ssh/config:{name}", + "how": "facts/extract", + }) + return facts + + +def dedupe(facts: list[dict]) -> list[dict]: + seen = set() + out = [] + for f in facts: + key = f["text"] + if key in seen: + continue + seen.add(key) + out.append(f) + return out + + +def write_facts(facts: list[dict]) -> None: + from kblib import connect, init_schema, upsert_leaf + from kblib import VAR + VAR.mkdir(exist_ok=True) + db, conn = connect(VAR / "kb.lbug", read_only=False) + init_schema(conn) + from model2vec import StaticModel + model = StaticModel.from_pretrained("minishlab/potion-multilingual-128M") + for f in facts: + emb = model.encode([f["text"]])[0].astype(float).tolist() + upsert_leaf(conn, text=f["text"], root="facts", confidence="confirmed", + source=f["source"], source_rev=REPO, how=f["how"], + loc=f["loc"], type_="fact", embedding=emb) + conn.close() + db.close() + + +def main(argv: list[str]) -> int: + global COMPOSE_FILES, SSH_CONFIG + import argparse + p = argparse.ArgumentParser(description="acquire confirmed facts from ops sources") + p.add_argument("--json", action="store_true") + p.add_argument("--dry-run", action="store_true") + p.add_argument("--ssh", default=str(SSH_CONFIG)) + p.add_argument("--compose", action="append") + a = p.parse_args(argv) + + if a.compose: + COMPOSE_FILES = [Path(c) for c in a.compose] + if a.ssh: + SSH_CONFIG = Path(a.ssh) + + facts = dedupe(build_facts()) + if not a.dry_run and facts: + write_facts(facts) + + out = {"count": len(facts), "facts": facts} + if a.json: + print(json.dumps(out, indent=2, ensure_ascii=False)) + else: + from yamlout import to_yaml + print(to_yaml(out)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) \ No newline at end of file diff --git a/docker/kb-watch b/bin/kb-watch similarity index 93% rename from docker/kb-watch rename to bin/kb-watch index 181d912..303b345 100644 --- a/docker/kb-watch +++ b/bin/kb-watch @@ -12,7 +12,7 @@ DIRS=("$@") [[ ${#DIRS[@]} -eq 0 ]] && DIRS=(${DEFAULT_DIRS}) INTERVAL="${KB_WATCH_INTERVAL:-30}" -index() { python3 /app/bin/kb/index; } +index() { "${KB_PY:-python3}" /app/bin/kb/index; } LAST_STAMP="" while true; do diff --git a/bin/kb/eval b/bin/kb/eval new file mode 100755 index 0000000..cae1e7a --- /dev/null +++ b/bin/kb/eval @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""kb/eval - recall@5 gate for the brain. + + bin/kb/eval [--json] + +Control questions are answered from the graph; recall@5 >= 0.95 gates CI. +Each question maps to leaf ids that MUST appear in the top 5 hits. +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "tools")) + +from kblib import open_readonly, query_fts # noqa: E402 +from yamlout import to_yaml # noqa: E402 + +RECALL_THRESHOLD = 0.95 + +# (query, expected leaf id) +CONTROL_QUESTIONS: list[tuple[str, str]] = [ + ("which database does the brain use", "facts:ladybug"), + ("hybrid search weights fts and vector equally", "info:hybrid"), +] + + +def hit_ids_of(query: str, conn, limit: int = 5) -> list[str]: + try: + hits = query_fts(conn, query, limit) + return [h["id"] for h in hits] + except Exception: + return [] + + +def main(argv: list[str]) -> int: + import argparse + p = argparse.ArgumentParser(description="recall@5 gate") + p.add_argument("--json", action="store_true") + a = p.parse_args(argv) + + db, conn = open_readonly() + recalled = 0 + detail = [] + for query, expected in CONTROL_QUESTIONS: + hits = hit_ids_of(query, conn) + ok = any(expected in h or h in expected for h in hits) + recalled += int(ok) + detail.append({"q": query, "expected": expected, "in_top5": ok, "hits": hits[:5]}) + recall = recalled / len(CONTROL_QUESTIONS) if CONTROL_QUESTIONS else 1.0 + passed = recall >= RECALL_THRESHOLD + out = {"recall@5": round(recall, 3), "passed": passed, "gate": len(CONTROL_QUESTIONS), "details": detail} + print(json.dumps(out, indent=2) if a.json else to_yaml(out)) + conn.close() + db.close() + return 0 if passed else 2 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) \ No newline at end of file diff --git a/bin/kb/get b/bin/kb/get new file mode 100755 index 0000000..3054a1b --- /dev/null +++ b/bin/kb/get @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""kb/get - read one leaf by id. + + bin/kb/get # metadata + snippet + bin/kb/get --body # full text +""" +from __future__ import annotations + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "tools")) + +from kblib import open_readonly # noqa: E402 +from yamlout import to_yaml # noqa: E402 + + +def main(argv: list[str]) -> int: + import argparse + p = argparse.ArgumentParser(description="read one leaf by id") + p.add_argument("id") + p.add_argument("--body", action="store_true") + a = p.parse_args(argv) + + db, conn = open_readonly() + r = conn.execute( + "MATCH (l:Leaf {id:$id}) RETURN l.id, l.text, l.root, l.confidence, l.source, l.type", + parameters={"id": a.id}, + ) + rows = r.get_all() + if not rows: + print(f"kb/get: no leaf {a.id}", file=sys.stderr) + conn.close() + db.close() + return 1 + row = rows[0] + out = {"id": row[0], "root": row[2], "confidence": row[3], "source": row[4], "type": row[5]} + if a.body: + out["text"] = row[1] + else: + out["snippet"] = row[1][:280] + print(to_yaml(out)) + conn.close() + db.close() + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) \ No newline at end of file diff --git a/bin/kb/index b/bin/kb/index new file mode 100755 index 0000000..168f044 --- /dev/null +++ b/bin/kb/index @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""kb/index - build the 2dph brain from markdown + factual leafs. + + bin/kb/index [--corpus DIR] [--rebuild] [--limit N] + bin/kb/index --json # emit stats as JSON + +Reads every .md under the corpus (default: repo root docs, skills, READMEs) +as `info` leafs, embeds them with model2vec (potion-multilingual-128M), and +writes them into var/kb.lbug with FTS + HNSW indexes. `facts` leafs come +from bin/facts/extract (docker x compose x ssh-config pairing). + +--rebuild drops the database file and indexes from scratch. Without it a run +is idempotent (MERGE by (source,text) id). +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "tools")) + +from kblib import ( # noqa: E402 + connect, create_fts_and_vector, init_schema, upsert_leaf, + open_readonly, stats, +) +from mdleaves import read_markdown, to_all, walk_markdown # noqa: E402 + +CORPUS_DEFAULTS = ["README.md", "PLAN.md", "AGENTS.md", "docs", "skills"] + + +def load_corpus(root: Path) -> list[dict]: + files: list[Path] = [] + for entry in CORPUS_DEFAULTS: + p = root / entry + if p.is_file(): + files.append(p) + elif p.is_dir(): + files.extend(walk_markdown(p)) + leafs: list[dict] = [] + for path in files: + try: + leafs.extend(to_all(read_markdown(path), path, repo="eSlider/2dph")) + except OSError as e: + print(f"kb/index: skip {path}: {e}", file=sys.stderr) + return leafs + + +def load_corpus_glob(source: str) -> list[dict]: + """Add arbitrary markdown dirs/files as corpus roots (repo=dirname).""" + root = Path(source) + if not root.exists(): + print(f"kb/index: skip missing corpus {source}", file=sys.stderr) + return [] + files = [root] if root.is_file() else walk_markdown(root) + repo = root.name if root.is_dir() else root.parent.name + leafs: list[dict] = [] + for path in files: + try: + leafs.extend(to_all(read_markdown(path), path, repo=repo)) + except OSError as e: + print(f"kb/index: skip {path}: {e}", file=sys.stderr) + return leafs + + +def index_leafs(conn, leafs: list[dict], embed_fn, limit: int) -> tuple[int, int]: + count = 0 + for lf in leafs[:limit] if limit else leafs: + query = f"{lf['heading']}\n\n{lf['text']}" + emb = embed_fn(lf["text"]) if lf["text"] else None + upsert_leaf(conn, text=query, root="info", confidence="confirmed", + source=lf["source"], source_rev="working-tree", + how="kb/index", loc=lf["source"], type_=lf.get("type", "reference"), + embedding=emb) + count += 1 + return count, len(leafs) + + +def embedder(): + from model2vec import StaticModel + model = StaticModel.from_pretrained("minishlab/potion-multilingual-128M") + return lambda text: model.encode([text])[0].astype(float).tolist() + + +def main(argv: list[str]) -> int: + import argparse + p = argparse.ArgumentParser(description="build the 2dph brain index") + p.add_argument("--corpus", action="append", help="extra markdown dir/file to index (may repeat)") + p.add_argument("--rebuild", action="store_true", help="fresh db + indexes") + p.add_argument("--limit", type=int, default=0, help="max leafs to embed") + p.add_argument("--json", action="store_true") + a = p.parse_args(argv) + + from kblib import DB_PATH, VAR + VAR.mkdir(exist_ok=True) + if a.rebuild and DB_PATH.exists(): + DB_PATH.unlink() + + leafs = load_corpus(ROOT) + if a.corpus: + for source in a.corpus: + leafs.extend(load_corpus_glob(source)) + + db, conn = connect(DB_PATH, read_only=False) + init_schema(conn) + + if not (a.rebuild or _already_indexed(conn)): + create_fts_and_vector(conn, force=True) + embed = embedder() + done, total = index_leafs(conn, leafs, embed, a.limit) + create_fts_and_vector(conn, force=(done > 0 or a.rebuild)) + s = stats(conn) + conn.close() + db.close() + + result = {"indexed": done, "corpus_total": total, **{k: v for k, v in s.items() if k in ("total", "by_root")}} + print(json.dumps(result, indent=2) if a.json else f"indexed {done}/{total} leafs; db total {s['total']}") + return 0 + + +def _already_indexed(conn) -> bool: + try: + return conn.execute("MATCH (l:Leaf) RETURN count(*)").get_all()[0][0] > 0 + except Exception: + return False + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) \ No newline at end of file diff --git a/bin/kb/search b/bin/kb/search new file mode 100755 index 0000000..55d95bd --- /dev/null +++ b/bin/kb/search @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""kb/search - deduction search over the 2dph brain. + + bin/kb/search "query" # hybrid facts+info, YAML out + bin/kb/search "query" --root facts # confirmed facts only + bin/kb/search "query" --hop 1 # follow graph edges after hitting + bin/kb/search "query" --json | yq '.' + bin/kb/search "query" -n 5 # more results + +Deduction order: facts root first (confirmed answers with evidence links), +then info root (marked `(not confirmed)`). --root restricts to one root. +--hop N walks FROM_FILE edges (sibling leafs in the same source file). +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "tools")) + +from kblib import connect, hybrid_search, init_schema, open_readonly, query_fts # noqa: E402 +from yamlout import to_yaml # noqa: E402 +import ladybug # noqa: E402 + + +def main(argv: list[str]) -> int: + import argparse + p = argparse.ArgumentParser(description="deduction search over the brain") + p.add_argument("query") + p.add_argument("--root", choices=("facts", "info", None), default=None) + p.add_argument("--hop", type=int, default=0) + p.add_argument("-n", "--limit", type=int, default=10) + p.add_argument("--json", action="store_true") + a = p.parse_args(argv) + + try: + db, conn = open_readonly() + except FileNotFoundError as e: + print(e, file=sys.stderr) + return 1 + + from model2vec import StaticModel + model = StaticModel.from_pretrained("minishlab/potion-multilingual-128M") + emb = model.encode([a.query])[0].astype(float).tolist() + + rhs: list[dict] = [] + try: + rhs = query_fts(conn, a.query, a.limit * 2) + except Exception: + rhs = [] + + results = hybrid_search(conn, emb, rhs, a.limit) + if a.root: + results = [h for h in results if h["root"] == a.root] + + for hit in results: + hit.pop("rrf", None) + if hit.get("text"): + hit["snippet"] = hit["text"][:280] + + out = {"query": a.query, "root_filter": a.root or "facts+info", + "count": len(results), "results": results} + print(json.dumps(out, indent=2, ensure_ascii=False) if a.json else to_yaml(out)) + conn.close() + db.close() + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) \ No newline at end of file diff --git a/bin/kb/stats b/bin/kb/stats new file mode 100755 index 0000000..e169ab0 --- /dev/null +++ b/bin/kb/stats @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""kb/stats - index health for the 2dph brain. + + bin/kb/stats # leaf counts by root, db size, model + bin/kb/stats --json # machine-readable +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "tools")) + +from kblib import open_readonly, stats # noqa: E402 +from yamlout import to_yaml # noqa: E402 + + +def main(argv: list[str]) -> int: + import argparse + p = argparse.ArgumentParser(description="brain index health") + p.add_argument("--json", action="store_true") + a = p.parse_args(argv) + + try: + db, conn = open_readonly() + except FileNotFoundError as e: + print(e, file=sys.stderr) + return 1 + + s = stats(conn) + conn.close() + db.close() + print(json.dumps(s, indent=2) if a.json else to_yaml(s)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) \ No newline at end of file diff --git a/bin/md/import b/bin/md/import new file mode 100755 index 0000000..bf6ebae --- /dev/null +++ b/bin/md/import @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +import lib +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from mdleaves import leaves_to_json, read_markdown, to_all, walk_markdown # noqa: E402 +from yamlout import to_yaml # noqa: E402 + + +def main(argv: list[str]) -> int: + import argparse + p = argparse.ArgumentParser(description="md/import - split markdown corpus into leafs") + p.add_argument("root", nargs="?", default=".", help="directory to walk for .md files") + p.add_argument("--files", action="store", help="comma-separated file list") + p.add_argument("--json", action="store_true") + a = p.parse_args(argv) + + paths: list[Path] = [] + if a.files: + paths = [Path(f) for f in a.files.split(",")] + else: + root = Path(a.root) + if not root.exists(): + print(f"md/import: no such path {root}", file=sys.stderr) + return 2 + paths = [root] if root.is_file() else walk_markdown(root) + + if not paths: + print("md/import: no markdown files", file=sys.stderr) + return 1 + + all_leafs: list[dict] = [] + for path in paths: + try: + text = read_markdown(path) + except OSError as e: + print(f"md/import: {path}: {e}", file=sys.stderr) + continue + all_leafs.extend(to_all(text, path)) + + out = leaves_to_json(all_leafs) + print(out if a.json else to_yaml(__import__("json").loads(out))) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) \ No newline at end of file diff --git a/docker/compose.yaml b/compose.yaml similarity index 83% rename from docker/compose.yaml rename to compose.yaml index 6c112a9..106ee0a 100644 --- a/docker/compose.yaml +++ b/compose.yaml @@ -1,7 +1,8 @@ -# 2dph — best-practice docker composition +# 2dph — docker composition # # docker compose run --rm brain index # rebuild graph # docker compose run --rm brain search "Matrix fed" # one-shot query +# docker compose run --rm brain serve # async Go server # docker compose up brain-watch # auto re-index # # Caching: the 128M model (HF_HOME) and kb.lbug (VAR_DIR) live in named @@ -15,8 +16,8 @@ services: brain: image: ghcr.io/eslider/2dph:latest build: - context: .. - dockerfile: docker/Dockerfile + context: . + dockerfile: Dockerfile cache_from: - ghcr.io/eslider/2dph:cache command: ["brain", "search", "help"] @@ -25,11 +26,13 @@ services: BRAIN_SEARCH_CACHE: /data/cache/web-search.sqlite BRAIN_DB_PROFILES: /secret/db-profiles.yml BRAIN_SEARCH_ENV: /secret/search.env + KB_SEARCH_CMD: /app/bin/kb/search + KB_WORKERS: "4" volumes: - kb-model:/data/hf - kb-var:/data # corpus is read-only on the host, never written from the container - - ../..:/corpus:ro + - ..:/corpus:ro - ~/.config/brain:/secret:ro read_only: true tmpfs: @@ -42,14 +45,14 @@ services: restart: unless-stopped stop_grace_period: 20s - # watcher: re-index on corpus file change (inotify via watchdog script) + # watcher: re-index on corpus file change (watchdog script) brain-watch: image: ghcr.io/eslider/2dph:latest environment: *env volumes: - kb-model:/data/hf - kb-var:/data - - ../..:/corpus:ro + - ..:/corpus:ro - ~/.config/brain:/secret:ro command: ["brain", "watch", "/corpus"] read_only: true diff --git a/docker/serve b/docker/serve deleted file mode 100644 index 5223874..0000000 --- a/docker/serve +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python3 -"""serve - tiny read-only HTTP wrapper over bin/kb/search. - - serve [port] default 8630, binds 127.0.0.1 - -GET /health -> {"status":"ok"} -GET /search?q=... -> YAML from kb/search (transparent, cached by client) -""" -from __future__ import annotations - -import json -import sys -import urllib.parse -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer - -PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 8630 - - -class Handler(BaseHTTPRequestHandler): - def log_message(self, *args): # keep stdout clean - pass - - def do_GET(self): # noqa: N802 - url = urllib.parse.urlparse(self.path) - if url.path.rstrip("/") == "/health": - self._json({"status": "ok"}) - return - if url.path.rstrip("/") == "/search": - q = urllib.parse.parse_qs(url.query).get("q", [""])[0] - if not q: - self._json({"error": "q required"}, code=400) - return - self._json({"query": q, "note": "search via bin/kb/search is offline; rewire to ladybug reads"}) - return - self._json({"error": "not found"}, code=404) - - def _json(self, obj, code=200): - body = json.dumps(obj, ensure_ascii=False).encode() - self.send_response(code) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - -if __name__ == "__main__": - server = ThreadingHTTPServer(("127.0.0.1", PORT), Handler) - print(f"serve: 127.0.0.1:{PORT}", file=sys.stderr) - server.serve_forever() \ No newline at end of file diff --git a/release-please-config.json b/release-please-config.json deleted file mode 100644 index 137d666..0000000 --- a/release-please-config.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json", - "release-type": "simple", - "bump-minor-pre-major": true, - "bump-patch-for-minor-pre-major": false, - "include-v-in-tag": true, - "include-component-in-tag": false, - "pull-request-title-pattern": "chore${scope}: release ${version}", - "changelog-path": "CHANGELOG.md", - "changelog-sections": [ - { "type": "feat", "section": "Features" }, - { "type": "fix", "section": "Bug Fixes" }, - { "type": "perf", "section": "Performance Improvements" }, - { "type": "revert", "section": "Reverts" }, - { "type": "refactor", "section": "Code Refactoring" }, - { "type": "docs", "section": "Documentation" }, - { "type": "test", "section": "Tests", "hidden": true }, - { "type": "build", "section": "Build System", "hidden": true }, - { "type": "ci", "section": "Continuous Integration", "hidden": true }, - { "type": "chore", "section": "Miscellaneous", "hidden": true }, - { "type": "style", "section": "Styles", "hidden": true } - ], - "packages": { - ".": { - "package-name": "2dph" - } - } -} \ No newline at end of file diff --git a/serve/go.mod b/serve/go.mod new file mode 100644 index 0000000..808dbab --- /dev/null +++ b/serve/go.mod @@ -0,0 +1,3 @@ +module github.com/eSlider/2dph/serve + +go 1.25 \ No newline at end of file diff --git a/serve/main.go b/serve/main.go new file mode 100644 index 0000000..5656de8 --- /dev/null +++ b/serve/main.go @@ -0,0 +1,143 @@ +// Package main serves the 2dph brain over HTTP. +// +// Async by design: every request runs on its own goroutine, and CPU-heavy +// searches are serialized through a bounded worker pool (a counting +// semaphore) so N requests can't spawn N Python interpreters at once. +package main + +import ( + "context" + "encoding/json" + "errors" + "log" + "net/http" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" +) + +type Searcher interface { + Search(ctx context.Context, query string, limit int) ([]byte, error) +} + +type Server struct { + searcher Searcher + semaphore chan struct{} +} + +const defaultPort = 8630 + +func NewServer(searcher Searcher, workers int) http.Handler { + return &Server{ + searcher: searcher, + semaphore: make(chan struct{}, workers), + } +} + +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/health": + writeJSON(w, http.StatusOK, map[string]any{"status": "ok"}) + case r.URL.Path == "/search": + s.handleSearch(w, r) + default: + writeJSON(w, http.StatusNotFound, map[string]any{"error": "not found"}) + } +} + +func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) { + q := strings.TrimSpace(r.URL.Query().Get("q")) + if q == "" { + writeJSON(w, http.StatusBadRequest, map[string]any{"error": "q required"}) + return + } + limit := 10 + if raw := r.URL.Query().Get("n"); raw != "" { + n, err := strconv.Atoi(raw) + if err != nil || n < 1 || n > 100 { + writeJSON(w, http.StatusBadRequest, map[string]any{"error": "n must be int 1..100"}) + return + } + limit = n + } + + // Worker pool: block until a slot frees, so burst concurrency still + // bounds memory (no unbounded python processes). + select { + case s.semaphore <- struct{}{}: + defer func() { <-s.semaphore }() + case <-r.Context().Done(): + return + } + + body, err := s.searcher.Search(r.Context(), q, limit) + if err != nil { + writeJSON(w, http.StatusGatewayTimeout, map[string]any{"error": err.Error()}) + return + } + writeRaw(w, http.StatusOK, body) +} + +func writeJSON(w http.ResponseWriter, code int, obj any) { + body, _ := json.Marshal(obj) + writeRaw(w, code, body) +} + +func writeRaw(w http.ResponseWriter, code int, body []byte) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Length", strconv.Itoa(len(body))) + w.WriteHeader(code) + w.Write(body) +} + +// brainSearcher shells out to bin/kb/search --json. A single python search +// is bounded and short-lived; the worker pool keeps at most N live. +type brainSearcher struct { + cmdPath string + timeout time.Duration +} + +func (b *brainSearcher) Search(ctx context.Context, query string, limit int) ([]byte, error) { + ctx, cancel := context.WithTimeout(ctx, b.timeout) + defer cancel() + cmd := exec.CommandContext(ctx, b.cmdPath, "--json", "-n", strconv.Itoa(limit), query) + out, err := cmd.Output() + if err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return nil, errors.New("search backend failed: " + strings.TrimSpace(string(exitErr.Stderr))) + } + return nil, err + } + return out, nil +} + +func main() { + searchPath := os.Getenv("KB_SEARCH_CMD") + if searchPath == "" { + searchPath = filepath.Join("bin", "kb", "search") + } + workers := 4 + if raw := os.Getenv("KB_WORKERS"); raw != "" { + if n, err := strconv.Atoi(raw); err == nil && n > 0 { + workers = n + } + } + port := defaultPort + if raw := os.Getenv("KB_PORT"); raw != "" { + if n, err := strconv.Atoi(raw); err == nil && n > 0 { + port = n + } + } + + searcher := &brainSearcher{cmdPath: searchPath, timeout: 60 * time.Second} + handler := NewServer(searcher, workers) + addr := "127.0.0.1:" + strconv.Itoa(port) + log.Printf("serve: %s (workers=%d)", addr, workers) + if err := http.ListenAndServe(addr, handler); err != nil { + log.Fatal(err) + } +} \ No newline at end of file diff --git a/serve/main_test.go b/serve/main_test.go new file mode 100644 index 0000000..8c31ab7 --- /dev/null +++ b/serve/main_test.go @@ -0,0 +1,177 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" +) + +// fakeSearcher is an injectable Searcher for tests (no python involved). +type fakeSearcher struct { + mu sync.Mutex + delay time.Duration + calls int + active atomic.Int32 + maxSeen atomic.Int32 + callback func(q string, limit int) ([]byte, error) +} + +func (f *fakeSearcher) Search(ctx context.Context, query string, limit int) ([]byte, error) { + f.mu.Lock() + f.calls++ + f.mu.Unlock() + n := f.active.Add(1) + for { + old := f.maxSeen.Load() + if n <= old || f.maxSeen.CompareAndSwap(old, n) { + break + } + } + defer f.active.Add(-1) + if f.delay > 0 { + select { + case <-time.After(f.delay): + case <-ctx.Done(): + return nil, ctx.Err() + } + } + if f.callback != nil { + return f.callback(query, limit) + } + return []byte(`{"query":"` + query + `","count":0,"results":[]}`), nil +} + +func (f *fakeSearcher) count() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.calls +} + +func newTestServer(s Searcher, workers int) http.Handler { + return NewServer(s, workers) +} + +func get(t *testing.T, h http.Handler, path string) (int, []byte) { + t.Helper() + req := httptest.NewRequest(http.MethodGet, path, nil) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + return rec.Code, rec.Body.Bytes() +} + +func TestHealth(t *testing.T) { + h := newTestServer(&fakeSearcher{}, 1) + code, body := get(t, h, "/health") + if code != http.StatusOK { + t.Fatalf("health code = %d, want 200", code) + } + var out map[string]any + if err := json.Unmarshal(body, &out); err != nil { + t.Fatalf("health body not json: %v (%s)", err, body) + } + if out["status"] != "ok" { + t.Fatalf("health status = %v, want ok", out["status"]) + } +} + +func TestSearchMissingQuery(t *testing.T) { + h := newTestServer(&fakeSearcher{}, 1) + if code, _ := get(t, h, "/search"); code != http.StatusBadRequest { + t.Fatalf("code = %d, want 400", code) + } +} + +func TestSearchReturnsSearcherResult(t *testing.T) { + fs := &fakeSearcher{callback: func(q string, limit int) ([]byte, error) { + return []byte(`{"query":"` + q + `","count":1,"results":[{"id":"x"}]}`), nil + }} + h := newTestServer(fs, 1) + code, body := get(t, h, "/search?q=matrix") + if code != http.StatusOK { + t.Fatalf("code = %d, want 200", code) + } + var out map[string]any + if err := json.Unmarshal(body, &out); err != nil { + t.Fatalf("body not json: %v (%s)", err, body) + } + if out["query"] != "matrix" { + t.Fatalf("query = %v, want matrix", out["query"]) + } + if fs.count() != 1 { + t.Fatalf("searcher called %d times, want 1", fs.count()) + } +} + +func TestSearchConcurrencyBounded(t *testing.T) { + // 8 parallel requests on a 3-worker pool: at most 3 concurrent searches. + fs := &fakeSearcher{delay: 20 * time.Millisecond} + h := newTestServer(fs, 3) + + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + req := httptest.NewRequest(http.MethodGet, "/search?q=abc", nil) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Errorf("code = %d, want 200", rec.Code) + } + }() + } + wg.Wait() + + if calls := fs.count(); calls != 8 { + t.Fatalf("searcher called %d times, want 8", calls) + } + if max := fs.maxSeen.Load(); max > 3 { + t.Fatalf("max concurrent = %d, want <= 3", max) + } + if max := fs.maxSeen.Load(); max < 1 { + t.Fatalf("max concurrent = %d, want >= 1", max) + } +} + +func TestSearchRejectsBadLimit(t *testing.T) { + h := newTestServer(&fakeSearcher{}, 1) + if code, _ := get(t, h, "/search?q=x&n=hundred"); code != http.StatusBadRequest { + t.Fatalf("code = %d, want 400", code) + } +} + +func TestSearchTimeout(t *testing.T) { + fs := &fakeSearcher{delay: time.Second} + h := NewServer(fs, 1) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + req := httptest.NewRequest(http.MethodGet, "/search?q=slow", nil).WithContext(ctx) + rec := httptest.NewRecorder() + + done := make(chan struct{}) + go func() { + h.ServeHTTP(rec, req) + close(done) + }() + + select { + case <-done: + // ServeHTTP returned; body should be an error json (we don't require a + // specific code for the pathological ctx-cancel timing, only that it + // does not hang forever). + return + case <-time.After(500 * time.Millisecond): + t.Fatal("request hung after context cancellation") + } + _ = io.Discard + _ = bytes.MinRead + _ = fmt.Sprintf +} \ No newline at end of file diff --git a/tools/ci/test_semver_bump.py b/tools/ci/test_semver_bump.py new file mode 100644 index 0000000..cbb5a9a --- /dev/null +++ b/tools/ci/test_semver_bump.py @@ -0,0 +1,50 @@ +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from semver import bump_type, bump_version # noqa: E402 + + +class BumpTypeTest(unittest.TestCase): + def test_empty_is_none(self): + self.assertEqual(bump_type([]), "none") + + def test_feat_is_minor(self): + self.assertEqual(bump_type(["feat: add search"]), "minor") + + def test_fix_is_patch(self): + self.assertEqual(bump_type(["fix: typo"]), "patch") + + def test_chore_and_docs_still_release(self): + self.assertEqual(bump_type(["docs: readme"]), "patch") + self.assertEqual(bump_type(["ci: green"]), "patch") + + def test_breaking_marker_is_major(self): + self.assertEqual(bump_type(["feat!: break api"]), "major") + self.assertEqual(bump_type(["fix: x\n\nBREAKING CHANGE: y"]), "major") + + def test_mixed_commits_choose_highest(self): + self.assertEqual(bump_type(["fix: a", "feat: b"]), "minor") + + +class BumpVersionTest(unittest.TestCase): + def test_patch(self): + self.assertEqual(bump_version("v0.1.0", "patch"), "v0.1.1") + + def test_minor(self): + self.assertEqual(bump_version("v0.1.0", "minor"), "v0.2.0") + + def test_major(self): + self.assertEqual(bump_version("v0.1.0", "major"), "v1.0.0") + + def test_initial_when_no_tag(self): + self.assertEqual(bump_version(None, "patch"), "v0.0.1") + + def test_none_returns_none(self): + self.assertIsNone(bump_version("v0.1.0", "none")) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tools/kblib.py b/tools/kblib.py new file mode 100644 index 0000000..45f3612 --- /dev/null +++ b/tools/kblib.py @@ -0,0 +1,152 @@ +"""kblib - the 2dph brain core over LadybugDB. + +Single embedded graph `var/kb.lbug`. Two roots: facts (assertions backed by +>=2 independent sources) and info (narrative leafs). Hybrid retrieval: BM25 +(FTS extension) + HNSW cosine (VECTOR extension) + Cypher graph hops. + +All access is read-only unless `--rebuild` is passed to kb/index. +""" +from __future__ import annotations + +import hashlib +import json +import time +import zlib +from pathlib import Path + +import ladybug + +MODEL = "minishlab/potion-multilingual-128M" +EMBED_DIM = 256 +ROOT_FACTS = "facts" +ROOT_INFO = "info" +CONF_CONFIRMED = "confirmed" + +VAR = Path(__file__).resolve().parents[1] / "var" +DB_PATH = VAR / "kb.lbug" + + +def sha256_b64(text: str) -> str: + return hashlib.sha256(text.encode()).hexdigest() + + +def connect(path: Path | str | None = None, read_only: bool = True) -> tuple[ladybug.Database, ladybug.Connection]: + db = ladybug.Database(str(path or DB_PATH), read_only=read_only) + conn = ladybug.Connection(db) + conn.execute("LOAD EXTENSION FTS") + conn.execute("LOAD EXTENSION VECTOR") + return db, conn + + +def init_schema(conn: ladybug.Connection) -> None: + conn.execute( + "CREATE NODE TABLE IF NOT EXISTS Leaf (" + " id STRING, text STRING, root STRING, confidence STRING, " + " sha256 STRING, source STRING, source_rev STRING, observed_at STRING, " + " how STRING, loc STRING, type STRING, embedding FLOAT[256], " + " PRIMARY KEY(id))" + ) + conn.execute( + "CREATE NODE TABLE IF NOT EXISTS File (" + " id STRING, path STRING, repo STRING, mtime STRING, PRIMARY KEY(id))" + ) + conn.execute( + "CREATE REL TABLE IF NOT EXISTS FROM_FILE (FROM Leaf TO File)" + ) + conn.execute( + "CREATE NODE TABLE IF NOT EXISTS Host (id STRING, hostname STRING, user STRING, PRIMARY KEY(id))" + ) + conn.execute( + "CREATE REL TABLE IF NOT EXISTS RUNS_ON (FROM Leaf TO Host)" + ) + + +def leaf_id(text: str, source: str) -> str: + return sha256_b64(f"{source}\0{text}")[:24] + + +def upsert_leaf(conn: ladybug.Connection, *, text: str, root: str, confidence: str, + source: str, source_rev: str, how: str, loc: str, type_: str, + embedding: list[float] | None) -> str: + lid = leaf_id(text, source) + obs = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + conn.execute( + "MERGE (l:Leaf {id:$id}) " + "SET l.text=$text, l.root=$root, l.confidence=$confidence, " + " l.sha256=$sha, l.source=$source, l.source_rev=$rev, l.observed_at=$obs, " + " l.how=$how, l.loc=$location, l.type=$type" + + (", l.embedding=$emb" if embedding else ""), + parameters={ + "id": lid, "text": text, "root": root, "confidence": confidence, + "sha": sha256_b64(text), "source": source, "rev": source_rev, + "obs": obs, "how": how, "location": loc, "type": type_, + "emb": (embedding if embedding else None), + }, + ) + return lid + + +def create_fts_and_vector(conn: ladybug.Connection, force: bool = False) -> None: + if force: + conn.execute("DROP INDEX IF EXISTS Leaf.Leaf_fts") + conn.execute("DROP INDEX IF EXISTS Leaf.Leaf_vec") + try: + conn.execute("CALL CREATE_FTS_INDEX('Leaf', 'id', ['text'])") + except Exception: + pass + try: + conn.execute("CALL CREATE_VECTOR_INDEX('Leaf', 'Leaf_vec', 'embedding', metric := 'cosine')") + except Exception: + pass + + +def query_fts(conn: ladybug.Connection, text: str, limit: int = 10) -> list[dict]: + r = conn.execute( + "CALL QUERY_FTS_INDEX('Leaf', 'id', $q) " + "RETURN node.id, node.text, node.root, score ORDER BY score DESC LIMIT $n", + parameters={"q": text, "n": limit}, + ) + return [{"id": row[0], "text": row[1], "root": row[2], "score": row[3]} for row in r.get_all()] + + +def query_vector(conn: ladybug.Connection, embedding: list[float], limit: int = 10) -> list[dict]: + r = conn.execute( + "CALL QUERY_VECTOR_INDEX('Leaf', 'Leaf_vec', $q, $n) " + "RETURN node.id, node.text, node.root, distance ORDER BY distance LIMIT $n", + parameters={"q": embedding, "n": limit}, + ) + out = [] + for row in r.get_all(): + # distance -> similarity reasonable for cosine + score = 1.0 - row[3] if row[3] is not None else 0.0 + out.append({"id": row[0], "text": row[1], "root": row[2], "score": score}) + return out + + +def hybrid_search(conn: ladybug.Connection, embedding: list[float], fts_hits: list[dict], + limit: int = 10) -> list[dict]: + """Merge FTS + vector by reciprocal rank fusion.""" + fused: dict[str, dict] = {} + for rank, hit in enumerate(fts_hits): + fused.setdefault(hit["id"], {**hit, "rrf": 0.0})["rrf"] = 1.0 / (60 + rank + 1) + for rank, hit in enumerate(query_vector(conn, embedding, limit * 3)): + entry = fused.setdefault(hit["id"], {**hit, "rrf": 0.0}) + entry["rrf"] += 1.0 / (60 + rank + 1) + entry.setdefault("score", hit.get("score", 0.0)) + ranked = sorted(fused.values(), key=lambda h: h.get("rrf", 0.0), reverse=True) + return ranked[:limit] + + +def stats(conn: ladybug.Connection) -> dict: + r = conn.execute("MATCH (l:Leaf) RETURN l.root, count(*)") + rows = {row[0]: row[1] for row in r.get_all()} + total = conn.execute("MATCH (l:Leaf) RETURN count(*)").get_all()[0][0] + return {"total": total, "by_root": rows, "db": str(DB_PATH), "model": MODEL} + + +def open_readonly() -> tuple[ladybug.Database, ladybug.Connection]: + if not DB_PATH.exists(): + raise FileNotFoundError(f"{DB_PATH} missing - run bin/kb/index first") + db, conn = connect(read_only=True) + init_schema(conn) + return db, conn \ No newline at end of file diff --git a/tools/mdleaves.py b/tools/mdleaves.py new file mode 100644 index 0000000..44762fc --- /dev/null +++ b/tools/mdleaves.py @@ -0,0 +1,84 @@ +import json +import re +from pathlib import Path + +import mistune + + +def extract_frontmatter(text: str) -> tuple[dict, str]: + """Return (frontmatter dict, body). Accepts leading --- yaml ---.""" + if not text.startswith("---"): + return {}, text + end = text.find("\n---", 3) + if end == -1: + return {}, text + fm = text[3:end].strip() + body = text[end + 4 :] + meta: dict = {} + for line in fm.splitlines(): + if ":" in line: + key, _, value = line.partition(":") + meta[key.strip()] = value.strip().strip("\"'") + return meta, body + + +def split_leafs(meta: dict, body: str) -> list[dict]: + """Split a markdown body into leaf chunks on H2 (##) boundaries. + + Each leaf keeps the document-level frontmatter (type, related) and gets + its own heading + text. H1 is treated as document title, prepended to the + first chunk. + """ + title = "" + lines = body.splitlines() + headers: list[tuple[str, int]] = [] + for i, line in enumerate(lines): + if re.match(r"^# \S", line): + title = line.lstrip("#").strip() + elif re.match(r"^## \S", line): + headers.append((line.lstrip("##").strip(), i)) + if not headers: + text = "\n".join(l for l in lines if l.strip()) + return [{"heading": title, "text": text.strip()}] + + leafs: list[dict] = [] + for idx, (heading, start) in enumerate(headers): + end = headers[idx + 1][1] if idx + 1 < len(headers) else len(lines) + chunk = "\n".join(l for l in lines[start:end] if l.strip()) + text = chunk + if idx == 0 and title: + text = f"{title}\n\n{chunk}" + leafs.append({"heading": heading, "text": text.strip()}) + return leafs + + +def to_all(text: str, path: str | Path, repo: str = "") -> list[dict]: + meta, body = extract_frontmatter(text) + meta.setdefault("type", "reference") + meta.setdefault("status", "current") + path = str(path) + leafs = split_leafs(meta, body) + out = [] + for lf in leafs: + out.append({ + "source": path, + "repo": repo, + "heading": lf["heading"], + "text": lf["text"], + "type": meta.get("type", "reference"), + "status": meta.get("status", "current"), + "related": meta.get("related", ""), + }) + return out + + +def read_markdown(path: Path) -> str: + return path.read_text(encoding="utf-8", errors="replace") + + +def walk_markdown(root: Path) -> list[Path]: + return sorted(p for p in root.rglob("*") if p.suffix.lower() in (".md", ".markdown")) + + +def leaves_to_json(leaves: list[dict]) -> str: + return json.dumps(leaves, ensure_ascii=False, indent=2) \ No newline at end of file diff --git a/tools/semver.py b/tools/semver.py new file mode 100644 index 0000000..672ed71 --- /dev/null +++ b/tools/semver.py @@ -0,0 +1,33 @@ +"""semver logic shared by bin/ci/semver and its tests. No git IO here.""" + +from __future__ import annotations + +BREAKING_MARKERS = ("BREAKING CHANGE", "breaking-change") +BUMP_PATCH_TYPES = ("fix", "perf", "refactor", "build", "ci", "docs", "chore", "test", "style", "revert") +BUMP_MINOR_TYPE = "feat" + + +def bump_type(subjects: list[str]) -> str: + if not subjects: + return "none" + for subject in subjects: + text = subject.lower() + if any(m.lower() in text for m in BREAKING_MARKERS): + return "major" + if "!" in subject.split(":")[0]: + return "major" + for subject in subjects: + if subject.startswith(f"{BUMP_MINOR_TYPE}:"): + return "minor" + return "patch" + + +def bump_version(current: str | None, bump: str) -> str | None: + if bump == "none": + return None + major, minor, patch = [int(n) for n in (current or "0.0.0").lstrip("v").split(".")] + if bump == "major": + return f"v{major + 1}.0.0" + if bump == "minor": + return f"v{major}.{minor + 1}.0" + return f"v{major}.{minor}.{patch + 1}" \ No newline at end of file diff --git a/tools/test_kblib.py b/tools/test_kblib.py new file mode 100644 index 0000000..6dc65f9 --- /dev/null +++ b/tools/test_kblib.py @@ -0,0 +1,73 @@ +import os +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import kblib # noqa: E402 + + +def make_emb(value: float) -> list[float]: + vec = [0.0] * kblib.EMBED_DIM + vec[0] = value + return vec + + +class KblibTest(unittest.TestCase): + def setUp(self): + self.dir = tempfile.mkdtemp() + self.dbpath = os.path.join(self.dir, "kb.lbug") + self.db, self.conn = kblib.connect(self.dbpath, read_only=False) + kblib.init_schema(self.conn) + + def tearDown(self): + self.conn.close() + self.db.close() + + def test_leaf_id_is_stable(self): + self.assertEqual(kblib.leaf_id("abc", "src"), kblib.leaf_id("abc", "src")) + self.assertNotEqual(kblib.leaf_id("abc", "src"), kblib.leaf_id("abd", "src")) + + def test_upsert_roundtrip(self): + kblib.upsert_leaf(self.conn, text="the quick brown fox", root="info", + confidence="confirmed", source="s", source_rev="r1", + how="test", loc="/tmp", type_="reference", + embedding=make_emb(1.0)) + kblib.create_fts_and_vector(self.conn, force=True) + hits = kblib.query_fts(self.conn, "fox", 5) + self.assertEqual(len(hits), 1) + self.assertEqual(hits[0]["root"], "info") + + def test_hybrid_ranks_vector_match(self): + kblib.upsert_leaf(self.conn, text="the quick brown fox", root="info", + confidence="confirmed", source="s", source_rev="r1", + how="test", loc="/tmp", type_="reference", + embedding=make_emb(1.0)) + kblib.upsert_leaf(self.conn, text="a lazy dog sleeps", root="info", + confidence="confirmed", source="s", source_rev="r1", + how="test", loc="/tmp", type_="reference", + embedding=make_emb(0.0)) + kblib.create_fts_and_vector(self.conn, force=True) + result = kblib.hybrid_search(self.conn, make_emb(1.0), [], 5) + self.assertTrue(result) + self.assertIn("rrf", result[0]) + self.assertEqual(result[0]["text"], "the quick brown fox") + + def test_stats_counts_roots(self): + kblib.upsert_leaf(self.conn, text="a fact leaf", root="facts", + confidence="confirmed", source="s", source_rev="r1", + how="test", loc="/tmp", type_="reference", + embedding=make_emb(0.5)) + kblib.upsert_leaf(self.conn, text="an info leaf", root="info", + confidence="confirmed", source="s", source_rev="r1", + how="test", loc="/tmp", type_="reference", + embedding=make_emb(0.5)) + stats = kblib.stats(self.conn) + self.assertEqual(stats["total"], 2) + self.assertEqual(stats["by_root"], {"facts": 1, "info": 1}) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file