diff --git a/.dockerignore b/.dockerignore index aec964f..8a687c1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,5 +10,4 @@ __pycache__ .cache .secrets .skills-tmp -serve/serve docs/.build \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02c33d9..1cca26e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,18 +31,16 @@ jobs: run: | bash -n bin/db/psql-yq bash -n bin/db/ssh-tunnel - bash -n bin/kb-watch bash -n bin/docker-entrypoint - name: Python unit tests (offline, vendored tools) run: | - uv run python -m unittest discover -s tools -t . + uv run python -m unittest discover -s bin/tools -t . - - name: Go serve tests (async, goroutine-bounded) + - name: Go tests (server + watch packages) run: | go vet ./... go test ./... -count=1 - working-directory: serve - name: facts/audit self (lexicon consistency, no network) run: | diff --git a/AGENTS.md b/AGENTS.md index 714cac1..0a6e2b4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,12 +36,13 @@ PLAN.md decisions + execution + open questions docs/ published docs skills/ in-project agent skills (vendored, no external links) bin/ self-describing tools bin/{subject}/{method} (shebang) -bin/kb-watch corpus watcher (mtimes, no inotify deps) +bin/serve.go async Go HTTP server entry (self-executing go run shebang) +bin/watch/ corpus watcher Go package (mtimes, no inotify deps) +bin/server/ async Go HTTP server (goroutines, bounded worker pool) +bin/tools/ vendored python libs behind bin/* (kblib, yamlout, websearch) bin/docker-entrypoint container entrypoint (brain index|search|serve|watch) -serve/ async Go HTTP server (goroutines, bounded worker pool) -tools/ vendored python libs behind bin/* (yamlout, websearch) compose.yaml docker composition (root level, not docker/) -Dockerfile multi-stage: python deps + static Go serve +Dockerfile multi-stage: python deps + static Go binaries var/ kb.lbug, caches (gitignored) .venv/ ladybug + model2vec + mistune ``` diff --git a/Dockerfile b/Dockerfile index 6f14715..1b46f38 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,23 +14,27 @@ COPY requirements.lock.txt /tmp/requirements.lock.txt RUN python -m pip install --no-cache-dir -r /tmp/requirements.lock.txt \ && rm /tmp/requirements.lock.txt -# Go serve: static binary, no interpreter at runtime -FROM golang:1.25 AS serve-build -WORKDIR /src/serve -COPY serve/go.mod serve/go.sum* ./ -COPY serve . -RUN CGO_ENABLED=0 go build -o /serve -ldflags="-s -w" . +# Go services: static binaries, no interpreter at runtime +FROM golang:1.25 AS go-build +WORKDIR /src +COPY go.mod ./ +COPY bin/server ./bin/server +COPY bin/watch ./bin/watch +RUN CGO_ENABLED=0 go build -o /serve ./bin/server \ + && CGO_ENABLED=0 go build -o /watch ./bin/watch -# runtime: python toolchain + Go server +# runtime: python toolchain + Go services FROM base COPY . . -COPY --from=serve-build /serve /app/serve/serve -RUN chmod +x /app/bin/kb-watch /app/bin/docker-entrypoint \ +COPY --from=go-build /serve /app/bin/serve +COPY --from=go-build /watch /app/bin/watch +RUN chmod +x /app/bin/docker-entrypoint \ && chown -R 2dph:2dph /app USER 2dph ENV PATH="/app/bin:${PATH}" \ - KB_PY=python3 + KB_PY=python3 \ + KB_ROOT=/app HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ CMD python -c "import model2vec, ladybug, mistune; print('ok')" || exit 1 diff --git a/README.md b/README.md index 0704712..2d4d8fc 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ touches network/db is read-only, throttled, cached. Tests gate every commit. uv venv .venv # Python 3.12, uv-managed uv pip install -r requirements.lock.txt # pinned toolchain bin/facts/audit self # lexicon consistency gate -go test ./... && python -m unittest discover -s tools -t . +go test ./... && python -m unittest discover -s bin/tools -t . ``` Docker (optional, cached model + var volumes): diff --git a/bin/ci/semver b/bin/ci/semver index b98ce19..dfdd61a 100755 --- a/bin/ci/semver +++ b/bin/ci/semver @@ -17,7 +17,7 @@ import subprocess import sys from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tools")) +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools")) from semver import bump_type, bump_version # noqa: E402 diff --git a/bin/docker-entrypoint b/bin/docker-entrypoint index 451b26e..a00a4d7 100644 --- a/bin/docker-entrypoint +++ b/bin/docker-entrypoint @@ -4,8 +4,8 @@ # brain shell (default) # brain search bin/kb/search # brain index bin/kb/index -# brain watch watchdog re-indexer -# brain serve async Go HTTP server (serve/) +# brain watch watchdog re-indexer (bin/kb/watch) +# brain serve async Go HTTP server (bin/serve) # # Usage comment starts at line 2 (self-describing convention). set -euo pipefail @@ -17,7 +17,7 @@ case "$CMD" in shell) exec bash ;; search) exec "$KB_PY" /app/bin/kb/search "$@" ;; index) exec "$KB_PY" /app/bin/kb/index "$@" ;; - watch) exec bash /app/bin/kb-watch "$@" ;; - serve) exec /app/serve/serve "$@" ;; + watch) exec /app/bin/watch "$@" ;; + serve) exec /app/bin/serve "$@" ;; *) echo "unknown command: $CMD" >&2; exit 2 ;; esac \ No newline at end of file diff --git a/bin/facts/audit b/bin/facts/audit index 655062e..a377bfa 100755 --- a/bin/facts/audit +++ b/bin/facts/audit @@ -20,7 +20,7 @@ import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT / "tools")) +sys.path.insert(0, str(ROOT / "bin" / "tools")) def audit_db() -> list[str]: diff --git a/bin/facts/crm b/bin/facts/crm index 5183cf5..2a0ddbf 100755 --- a/bin/facts/crm +++ b/bin/facts/crm @@ -23,7 +23,7 @@ import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT / "tools")) +sys.path.insert(0, str(ROOT / "bin" / "tools")) from kblib import upsert_leaf, connect, leaf_id # noqa: E402 diff --git a/bin/facts/extract b/bin/facts/extract index 47d5f66..62a9d88 100755 --- a/bin/facts/extract +++ b/bin/facts/extract @@ -21,7 +21,7 @@ import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT / "tools")) +sys.path.insert(0, str(ROOT / "bin" / "tools")) COMPOSE_FILES = [ROOT / "docker" / "compose.yaml", ROOT / "compose.yaml"] DOC_MARKERS = ["README.md", "PLAN.md", "AGENTS.md"] diff --git a/bin/git/import b/bin/git/import new file mode 100755 index 0000000..aed6d74 --- /dev/null +++ b/bin/git/import @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""git/import - import git history (commits, authors, files) into the brain. + + bin/git/import [REPO] import all commits -> leafs + graph + bin/git/import --json emit import leafs as JSON, no write + bin/git/import --limit 100 cap commits processed + bin/git/import --since 2026-01-01 only recent commits + bin/git/import --root DIR run per repo dir under DIR + bin/git/import --no-env never read .env anywhere (default: true) + +Reads `git log --no-merges --name-only` from the repo, maps commits to +`info` leafs (root=info, type=commit) and writes the version graph +`File -[:HAS_VERSION]-> Commit -[:AUTHORED]-> Person` into var/kb.lbug. +Idempotent: leaf MERGE by (source,text via leaf_id), graph MERGE by sha. +""" +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "bin" / "tools")) + +from kblib import ( # noqa: E402 + connect, create_fts_and_vector, drop_indexes, init_schema, upsert_leaf, +) +from gitimport import commits_to_leafs, ensure_git_schema, index_commits, parse_log # noqa: E402 + +LOG_FMT = "--format=%x1e%H%x1f%an%x1f%ae%x1f%aI%x1f%s" + + +def git_log(repo: Path, limit: int = 0, since: str = "") -> str: + cmd = ["git", "-C", str(repo), "log", "--no-merges", "--name-only", LOG_FMT] + if since: + cmd += ["--since", since] + if limit: + cmd += ["-n", str(limit)] + try: + out = subprocess.run(cmd, capture_output=True, text=True, timeout=120) + except (FileNotFoundError, subprocess.TimeoutExpired): + return "" + if out.returncode != 0: + print(f"git/import: {repo}: {out.stderr.strip()}", file=sys.stderr) + return "" + return out.stdout + + +def repo_name(repo: Path) -> str: + try: + out = subprocess.run( + ["git", "-C", str(repo), "remote", "get-url", "origin"], + capture_output=True, text=True, timeout=20) + url = out.stdout.strip() + return url.rstrip("/").split("/")[-1].removesuffix(".git") if url else repo.name + except (FileNotFoundError, subprocess.TimeoutExpired): + return repo.name + + +def embedder(): + from model2vec import StaticModel + model = StaticModel.from_pretrained("minishlab/potion-multilingual-128M") + return lambda text: model.encode([text])[0].astype(float).tolist() + + +def import_repo(conn, repo: Path, embed, limit: int, since: str, + no_write: bool = False) -> tuple[int, int]: + raw = git_log(repo, limit, since) + commits = parse_log(raw) + leafs = commits_to_leafs(commits, repo_name(repo)) + if no_write: + return len(commits), 0 + written = 0 + for lf in leafs: + query = f"{lf['heading']}\n\n{lf['text']}" + emb = embed(lf["text"]) if lf["text"] else None + upsert_leaf(conn, text=query, root="info", confidence="confirmed", + source=lf["source"], source_rev="git", how="git/import", + loc=lf["source"], type_=lf.get("type", "commit"), + embedding=emb) + written += 1 + index_commits(conn, commits, repo_name(repo)) + return len(commits), written + + +def main(argv: list[str]) -> int: + import argparse + p = argparse.ArgumentParser(description="import git history into the brain") + p.add_argument("repo", nargs="?", default=None) + p.add_argument("--root", default=None, help="directory of repos to import (each git dir separately)") + p.add_argument("--limit", type=int, default=0) + p.add_argument("--since", default="") + p.add_argument("--json", action="store_true") + p.add_argument("--dry-run", action="store_true", help="parse + report, no db write") + a = p.parse_args(argv) + + repos: list[Path] = [] + if a.repo: + repos = [Path(a.repo)] + elif a.root: + root = Path(a.root) + if root.is_file(): + repos = [root] + else: + repos = [dp for dp in sorted(root.iterdir()) if (dp / ".git").exists() or dp.is_file()] + else: + repos = [ROOT] + + total_commits = 0 + results: list[dict] = [] + if a.dry_run: + for repo in repos: + if not repo.exists(): + continue + commits = parse_log(git_log(repo, a.limit, a.since)) + name = repo_name(repo) + total_commits += len(commits) + results.append({"repo": name, "commits": len(commits), + "leafs": len(commits_to_leafs(commits, name)), "path": str(repo)}) + if a.json: + print(json.dumps(results, indent=2)) + else: + for r in results: + print(f"{r['repo']:<24} {r['commits']:>5} commits -> {r['leafs']} leafs {r['path']}") + return 0 + + db, conn = connect(ROOT / "var" / "kb.lbug", read_only=False) + init_schema(conn) + drop_indexes(conn) + embed = embedder() + rows: list[dict] = [] + for repo in repos: + if not repo.exists(): + continue + reached, written = import_repo(conn, repo, embed, a.limit, a.since) + total_commits += reached + rows.append({"repo": repo_name(repo), "commits": reached, "written": written}) + create_fts_and_vector(conn, force=True) + conn.close() + db.close() + + if a.json: + print(json.dumps(rows, indent=2)) + else: + for r in rows: + print(f"imported {r['commits']:>5} commits -> {r['written']} leafs {r['repo']}") + print(f"total: {total_commits} commits") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) \ No newline at end of file diff --git a/bin/kb-watch b/bin/kb-watch deleted file mode 100644 index 303b345..0000000 --- a/bin/kb-watch +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -# kb-watch - re-index 2dph when corpus files change. -# -# kb-watch [dir...] [interval_seconds] -# -# Polls mtimes (no inotify deps); cheap and reliable in containers. Defaults: -# dirs = /corpus (compose) or . ; interval = 30s. -set -euo pipefail - -DEFAULT_DIRS="${KB_WATCH_DIRS:-/corpus}" -DIRS=("$@") -[[ ${#DIRS[@]} -eq 0 ]] && DIRS=(${DEFAULT_DIRS}) -INTERVAL="${KB_WATCH_INTERVAL:-30}" - -index() { "${KB_PY:-python3}" /app/bin/kb/index; } - -LAST_STAMP="" -while true; do - STAMP=$(find "${DIRS[@]}" -type f -newermt "-${INTERVAL} seconds" 2>/dev/null \ - | head -1 | md5sum) - if [[ -n "$STAMP" && "$STAMP" != "$LAST_STAMP" ]]; then - echo "kb-watch: changes detected, re-indexing" >&2 - index || echo "kb-watch: index failed; will retry" >&2 - LAST_STAMP="$STAMP" - fi - sleep "$INTERVAL" -done \ No newline at end of file diff --git a/bin/kb/eval b/bin/kb/eval index 72f1edb..828f75c 100755 --- a/bin/kb/eval +++ b/bin/kb/eval @@ -13,7 +13,7 @@ import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT / "tools")) +sys.path.insert(0, str(ROOT / "bin" / "tools")) from kblib import open_readonly, query_fts # noqa: E402 from yamlout import to_yaml # noqa: E402 diff --git a/bin/kb/get b/bin/kb/get index 3054a1b..cf166ea 100755 --- a/bin/kb/get +++ b/bin/kb/get @@ -10,7 +10,7 @@ import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT / "tools")) +sys.path.insert(0, str(ROOT / "bin" / "tools")) from kblib import open_readonly # noqa: E402 from yamlout import to_yaml # noqa: E402 diff --git a/bin/kb/index b/bin/kb/index index c108516..4e22f70 100755 --- a/bin/kb/index +++ b/bin/kb/index @@ -19,7 +19,7 @@ import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT / "tools")) +sys.path.insert(0, str(ROOT / "bin" / "tools")) from kblib import ( # noqa: E402 connect, create_fts_and_vector, init_schema, upsert_leaf, diff --git a/bin/kb/search b/bin/kb/search index 55d95bd..e759371 100755 --- a/bin/kb/search +++ b/bin/kb/search @@ -18,7 +18,7 @@ import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT / "tools")) +sys.path.insert(0, str(ROOT / "bin" / "tools")) from kblib import connect, hybrid_search, init_schema, open_readonly, query_fts # noqa: E402 from yamlout import to_yaml # noqa: E402 @@ -30,6 +30,7 @@ def main(argv: list[str]) -> int: p = argparse.ArgumentParser(description="deduction search over the brain") p.add_argument("query") p.add_argument("--root", choices=("facts", "info", None), default=None) + p.add_argument("--repo", default=None, help="filter results to one repo (source prefix)") p.add_argument("--hop", type=int, default=0) p.add_argument("-n", "--limit", type=int, default=10) p.add_argument("--json", action="store_true") @@ -54,6 +55,9 @@ def main(argv: list[str]) -> int: results = hybrid_search(conn, emb, rhs, a.limit) if a.root: results = [h for h in results if h["root"] == a.root] + if a.repo: + repo = a.repo + results = [h for h in results if repo in (h.get("source") or "")] for hit in results: hit.pop("rrf", None) diff --git a/bin/kb/stats b/bin/kb/stats index e169ab0..a25ba6d 100755 --- a/bin/kb/stats +++ b/bin/kb/stats @@ -11,7 +11,7 @@ import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT / "tools")) +sys.path.insert(0, str(ROOT / "bin" / "tools")) from kblib import open_readonly, stats # noqa: E402 from yamlout import to_yaml # noqa: E402 diff --git a/bin/kb/watch.go b/bin/kb/watch.go new file mode 100755 index 0000000..261b3ad --- /dev/null +++ b/bin/kb/watch.go @@ -0,0 +1,23 @@ +//usr/bin/env go run "$0" "$@"; exit +// bin/kb/watch.go - re-index the 2dph brain when corpus files change. +// +// Usage: +// +// ./bin/kb/watch.go [dir...] # dirs default /corpus +// KB_WATCH_INTERVAL=15 ./bin/kb/watch.go +// +// Shebang trick: first line is a Go `//` comment; the real code lives in the +// importable package (module path, never a relative import). +// NOTE: never run `gofmt -w` on this file - it rewrites `//usr/bin/env` to +// `// usr/...` and breaks the shebang. +package main + +import ( + "os" + + "github.com/eSlider/2dph/bin/watch" +) + +func main() { + watch.Run(os.Args[1:]) +} diff --git a/bin/md/import b/bin/md/import index bf6ebae..85328a1 100755 --- a/bin/md/import +++ b/bin/md/import @@ -1,9 +1,10 @@ #!/usr/bin/env python3 -import lib +from __future__ import annotations + import sys from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent)) +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools")) from mdleaves import leaves_to_json, read_markdown, to_all, walk_markdown # noqa: E402 from yamlout import to_yaml # noqa: E402 diff --git a/bin/serve.go b/bin/serve.go new file mode 100755 index 0000000..ada62bc --- /dev/null +++ b/bin/serve.go @@ -0,0 +1,27 @@ +//usr/bin/env go run "$0" "$@"; exit +// bin/serve.go - async Go HTTP server for the 2dph brain (see bin/server). +// +// KB_ROOT=/path/to/2dph ./bin/serve.go # serve the brain +// KB_SEARCH_CMD=... KB_WORKERS=4 KB_PORT=8630 ./bin/serve.go +// +// Shebang trick: the first line is a Go `//` comment; when executed, env runs +// `go run "$0"` so this file doubles as an executable script. The real code +// lives in the importable package (module path, never a relative import). +// NOTE: never run `gofmt -w` on this file - it rewrites `//usr/bin/env` to +// `// usr/...` and breaks the shebang. +package main + +import ( + "os" + + "github.com/eSlider/2dph/bin/server" +) + +func main() { + if env := os.Getenv("KB_ROOT"); env == "" { + if wd, err := os.Getwd(); err == nil { + os.Setenv("KB_ROOT", wd) + } + } + server.Run() +} diff --git a/serve/main.go b/bin/server/server.go similarity index 86% rename from serve/main.go rename to bin/server/server.go index 5656de8..9e23824 100644 --- a/serve/main.go +++ b/bin/server/server.go @@ -1,9 +1,16 @@ -// Package main serves the 2dph brain over HTTP. +// Package server serves the 2dph brain over HTTP. // // Async by design: every request runs on its own goroutine, and CPU-heavy // searches are serialized through a bounded worker pool (a counting // semaphore) so N requests can't spawn N Python interpreters at once. -package main +// +// Used by bin/serve.go which is a self-executing shebang script: +// +// ///usr/bin/env go run "$0" "$@"; exit +// package main +// import "github.com/eSlider/2dph/bin/server" +// func main() { server.Run() } +package server import ( "context" @@ -115,10 +122,14 @@ func (b *brainSearcher) Search(ctx context.Context, query string, limit int) ([] return out, nil } -func main() { +// Run starts the HTTP server. Reads env: KB_SEARCH_CMD (default bin/kb/search, +// relative to the repo root given by KB_ROOT), KB_WORKERS (default 4), KB_PORT +// (default 8630). +func Run() { + root := os.Getenv("KB_ROOT") searchPath := os.Getenv("KB_SEARCH_CMD") if searchPath == "" { - searchPath = filepath.Join("bin", "kb", "search") + searchPath = filepath.Join(root, "bin", "kb", "search") } workers := 4 if raw := os.Getenv("KB_WORKERS"); raw != "" { @@ -140,4 +151,4 @@ func main() { if err := http.ListenAndServe(addr, handler); err != nil { log.Fatal(err) } -} \ No newline at end of file +} diff --git a/serve/main_test.go b/bin/server/server_test.go similarity index 92% rename from serve/main_test.go rename to bin/server/server_test.go index 8c31ab7..da5641e 100644 --- a/serve/main_test.go +++ b/bin/server/server_test.go @@ -1,11 +1,8 @@ -package main +package server import ( - "bytes" "context" "encoding/json" - "fmt" - "io" "net/http" "net/http/httptest" "sync" @@ -55,10 +52,6 @@ func (f *fakeSearcher) count() int { return f.calls } -func newTestServer(s Searcher, workers int) http.Handler { - return NewServer(s, workers) -} - func get(t *testing.T, h http.Handler, path string) (int, []byte) { t.Helper() req := httptest.NewRequest(http.MethodGet, path, nil) @@ -68,7 +61,7 @@ func get(t *testing.T, h http.Handler, path string) (int, []byte) { } func TestHealth(t *testing.T) { - h := newTestServer(&fakeSearcher{}, 1) + h := NewServer(&fakeSearcher{}, 1) code, body := get(t, h, "/health") if code != http.StatusOK { t.Fatalf("health code = %d, want 200", code) @@ -83,7 +76,7 @@ func TestHealth(t *testing.T) { } func TestSearchMissingQuery(t *testing.T) { - h := newTestServer(&fakeSearcher{}, 1) + h := NewServer(&fakeSearcher{}, 1) if code, _ := get(t, h, "/search"); code != http.StatusBadRequest { t.Fatalf("code = %d, want 400", code) } @@ -93,7 +86,7 @@ func TestSearchReturnsSearcherResult(t *testing.T) { fs := &fakeSearcher{callback: func(q string, limit int) ([]byte, error) { return []byte(`{"query":"` + q + `","count":1,"results":[{"id":"x"}]}`), nil }} - h := newTestServer(fs, 1) + h := NewServer(fs, 1) code, body := get(t, h, "/search?q=matrix") if code != http.StatusOK { t.Fatalf("code = %d, want 200", code) @@ -113,7 +106,7 @@ func TestSearchReturnsSearcherResult(t *testing.T) { func TestSearchConcurrencyBounded(t *testing.T) { // 8 parallel requests on a 3-worker pool: at most 3 concurrent searches. fs := &fakeSearcher{delay: 20 * time.Millisecond} - h := newTestServer(fs, 3) + h := NewServer(fs, 3) var wg sync.WaitGroup for i := 0; i < 8; i++ { @@ -142,7 +135,7 @@ func TestSearchConcurrencyBounded(t *testing.T) { } func TestSearchRejectsBadLimit(t *testing.T) { - h := newTestServer(&fakeSearcher{}, 1) + h := NewServer(&fakeSearcher{}, 1) if code, _ := get(t, h, "/search?q=x&n=hundred"); code != http.StatusBadRequest { t.Fatalf("code = %d, want 400", code) } @@ -171,7 +164,4 @@ func TestSearchTimeout(t *testing.T) { case <-time.After(500 * time.Millisecond): t.Fatal("request hung after context cancellation") } - _ = io.Discard - _ = bytes.MinRead - _ = fmt.Sprintf -} \ No newline at end of file +} diff --git a/tools/__init__.py b/bin/tools/__init__.py similarity index 100% rename from tools/__init__.py rename to bin/tools/__init__.py diff --git a/tools/ci/test_semver_bump.py b/bin/tools/ci/test_semver_bump.py similarity index 100% rename from tools/ci/test_semver_bump.py rename to bin/tools/ci/test_semver_bump.py diff --git a/tools/crmfacts.py b/bin/tools/crmfacts.py similarity index 100% rename from tools/crmfacts.py rename to bin/tools/crmfacts.py diff --git a/bin/tools/gitimport.py b/bin/tools/gitimport.py new file mode 100644 index 0000000..6c71780 --- /dev/null +++ b/bin/tools/gitimport.py @@ -0,0 +1,117 @@ +"""gitimport - parse `git log` output and turn commits into brain leafs. + +Pure, testable functions. Field grammar (see bin/git/import): + + git log --no-merges --name-only \ + --format='%x1e%H%x1f%an%x1f%ae%x1f%aI%x1f%s' + + 0x1e = record separator, 0x1f = field separator. + Files: newline-separated lines following each record's subject. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +REC_SEP = "\x1e" +FIELD_SEP = "\x1f" + + +@dataclass +class Commit: + sha: str + author: str + email: str + date: str + subject: str + files: list[str] = field(default_factory=list) + + def leaf_text(self, repo: str) -> str: + head = f"commit {self.sha[:12]} in {repo} — {self.subject}" + body = [head, f"Author: {self.author} <{self.email}>", f"Date: {self.date}"] + if self.files: + body.append("Changing: " + ", ".join(self.files)) + return "\n".join(body) + + +def parse_log(text: str) -> list[Commit]: + """Parse `git log` output into Commit records. + + Records are separated by 0x1e. A record is fields joined by 0x1f, + followed by optional newline-separated file paths inside the next + segment (git emits blank line + files after each record). + """ + commits: list[Commit] = [] + # field records and file lists alternate; simpler: split on REC_SEP, + # each chunk = header line, possibly followed by newline + files. + for chunk in text.split(REC_SEP): + chunk = chunk.strip("\n") + if not chunk: + continue + lines = chunk.split("\n", 1) + header = lines[0].split(FIELD_SEP) + if len(header) < 5: + continue + sha, author, email, date, subject = header[:5] + files = [ln.strip() for ln in lines[1].splitlines() if ln.strip()] if len(lines) > 1 else [] + commits.append(Commit(sha=sha, author=author, email=email, + date=date, subject=subject, files=files)) + return commits + + +def commits_to_leafs(commits: list[Commit], repo: str) -> list[dict]: + """Map commits to the leaf shape bin/kb/index expects (source/repo/...).""" + out: list[dict] = [] + for c in commits: + out.append({ + "source": f"{repo}@{c.sha}", + "repo": repo, + "heading": f"commit {c.sha[:12]} — {c.subject}", + "text": c.leaf_text(repo), + "type": "commit", + "status": "current", + "related": ",".join(c.files), + }) + return out + + +GIT_SCHEMA = ( + "CREATE NODE TABLE IF NOT EXISTS Commit (id STRING, repo STRING, subject STRING, " + "author STRING, email STRING, date STRING, PRIMARY KEY(id))", + "CREATE NODE TABLE IF NOT EXISTS Person (id STRING, name STRING, email STRING, PRIMARY KEY(id))", + "CREATE REL TABLE IF NOT EXISTS HAS_VERSION (FROM File TO Commit)", + "CREATE REL TABLE IF NOT EXISTS AUTHORED (FROM Commit TO Person)", +) + + +def ensure_git_schema(conn) -> None: + for stmt in GIT_SCHEMA: + conn.execute(stmt) + + +def index_commits(conn, commits: list[Commit], repo: str) -> int: + """Write Commit/File/Person nodes + edges, one per commit (idempotent by sha).""" + ensure_git_schema(conn) + for c in commits: + conn.execute( + "MERGE (c:Commit {id:$sha}) SET c.repo=$repo, c.subject=$subject, " + "c.author=$author, c.email=$email, c.date=$date", + parameters={"sha": c.sha, "repo": repo, "subject": c.subject, + "author": c.author, "email": c.email, "date": c.date}, + ) + conn.execute( + "MERGE (p:Person {id:$email}) SET p.name=$name, p.email=$email", + parameters={"email": c.email, "name": c.author}, + ) + conn.execute("MATCH (c:Commit {id:$sha}), (p:Person {id:$email}) " + "MERGE (c)-[:AUTHORED]->(p)", + parameters={"sha": c.sha, "email": c.email}) + for path in c.files: + conn.execute( + "MERGE (f:File {id:$fid}) SET f.path=$path, f.repo=$repo", + parameters={"fid": f"{repo}:{path}", "path": path, "repo": repo}, + ) + conn.execute("MATCH (f:File {id:$fid}), (c:Commit {id:$sha}) " + "MERGE (f)-[:HAS_VERSION]->(c)", + parameters={"fid": f"{repo}:{path}", "sha": c.sha}) + return len(commits) \ No newline at end of file diff --git a/tools/kblib.py b/bin/tools/kblib.py similarity index 81% rename from tools/kblib.py rename to bin/tools/kblib.py index 33debf5..4d40dd4 100644 --- a/tools/kblib.py +++ b/bin/tools/kblib.py @@ -22,7 +22,17 @@ ROOT_FACTS = "facts" ROOT_INFO = "info" CONF_CONFIRMED = "confirmed" -VAR = Path(__file__).resolve().parents[1] / "var" +def _repo_root() -> Path: + p = Path(__file__).resolve().parent + while True: + if (p / "var").is_dir() or (p / ".git").is_dir() or (p / "pyproject.toml").is_file(): + return p + if p.parent == p: + return Path(__file__).resolve().parents[2] + p = p.parent + + +VAR = _repo_root() / "var" DB_PATH = VAR / "kb.lbug" @@ -68,6 +78,19 @@ def init_schema(conn: ladybug.Connection) -> None: conn.execute( "CREATE REL TABLE IF NOT EXISTS RUNS_ON (FROM Leaf TO Host)" ) + conn.execute( + "CREATE NODE TABLE IF NOT EXISTS Commit (id STRING, repo STRING, subject STRING, " + "author STRING, email STRING, date STRING, PRIMARY KEY(id))" + ) + conn.execute( + "CREATE NODE TABLE IF NOT EXISTS Person (id STRING, name STRING, email STRING, PRIMARY KEY(id))" + ) + conn.execute( + "CREATE REL TABLE IF NOT EXISTS HAS_VERSION (FROM File TO Commit)" + ) + conn.execute( + "CREATE REL TABLE IF NOT EXISTS AUTHORED (FROM Commit TO Person)" + ) def leaf_id(text: str, source: str) -> str: @@ -109,6 +132,18 @@ def create_fts_and_vector(conn: ladybug.Connection, force: bool = False) -> None pass +def drop_indexes(conn: ladybug.Connection) -> None: + """Drop FTS + vector indexes so bulk MERGEs don't corrupt them. + + Ladybug's FTS index goes inconsistent when rows are inserted while the + index exists ("document for node offset N is missing during delete"). + Importers that add many leafs must drop indexes first, write, then + recreate via create_fts_and_vector(). + """ + conn.execute("DROP INDEX IF EXISTS Leaf.Leaf_fts") + conn.execute("DROP INDEX IF EXISTS Leaf.Leaf_vec") + + def query_fts(conn: ladybug.Connection, text: str, limit: int = 10) -> list[dict]: r = conn.execute( "CALL QUERY_FTS_INDEX('Leaf', 'id', $q) " diff --git a/tools/mdleaves.py b/bin/tools/mdleaves.py similarity index 100% rename from tools/mdleaves.py rename to bin/tools/mdleaves.py diff --git a/tools/semver.py b/bin/tools/semver.py similarity index 100% rename from tools/semver.py rename to bin/tools/semver.py diff --git a/tools/test_crm_facts.py b/bin/tools/test_crm_facts.py similarity index 100% rename from tools/test_crm_facts.py rename to bin/tools/test_crm_facts.py diff --git a/bin/tools/test_gitgraph.py b/bin/tools/test_gitgraph.py new file mode 100644 index 0000000..a5a19eb --- /dev/null +++ b/bin/tools/test_gitgraph.py @@ -0,0 +1,71 @@ +import os +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import kblib # noqa: E402 +import gitimport # noqa: E402 + +SAMPLE = ( + "\x1e" + "a1b2c3d" + "\x1f" + "Ada Lovelace" + "\x1f" + "ada@example.com" + + "\x1f" + "2026-08-10T12:00:00+01:00" + "\x1f" + "feat: first commit" + + "\n\nREADME.md\nsrc/main.c\n" +) + +COMMIT_PERSON_SCHEMA = ( + "CREATE NODE TABLE IF NOT EXISTS Commit (id STRING, repo STRING, subject STRING, " + "author STRING, email STRING, date STRING, PRIMARY KEY(id))" +) +PERSON_SCHEMA = ( + "CREATE NODE TABLE IF NOT EXISTS Person (id STRING, name STRING, email STRING, PRIMARY KEY(id))" +) +HAS_VERSION_SCHEMA = "CREATE REL TABLE IF NOT EXISTS HAS_VERSION (FROM File TO Commit)" +AUTHORED_SCHEMA = "CREATE REL TABLE IF NOT EXISTS AUTHORED (FROM Commit TO Person)" + + +class GitGraphTest(unittest.TestCase): + def setUp(self): + self.dir = tempfile.mkdtemp() + self.dbpath = os.path.join(self.dir, "kb.lbug") + self.db, self.conn = kblib.connect(self.dbpath, read_only=False) + kblib.init_schema(self.conn) + self.conn.execute(COMMIT_PERSON_SCHEMA) + self.conn.execute(PERSON_SCHEMA) + self.conn.execute(HAS_VERSION_SCHEMA) + self.conn.execute(AUTHORED_SCHEMA) + + def tearDown(self): + self.conn.close() + self.db.close() + + def test_index_commits_creates_nodes_and_edges(self): + cs = gitimport.parse_log(SAMPLE) + gitimport.index_commits(self.conn, cs, "sample-repo") + rp = self.conn.execute("MATCH (p:Person) RETURN p.name, p.email").get_all() + self.assertEqual([tuple(r) for r in rp], [("Ada Lovelace", "ada@example.com")]) + rc = self.conn.execute("MATCH (c:Commit) RETURN c.id, c.repo").get_all() + self.assertEqual(len(rc), 1) + self.assertEqual(rc[0][1], "sample-repo") + # File -[:HAS_VERSION]-> Commit -[:AUTHORED]-> Person + rf = self.conn.execute( + "MATCH (f:File)-[:HAS_VERSION]->(c:Commit)-[:AUTHORED]->(p:Person) " + "RETURN f.path, c.id, p.email").get_all() + paths = sorted(r[0] for r in rf) + self.assertEqual(paths, ["README.md", "src/main.c"]) + self.assertTrue(all(r[2] == "ada@example.com" for r in rf)) + + def test_index_commits_idempotent(self): + cs = gitimport.parse_log(SAMPLE) + gitimport.index_commits(self.conn, cs, "sample-repo") + gitimport.index_commits(self.conn, cs, "sample-repo") + n = self.conn.execute("MATCH (c:Commit) RETURN count(*)").get_all()[0][0] + self.assertEqual(n, 1) + p = self.conn.execute("MATCH (p:Person) RETURN count(*)").get_all()[0][0] + self.assertEqual(p, 1) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/bin/tools/test_gitimport.py b/bin/tools/test_gitimport.py new file mode 100644 index 0000000..14d8883 --- /dev/null +++ b/bin/tools/test_gitimport.py @@ -0,0 +1,57 @@ +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import gitimport # noqa: E402 + +SAMPLE = ( + "\x1e" + "a1b2c3d" + "\x1f" + "Ada Lovelace" + "\x1f" + "ada@example.com" + + "\x1f" + "2026-08-10T12:00:00+01:00" + "\x1f" + "feat: first commit" + + "\n\nREADME.md\nsrc/main.c\n" + + "\x1e" + "e4f5a6b" + "\x1f" + "Bob Babbage" + "\x1f" + "bob@example.com" + + "\x1f" + "2026-08-11T09:30:00+01:00" + "\x1f" + "fix: typo" + + "\n\ndocs/notes.md" +) + + +class GitparseTest(unittest.TestCase): + def test_parses_records(self): + cs = gitimport.parse_log(SAMPLE) + self.assertEqual(len(cs), 2) + + def test_parses_commit_fields(self): + cs = gitimport.parse_log(SAMPLE) + c = cs[0] + self.assertEqual(c.sha, "a1b2c3d") + self.assertEqual(c.author, "Ada Lovelace") + self.assertEqual(c.email, "ada@example.com") + self.assertEqual(c.date, "2026-08-10T12:00:00+01:00") + self.assertEqual(c.subject, "feat: first commit") + + def test_parses_changed_files(self): + cs = gitimport.parse_log(SAMPLE) + self.assertEqual(cs[0].files, ["README.md", "src/main.c"]) + self.assertEqual(cs[1].files, ["docs/notes.md"]) + + def test_ignores_empty(self): + self.assertEqual(gitimport.parse_log(""), []) + + def test_skip_malformed_record(self): + self.assertEqual(gitimport.parse_log("\x1eweird\x1e"), []) + + def test_commit_leaf_shape(self): + leafs = gitimport.commits_to_leafs(gitimport.parse_log(SAMPLE), "sample-repo") + self.assertEqual(len(leafs), 2) + lf = leafs[0] + self.assertEqual(lf["type"], "commit") + self.assertEqual(lf["repo"], "sample-repo") + self.assertEqual(lf["source"], "sample-repo@a1b2c3d") + self.assertIn("Ada Lovelace", lf["text"]) + self.assertIn("README.md", lf["related"]) + self.assertIn("feat: first commit", lf["heading"]) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tools/test_kblib.py b/bin/tools/test_kblib.py similarity index 100% rename from tools/test_kblib.py rename to bin/tools/test_kblib.py diff --git a/tools/web-search/__init__.py b/bin/tools/web-search/__init__.py similarity index 100% rename from tools/web-search/__init__.py rename to bin/tools/web-search/__init__.py diff --git a/tools/web-search/fixtures/healthy.json b/bin/tools/web-search/fixtures/healthy.json similarity index 100% rename from tools/web-search/fixtures/healthy.json rename to bin/tools/web-search/fixtures/healthy.json diff --git a/tools/web-search/fixtures/throttled.json b/bin/tools/web-search/fixtures/throttled.json similarity index 100% rename from tools/web-search/fixtures/throttled.json rename to bin/tools/web-search/fixtures/throttled.json diff --git a/tools/web-search/test_websearch.py b/bin/tools/web-search/test_websearch.py similarity index 100% rename from tools/web-search/test_websearch.py rename to bin/tools/web-search/test_websearch.py diff --git a/tools/web-search/websearch.py b/bin/tools/web-search/websearch.py similarity index 100% rename from tools/web-search/websearch.py rename to bin/tools/web-search/websearch.py diff --git a/tools/yamlout.py b/bin/tools/yamlout.py similarity index 100% rename from tools/yamlout.py rename to bin/tools/yamlout.py diff --git a/bin/watch/watch.go b/bin/watch/watch.go new file mode 100644 index 0000000..c7669c0 --- /dev/null +++ b/bin/watch/watch.go @@ -0,0 +1,105 @@ +// Package watch polls corpus directories for changes and re-runs bin/kb/index. +// +// Port of the former bin/kb-watch bash script to an importable, testable Go +// package. Polls file mtimes (no inotify deps); cheap and reliable. +package watch + +import ( + "log" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" +) + +// Options controls the polling loop. Zero value uses defaults. +type Options struct { + Dirs []string + Interval time.Duration + // IndexCmd is the kb/index command template. %s is replaced by the repo + // root (from KB_ROOT). Defaults to `python3 /bin/kb/index`. + IndexCmd string +} + +// Run blocks forever polling Dirs (defaults: KB_WATCH_DIRS or /corpus) every +// Interval (default 30s) and re-indexing when files change. KB_ROOT names the +// repo root used to locate bin/kb/index. +func Run(args []string) { + opts := fromEnv(args) + root, _ := os.Getwd() + if r := os.Getenv("KB_ROOT"); r != "" { + root = r + } + log.Printf("watch: dirs=%v interval=%s root=%s", opts.Dirs, opts.Interval, root) + var last string + for { + if flag := Stamp(opts.Dirs); flag != "" && flag != last { + last = flag + reindex(opts.IndexCmd, root) + } + time.Sleep(opts.Interval) + } +} + +func fromEnv(args []string) Options { + opts := Options{Interval: 30 * time.Second} + if raw := os.Getenv("KB_WATCH_INTERVAL"); raw != "" { + if n, err := strconv.Atoi(raw); err == nil && n > 0 { + opts.Interval = time.Duration(n) * time.Second + } + } + defDirs := "/corpus" + if raw := os.Getenv("KB_WATCH_DIRS"); raw != "" { + defDirs = raw + } + if len(args) > 0 { + opts.Dirs = args + } else { + for _, d := range strings.Split(defDirs, " ") { + if d != "" { + opts.Dirs = append(opts.Dirs, d) + } + } + } + pys := os.Getenv("KB_PY") + if pys == "" { + pys = "python3" + } + opts.IndexCmd = pys + " /bin/kb/index" + return opts +} + +// Stamp returns a rolling fingerprint (newest mtime under dirs) that changes +// whenever any corpus file is touched. Empty when no files found. +func Stamp(dirs []string) string { + var newest time.Time + for _, dir := range dirs { + _ = filepath.WalkDir(dir, func(path string, _ os.DirEntry, err error) error { + if err != nil { + return nil + } + if info, e := os.Stat(path); e == nil && info.ModTime().After(newest) { + newest = info.ModTime() + } + return nil + }) + } + if newest.IsZero() { + return "" + } + return strconv.FormatInt(newest.UnixNano(), 10) +} + +func reindex(template, root string) { + cmd := strings.ReplaceAll(template, "", root) + parts := strings.Fields(cmd) + c := exec.Command(parts[0], parts[1:]...) + out, err := c.CombinedOutput() + if err != nil { + log.Printf("watch: index failed: %v\n%s", err, out) + } else { + log.Printf("watch: re-indexed") + } +} diff --git a/bin/watch/watch_test.go b/bin/watch/watch_test.go new file mode 100644 index 0000000..2292def --- /dev/null +++ b/bin/watch/watch_test.go @@ -0,0 +1,49 @@ +package watch + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestStampChangesWhenFileTouched(t *testing.T) { + dir := t.TempDir() + a := filepath.Join(dir, "a.md") + if err := os.WriteFile(a, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + s1 := Stamp([]string{dir}) + if s1 == "" { + t.Fatal("stamp empty for a dir with a file") + } + time.Sleep(10 * time.Millisecond) + if err := os.WriteFile(a, []byte("y"), 0o644); err != nil { + t.Fatal(err) + } + if s2 := Stamp([]string{dir}); s2 == s1 { + t.Fatal("stamp did not change after the file was modified") + } +} + +func TestStampEmptyForMissingDir(t *testing.T) { + if s := Stamp([]string{filepath.Join(t.TempDir(), "nope")}); s != "" { + t.Fatalf("stamp = %q, want empty for missing dir", s) + } +} + +func TestFromEnvDefaults(t *testing.T) { + t.Setenv("KB_WATCH_INTERVAL", "") + t.Setenv("KB_WATCH_DIRS", "") + t.Setenv("KB_PY", "") + opts := fromEnv(nil) + if len(opts.Dirs) == 0 || opts.Dirs[0] != "/corpus" { + t.Fatalf("default dirs = %v, want [/corpus]", opts.Dirs) + } + if opts.Interval != 30*time.Second { + t.Fatalf("default interval = %s, want 30s", opts.Interval) + } + if opts.IndexCmd == "" { + t.Fatal("default index cmd is empty") + } +} diff --git a/bin/web/search b/bin/web/search index 865ba15..87984b8 100755 --- a/bin/web/search +++ b/bin/web/search @@ -25,7 +25,7 @@ import urllib.parse import urllib.request from pathlib import Path -TOOLS = Path(__file__).resolve().parents[1].parent / "tools" +TOOLS = Path(__file__).resolve().parents[1] / "tools" sys.path.insert(0, str(TOOLS)) sys.path.insert(0, str(TOOLS / "web-search")) diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..2aa0757 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/eSlider/2dph + +go 1.25 \ No newline at end of file diff --git a/serve/go.mod b/serve/go.mod deleted file mode 100644 index 808dbab..0000000 --- a/serve/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module github.com/eSlider/2dph/serve - -go 1.25 \ No newline at end of file