refactor(tools): bin/{subject}/{method} layout; Go serve+watch modules
Move serve/ (module) -> bin/server, tools/ -> bin/tools, replace bin/kb-watch bash with bin/watch Go package; self-executing Go shebangs bin/serve.go and bin/kb/watch.go; Docker + CI + git/import + docs repointed. Multi-stage image builds static serve+watch binaries (no Go runtime in container).
This commit is contained in:
+1
-1
@@ -17,7 +17,7 @@ import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tools"))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools"))
|
||||
|
||||
from semver import bump_type, bump_version # noqa: E402
|
||||
|
||||
|
||||
@@ -4,8 +4,8 @@
|
||||
# brain shell (default)
|
||||
# brain search <q> bin/kb/search
|
||||
# brain index bin/kb/index
|
||||
# brain watch <dir> watchdog re-indexer
|
||||
# brain serve async Go HTTP server (serve/)
|
||||
# brain watch <dir> watchdog re-indexer (bin/kb/watch)
|
||||
# brain serve async Go HTTP server (bin/serve)
|
||||
#
|
||||
# Usage comment starts at line 2 (self-describing convention).
|
||||
set -euo pipefail
|
||||
@@ -17,7 +17,7 @@ case "$CMD" in
|
||||
shell) exec bash ;;
|
||||
search) exec "$KB_PY" /app/bin/kb/search "$@" ;;
|
||||
index) exec "$KB_PY" /app/bin/kb/index "$@" ;;
|
||||
watch) exec bash /app/bin/kb-watch "$@" ;;
|
||||
serve) exec /app/serve/serve "$@" ;;
|
||||
watch) exec /app/bin/watch "$@" ;;
|
||||
serve) exec /app/bin/serve "$@" ;;
|
||||
*) echo "unknown command: $CMD" >&2; exit 2 ;;
|
||||
esac
|
||||
+1
-1
@@ -20,7 +20,7 @@ import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "tools"))
|
||||
sys.path.insert(0, str(ROOT / "bin" / "tools"))
|
||||
|
||||
|
||||
def audit_db() -> list[str]:
|
||||
|
||||
+1
-1
@@ -23,7 +23,7 @@ import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "tools"))
|
||||
sys.path.insert(0, str(ROOT / "bin" / "tools"))
|
||||
|
||||
from kblib import upsert_leaf, connect, leaf_id # noqa: E402
|
||||
|
||||
|
||||
+1
-1
@@ -21,7 +21,7 @@ import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "tools"))
|
||||
sys.path.insert(0, str(ROOT / "bin" / "tools"))
|
||||
|
||||
COMPOSE_FILES = [ROOT / "docker" / "compose.yaml", ROOT / "compose.yaml"]
|
||||
DOC_MARKERS = ["README.md", "PLAN.md", "AGENTS.md"]
|
||||
|
||||
Executable
+153
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env python3
|
||||
"""git/import - import git history (commits, authors, files) into the brain.
|
||||
|
||||
bin/git/import [REPO] import all commits -> leafs + graph
|
||||
bin/git/import --json emit import leafs as JSON, no write
|
||||
bin/git/import --limit 100 cap commits processed
|
||||
bin/git/import --since 2026-01-01 only recent commits
|
||||
bin/git/import --root DIR run per repo dir under DIR
|
||||
bin/git/import --no-env never read .env anywhere (default: true)
|
||||
|
||||
Reads `git log --no-merges --name-only` from the repo, maps commits to
|
||||
`info` leafs (root=info, type=commit) and writes the version graph
|
||||
`File -[:HAS_VERSION]-> Commit -[:AUTHORED]-> Person` into var/kb.lbug.
|
||||
Idempotent: leaf MERGE by (source,text via leaf_id), graph MERGE by sha.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "bin" / "tools"))
|
||||
|
||||
from kblib import ( # noqa: E402
|
||||
connect, create_fts_and_vector, drop_indexes, init_schema, upsert_leaf,
|
||||
)
|
||||
from gitimport import commits_to_leafs, ensure_git_schema, index_commits, parse_log # noqa: E402
|
||||
|
||||
LOG_FMT = "--format=%x1e%H%x1f%an%x1f%ae%x1f%aI%x1f%s"
|
||||
|
||||
|
||||
def git_log(repo: Path, limit: int = 0, since: str = "") -> str:
|
||||
cmd = ["git", "-C", str(repo), "log", "--no-merges", "--name-only", LOG_FMT]
|
||||
if since:
|
||||
cmd += ["--since", since]
|
||||
if limit:
|
||||
cmd += ["-n", str(limit)]
|
||||
try:
|
||||
out = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
|
||||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||||
return ""
|
||||
if out.returncode != 0:
|
||||
print(f"git/import: {repo}: {out.stderr.strip()}", file=sys.stderr)
|
||||
return ""
|
||||
return out.stdout
|
||||
|
||||
|
||||
def repo_name(repo: Path) -> str:
|
||||
try:
|
||||
out = subprocess.run(
|
||||
["git", "-C", str(repo), "remote", "get-url", "origin"],
|
||||
capture_output=True, text=True, timeout=20)
|
||||
url = out.stdout.strip()
|
||||
return url.rstrip("/").split("/")[-1].removesuffix(".git") if url else repo.name
|
||||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||||
return repo.name
|
||||
|
||||
|
||||
def embedder():
|
||||
from model2vec import StaticModel
|
||||
model = StaticModel.from_pretrained("minishlab/potion-multilingual-128M")
|
||||
return lambda text: model.encode([text])[0].astype(float).tolist()
|
||||
|
||||
|
||||
def import_repo(conn, repo: Path, embed, limit: int, since: str,
|
||||
no_write: bool = False) -> tuple[int, int]:
|
||||
raw = git_log(repo, limit, since)
|
||||
commits = parse_log(raw)
|
||||
leafs = commits_to_leafs(commits, repo_name(repo))
|
||||
if no_write:
|
||||
return len(commits), 0
|
||||
written = 0
|
||||
for lf in leafs:
|
||||
query = f"{lf['heading']}\n\n{lf['text']}"
|
||||
emb = embed(lf["text"]) if lf["text"] else None
|
||||
upsert_leaf(conn, text=query, root="info", confidence="confirmed",
|
||||
source=lf["source"], source_rev="git", how="git/import",
|
||||
loc=lf["source"], type_=lf.get("type", "commit"),
|
||||
embedding=emb)
|
||||
written += 1
|
||||
index_commits(conn, commits, repo_name(repo))
|
||||
return len(commits), written
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
import argparse
|
||||
p = argparse.ArgumentParser(description="import git history into the brain")
|
||||
p.add_argument("repo", nargs="?", default=None)
|
||||
p.add_argument("--root", default=None, help="directory of repos to import (each git dir separately)")
|
||||
p.add_argument("--limit", type=int, default=0)
|
||||
p.add_argument("--since", default="")
|
||||
p.add_argument("--json", action="store_true")
|
||||
p.add_argument("--dry-run", action="store_true", help="parse + report, no db write")
|
||||
a = p.parse_args(argv)
|
||||
|
||||
repos: list[Path] = []
|
||||
if a.repo:
|
||||
repos = [Path(a.repo)]
|
||||
elif a.root:
|
||||
root = Path(a.root)
|
||||
if root.is_file():
|
||||
repos = [root]
|
||||
else:
|
||||
repos = [dp for dp in sorted(root.iterdir()) if (dp / ".git").exists() or dp.is_file()]
|
||||
else:
|
||||
repos = [ROOT]
|
||||
|
||||
total_commits = 0
|
||||
results: list[dict] = []
|
||||
if a.dry_run:
|
||||
for repo in repos:
|
||||
if not repo.exists():
|
||||
continue
|
||||
commits = parse_log(git_log(repo, a.limit, a.since))
|
||||
name = repo_name(repo)
|
||||
total_commits += len(commits)
|
||||
results.append({"repo": name, "commits": len(commits),
|
||||
"leafs": len(commits_to_leafs(commits, name)), "path": str(repo)})
|
||||
if a.json:
|
||||
print(json.dumps(results, indent=2))
|
||||
else:
|
||||
for r in results:
|
||||
print(f"{r['repo']:<24} {r['commits']:>5} commits -> {r['leafs']} leafs {r['path']}")
|
||||
return 0
|
||||
|
||||
db, conn = connect(ROOT / "var" / "kb.lbug", read_only=False)
|
||||
init_schema(conn)
|
||||
drop_indexes(conn)
|
||||
embed = embedder()
|
||||
rows: list[dict] = []
|
||||
for repo in repos:
|
||||
if not repo.exists():
|
||||
continue
|
||||
reached, written = import_repo(conn, repo, embed, a.limit, a.since)
|
||||
total_commits += reached
|
||||
rows.append({"repo": repo_name(repo), "commits": reached, "written": written})
|
||||
create_fts_and_vector(conn, force=True)
|
||||
conn.close()
|
||||
db.close()
|
||||
|
||||
if a.json:
|
||||
print(json.dumps(rows, indent=2))
|
||||
else:
|
||||
for r in rows:
|
||||
print(f"imported {r['commits']:>5} commits -> {r['written']} leafs {r['repo']}")
|
||||
print(f"total: {total_commits} commits")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
@@ -1,27 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# kb-watch - re-index 2dph when corpus files change.
|
||||
#
|
||||
# kb-watch [dir...] [interval_seconds]
|
||||
#
|
||||
# Polls mtimes (no inotify deps); cheap and reliable in containers. Defaults:
|
||||
# dirs = /corpus (compose) or . ; interval = 30s.
|
||||
set -euo pipefail
|
||||
|
||||
DEFAULT_DIRS="${KB_WATCH_DIRS:-/corpus}"
|
||||
DIRS=("$@")
|
||||
[[ ${#DIRS[@]} -eq 0 ]] && DIRS=(${DEFAULT_DIRS})
|
||||
INTERVAL="${KB_WATCH_INTERVAL:-30}"
|
||||
|
||||
index() { "${KB_PY:-python3}" /app/bin/kb/index; }
|
||||
|
||||
LAST_STAMP=""
|
||||
while true; do
|
||||
STAMP=$(find "${DIRS[@]}" -type f -newermt "-${INTERVAL} seconds" 2>/dev/null \
|
||||
| head -1 | md5sum)
|
||||
if [[ -n "$STAMP" && "$STAMP" != "$LAST_STAMP" ]]; then
|
||||
echo "kb-watch: changes detected, re-indexing" >&2
|
||||
index || echo "kb-watch: index failed; will retry" >&2
|
||||
LAST_STAMP="$STAMP"
|
||||
fi
|
||||
sleep "$INTERVAL"
|
||||
done
|
||||
+1
-1
@@ -13,7 +13,7 @@ import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "tools"))
|
||||
sys.path.insert(0, str(ROOT / "bin" / "tools"))
|
||||
|
||||
from kblib import open_readonly, query_fts # noqa: E402
|
||||
from yamlout import to_yaml # noqa: E402
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@ import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "tools"))
|
||||
sys.path.insert(0, str(ROOT / "bin" / "tools"))
|
||||
|
||||
from kblib import open_readonly # noqa: E402
|
||||
from yamlout import to_yaml # noqa: E402
|
||||
|
||||
+1
-1
@@ -19,7 +19,7 @@ import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "tools"))
|
||||
sys.path.insert(0, str(ROOT / "bin" / "tools"))
|
||||
|
||||
from kblib import ( # noqa: E402
|
||||
connect, create_fts_and_vector, init_schema, upsert_leaf,
|
||||
|
||||
+5
-1
@@ -18,7 +18,7 @@ import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "tools"))
|
||||
sys.path.insert(0, str(ROOT / "bin" / "tools"))
|
||||
|
||||
from kblib import connect, hybrid_search, init_schema, open_readonly, query_fts # noqa: E402
|
||||
from yamlout import to_yaml # noqa: E402
|
||||
@@ -30,6 +30,7 @@ def main(argv: list[str]) -> int:
|
||||
p = argparse.ArgumentParser(description="deduction search over the brain")
|
||||
p.add_argument("query")
|
||||
p.add_argument("--root", choices=("facts", "info", None), default=None)
|
||||
p.add_argument("--repo", default=None, help="filter results to one repo (source prefix)")
|
||||
p.add_argument("--hop", type=int, default=0)
|
||||
p.add_argument("-n", "--limit", type=int, default=10)
|
||||
p.add_argument("--json", action="store_true")
|
||||
@@ -54,6 +55,9 @@ def main(argv: list[str]) -> int:
|
||||
results = hybrid_search(conn, emb, rhs, a.limit)
|
||||
if a.root:
|
||||
results = [h for h in results if h["root"] == a.root]
|
||||
if a.repo:
|
||||
repo = a.repo
|
||||
results = [h for h in results if repo in (h.get("source") or "")]
|
||||
|
||||
for hit in results:
|
||||
hit.pop("rrf", None)
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@ import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "tools"))
|
||||
sys.path.insert(0, str(ROOT / "bin" / "tools"))
|
||||
|
||||
from kblib import open_readonly, stats # noqa: E402
|
||||
from yamlout import to_yaml # noqa: E402
|
||||
|
||||
Executable
+23
@@ -0,0 +1,23 @@
|
||||
//usr/bin/env go run "$0" "$@"; exit
|
||||
// bin/kb/watch.go - re-index the 2dph brain when corpus files change.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// ./bin/kb/watch.go [dir...] # dirs default /corpus
|
||||
// KB_WATCH_INTERVAL=15 ./bin/kb/watch.go
|
||||
//
|
||||
// Shebang trick: first line is a Go `//` comment; the real code lives in the
|
||||
// importable package (module path, never a relative import).
|
||||
// NOTE: never run `gofmt -w` on this file - it rewrites `//usr/bin/env` to
|
||||
// `// usr/...` and breaks the shebang.
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/eSlider/2dph/bin/watch"
|
||||
)
|
||||
|
||||
func main() {
|
||||
watch.Run(os.Args[1:])
|
||||
}
|
||||
+3
-2
@@ -1,9 +1,10 @@
|
||||
#!/usr/bin/env python3
|
||||
import lib
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools"))
|
||||
|
||||
from mdleaves import leaves_to_json, read_markdown, to_all, walk_markdown # noqa: E402
|
||||
from yamlout import to_yaml # noqa: E402
|
||||
|
||||
Executable
+27
@@ -0,0 +1,27 @@
|
||||
//usr/bin/env go run "$0" "$@"; exit
|
||||
// bin/serve.go - async Go HTTP server for the 2dph brain (see bin/server).
|
||||
//
|
||||
// KB_ROOT=/path/to/2dph ./bin/serve.go # serve the brain
|
||||
// KB_SEARCH_CMD=... KB_WORKERS=4 KB_PORT=8630 ./bin/serve.go
|
||||
//
|
||||
// Shebang trick: the first line is a Go `//` comment; when executed, env runs
|
||||
// `go run "$0"` so this file doubles as an executable script. The real code
|
||||
// lives in the importable package (module path, never a relative import).
|
||||
// NOTE: never run `gofmt -w` on this file - it rewrites `//usr/bin/env` to
|
||||
// `// usr/...` and breaks the shebang.
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/eSlider/2dph/bin/server"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if env := os.Getenv("KB_ROOT"); env == "" {
|
||||
if wd, err := os.Getwd(); err == nil {
|
||||
os.Setenv("KB_ROOT", wd)
|
||||
}
|
||||
}
|
||||
server.Run()
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
// Package server serves the 2dph brain over HTTP.
|
||||
//
|
||||
// Async by design: every request runs on its own goroutine, and CPU-heavy
|
||||
// searches are serialized through a bounded worker pool (a counting
|
||||
// semaphore) so N requests can't spawn N Python interpreters at once.
|
||||
//
|
||||
// Used by bin/serve.go which is a self-executing shebang script:
|
||||
//
|
||||
// ///usr/bin/env go run "$0" "$@"; exit
|
||||
// package main
|
||||
// import "github.com/eSlider/2dph/bin/server"
|
||||
// func main() { server.Run() }
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Searcher interface {
|
||||
Search(ctx context.Context, query string, limit int) ([]byte, error)
|
||||
}
|
||||
|
||||
type Server struct {
|
||||
searcher Searcher
|
||||
semaphore chan struct{}
|
||||
}
|
||||
|
||||
const defaultPort = 8630
|
||||
|
||||
func NewServer(searcher Searcher, workers int) http.Handler {
|
||||
return &Server{
|
||||
searcher: searcher,
|
||||
semaphore: make(chan struct{}, workers),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.URL.Path == "/health":
|
||||
writeJSON(w, http.StatusOK, map[string]any{"status": "ok"})
|
||||
case r.URL.Path == "/search":
|
||||
s.handleSearch(w, r)
|
||||
default:
|
||||
writeJSON(w, http.StatusNotFound, map[string]any{"error": "not found"})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) {
|
||||
q := strings.TrimSpace(r.URL.Query().Get("q"))
|
||||
if q == "" {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]any{"error": "q required"})
|
||||
return
|
||||
}
|
||||
limit := 10
|
||||
if raw := r.URL.Query().Get("n"); raw != "" {
|
||||
n, err := strconv.Atoi(raw)
|
||||
if err != nil || n < 1 || n > 100 {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]any{"error": "n must be int 1..100"})
|
||||
return
|
||||
}
|
||||
limit = n
|
||||
}
|
||||
|
||||
// Worker pool: block until a slot frees, so burst concurrency still
|
||||
// bounds memory (no unbounded python processes).
|
||||
select {
|
||||
case s.semaphore <- struct{}{}:
|
||||
defer func() { <-s.semaphore }()
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
}
|
||||
|
||||
body, err := s.searcher.Search(r.Context(), q, limit)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusGatewayTimeout, map[string]any{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
writeRaw(w, http.StatusOK, body)
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, code int, obj any) {
|
||||
body, _ := json.Marshal(obj)
|
||||
writeRaw(w, code, body)
|
||||
}
|
||||
|
||||
func writeRaw(w http.ResponseWriter, code int, body []byte) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
|
||||
w.WriteHeader(code)
|
||||
w.Write(body)
|
||||
}
|
||||
|
||||
// brainSearcher shells out to bin/kb/search --json. A single python search
|
||||
// is bounded and short-lived; the worker pool keeps at most N live.
|
||||
type brainSearcher struct {
|
||||
cmdPath string
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
func (b *brainSearcher) Search(ctx context.Context, query string, limit int) ([]byte, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, b.timeout)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, b.cmdPath, "--json", "-n", strconv.Itoa(limit), query)
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
var exitErr *exec.ExitError
|
||||
if errors.As(err, &exitErr) {
|
||||
return nil, errors.New("search backend failed: " + strings.TrimSpace(string(exitErr.Stderr)))
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Run starts the HTTP server. Reads env: KB_SEARCH_CMD (default bin/kb/search,
|
||||
// relative to the repo root given by KB_ROOT), KB_WORKERS (default 4), KB_PORT
|
||||
// (default 8630).
|
||||
func Run() {
|
||||
root := os.Getenv("KB_ROOT")
|
||||
searchPath := os.Getenv("KB_SEARCH_CMD")
|
||||
if searchPath == "" {
|
||||
searchPath = filepath.Join(root, "bin", "kb", "search")
|
||||
}
|
||||
workers := 4
|
||||
if raw := os.Getenv("KB_WORKERS"); raw != "" {
|
||||
if n, err := strconv.Atoi(raw); err == nil && n > 0 {
|
||||
workers = n
|
||||
}
|
||||
}
|
||||
port := defaultPort
|
||||
if raw := os.Getenv("KB_PORT"); raw != "" {
|
||||
if n, err := strconv.Atoi(raw); err == nil && n > 0 {
|
||||
port = n
|
||||
}
|
||||
}
|
||||
|
||||
searcher := &brainSearcher{cmdPath: searchPath, timeout: 60 * time.Second}
|
||||
handler := NewServer(searcher, workers)
|
||||
addr := "127.0.0.1:" + strconv.Itoa(port)
|
||||
log.Printf("serve: %s (workers=%d)", addr, workers)
|
||||
if err := http.ListenAndServe(addr, handler); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeSearcher is an injectable Searcher for tests (no python involved).
|
||||
type fakeSearcher struct {
|
||||
mu sync.Mutex
|
||||
delay time.Duration
|
||||
calls int
|
||||
active atomic.Int32
|
||||
maxSeen atomic.Int32
|
||||
callback func(q string, limit int) ([]byte, error)
|
||||
}
|
||||
|
||||
func (f *fakeSearcher) Search(ctx context.Context, query string, limit int) ([]byte, error) {
|
||||
f.mu.Lock()
|
||||
f.calls++
|
||||
f.mu.Unlock()
|
||||
n := f.active.Add(1)
|
||||
for {
|
||||
old := f.maxSeen.Load()
|
||||
if n <= old || f.maxSeen.CompareAndSwap(old, n) {
|
||||
break
|
||||
}
|
||||
}
|
||||
defer f.active.Add(-1)
|
||||
if f.delay > 0 {
|
||||
select {
|
||||
case <-time.After(f.delay):
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
if f.callback != nil {
|
||||
return f.callback(query, limit)
|
||||
}
|
||||
return []byte(`{"query":"` + query + `","count":0,"results":[]}`), nil
|
||||
}
|
||||
|
||||
func (f *fakeSearcher) count() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.calls
|
||||
}
|
||||
|
||||
func get(t *testing.T, h http.Handler, path string) (int, []byte) {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, req)
|
||||
return rec.Code, rec.Body.Bytes()
|
||||
}
|
||||
|
||||
func TestHealth(t *testing.T) {
|
||||
h := NewServer(&fakeSearcher{}, 1)
|
||||
code, body := get(t, h, "/health")
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("health code = %d, want 200", code)
|
||||
}
|
||||
var out map[string]any
|
||||
if err := json.Unmarshal(body, &out); err != nil {
|
||||
t.Fatalf("health body not json: %v (%s)", err, body)
|
||||
}
|
||||
if out["status"] != "ok" {
|
||||
t.Fatalf("health status = %v, want ok", out["status"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchMissingQuery(t *testing.T) {
|
||||
h := NewServer(&fakeSearcher{}, 1)
|
||||
if code, _ := get(t, h, "/search"); code != http.StatusBadRequest {
|
||||
t.Fatalf("code = %d, want 400", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchReturnsSearcherResult(t *testing.T) {
|
||||
fs := &fakeSearcher{callback: func(q string, limit int) ([]byte, error) {
|
||||
return []byte(`{"query":"` + q + `","count":1,"results":[{"id":"x"}]}`), nil
|
||||
}}
|
||||
h := NewServer(fs, 1)
|
||||
code, body := get(t, h, "/search?q=matrix")
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("code = %d, want 200", code)
|
||||
}
|
||||
var out map[string]any
|
||||
if err := json.Unmarshal(body, &out); err != nil {
|
||||
t.Fatalf("body not json: %v (%s)", err, body)
|
||||
}
|
||||
if out["query"] != "matrix" {
|
||||
t.Fatalf("query = %v, want matrix", out["query"])
|
||||
}
|
||||
if fs.count() != 1 {
|
||||
t.Fatalf("searcher called %d times, want 1", fs.count())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchConcurrencyBounded(t *testing.T) {
|
||||
// 8 parallel requests on a 3-worker pool: at most 3 concurrent searches.
|
||||
fs := &fakeSearcher{delay: 20 * time.Millisecond}
|
||||
h := NewServer(fs, 3)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 8; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
req := httptest.NewRequest(http.MethodGet, "/search?q=abc", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Errorf("code = %d, want 200", rec.Code)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
if calls := fs.count(); calls != 8 {
|
||||
t.Fatalf("searcher called %d times, want 8", calls)
|
||||
}
|
||||
if max := fs.maxSeen.Load(); max > 3 {
|
||||
t.Fatalf("max concurrent = %d, want <= 3", max)
|
||||
}
|
||||
if max := fs.maxSeen.Load(); max < 1 {
|
||||
t.Fatalf("max concurrent = %d, want >= 1", max)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchRejectsBadLimit(t *testing.T) {
|
||||
h := NewServer(&fakeSearcher{}, 1)
|
||||
if code, _ := get(t, h, "/search?q=x&n=hundred"); code != http.StatusBadRequest {
|
||||
t.Fatalf("code = %d, want 400", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchTimeout(t *testing.T) {
|
||||
fs := &fakeSearcher{delay: time.Second}
|
||||
h := NewServer(fs, 1)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond)
|
||||
defer cancel()
|
||||
req := httptest.NewRequest(http.MethodGet, "/search?q=slow", nil).WithContext(ctx)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
h.ServeHTTP(rec, req)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
// ServeHTTP returned; body should be an error json (we don't require a
|
||||
// specific code for the pathological ctx-cancel timing, only that it
|
||||
// does not hang forever).
|
||||
return
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
t.Fatal("request hung after context cancellation")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from semver import bump_type, bump_version # noqa: E402
|
||||
|
||||
|
||||
class BumpTypeTest(unittest.TestCase):
|
||||
def test_empty_is_none(self):
|
||||
self.assertEqual(bump_type([]), "none")
|
||||
|
||||
def test_feat_is_minor(self):
|
||||
self.assertEqual(bump_type(["feat: add search"]), "minor")
|
||||
|
||||
def test_fix_is_patch(self):
|
||||
self.assertEqual(bump_type(["fix: typo"]), "patch")
|
||||
|
||||
def test_chore_and_docs_still_release(self):
|
||||
self.assertEqual(bump_type(["docs: readme"]), "patch")
|
||||
self.assertEqual(bump_type(["ci: green"]), "patch")
|
||||
|
||||
def test_breaking_marker_is_major(self):
|
||||
self.assertEqual(bump_type(["feat!: break api"]), "major")
|
||||
self.assertEqual(bump_type(["fix: x\n\nBREAKING CHANGE: y"]), "major")
|
||||
|
||||
def test_mixed_commits_choose_highest(self):
|
||||
self.assertEqual(bump_type(["fix: a", "feat: b"]), "minor")
|
||||
|
||||
|
||||
class BumpVersionTest(unittest.TestCase):
|
||||
def test_patch(self):
|
||||
self.assertEqual(bump_version("v0.1.0", "patch"), "v0.1.1")
|
||||
|
||||
def test_minor(self):
|
||||
self.assertEqual(bump_version("v0.1.0", "minor"), "v0.2.0")
|
||||
|
||||
def test_major(self):
|
||||
self.assertEqual(bump_version("v0.1.0", "major"), "v1.0.0")
|
||||
|
||||
def test_initial_when_no_tag(self):
|
||||
self.assertEqual(bump_version(None, "patch"), "v0.0.1")
|
||||
|
||||
def test_none_returns_none(self):
|
||||
self.assertIsNone(bump_version("v0.1.0", "none"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,29 @@
|
||||
"""crmfacts - pure helpers for bin/facts/crm (association proofing).
|
||||
|
||||
Shared with tools/ unit tests so the corpus-org parser is covered in CI.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
|
||||
def corpus_orgs(raw: str) -> dict[str, dict]:
|
||||
"""Parse the orgs block of the CV knowledge-mesh YAML into id -> fields.
|
||||
|
||||
Fields kept: label, kind, period, website. Stops at the first sibling
|
||||
top-level key (clients, timeline, ...).
|
||||
"""
|
||||
m = re.search(r"^orgs:\n(.*?)\n^(?:clients|timeline|tech_weights|nodes|edges):", raw, re.S | re.M)
|
||||
if not m:
|
||||
return {}
|
||||
orgs: dict[str, dict] = {}
|
||||
cur = None
|
||||
for line in m.group(1).splitlines():
|
||||
lm = re.match(r"^\s*- id:\s*(\S+)", line)
|
||||
if lm:
|
||||
cur = lm.group(1)
|
||||
orgs[cur] = {}
|
||||
continue
|
||||
fm = re.match(r"^\s+(\w+):\s*(.*)$", line)
|
||||
if fm and cur and fm.group(1) in ("label", "kind", "period", "website"):
|
||||
orgs[cur][fm.group(1)] = fm.group(2).strip()
|
||||
return orgs
|
||||
@@ -0,0 +1,117 @@
|
||||
"""gitimport - parse `git log` output and turn commits into brain leafs.
|
||||
|
||||
Pure, testable functions. Field grammar (see bin/git/import):
|
||||
|
||||
git log --no-merges --name-only \
|
||||
--format='%x1e%H%x1f%an%x1f%ae%x1f%aI%x1f%s'
|
||||
|
||||
0x1e = record separator, 0x1f = field separator.
|
||||
Files: newline-separated lines following each record's subject.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
REC_SEP = "\x1e"
|
||||
FIELD_SEP = "\x1f"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Commit:
|
||||
sha: str
|
||||
author: str
|
||||
email: str
|
||||
date: str
|
||||
subject: str
|
||||
files: list[str] = field(default_factory=list)
|
||||
|
||||
def leaf_text(self, repo: str) -> str:
|
||||
head = f"commit {self.sha[:12]} in {repo} — {self.subject}"
|
||||
body = [head, f"Author: {self.author} <{self.email}>", f"Date: {self.date}"]
|
||||
if self.files:
|
||||
body.append("Changing: " + ", ".join(self.files))
|
||||
return "\n".join(body)
|
||||
|
||||
|
||||
def parse_log(text: str) -> list[Commit]:
|
||||
"""Parse `git log` output into Commit records.
|
||||
|
||||
Records are separated by 0x1e. A record is fields joined by 0x1f,
|
||||
followed by optional newline-separated file paths inside the next
|
||||
segment (git emits blank line + files after each record).
|
||||
"""
|
||||
commits: list[Commit] = []
|
||||
# field records and file lists alternate; simpler: split on REC_SEP,
|
||||
# each chunk = header line, possibly followed by newline + files.
|
||||
for chunk in text.split(REC_SEP):
|
||||
chunk = chunk.strip("\n")
|
||||
if not chunk:
|
||||
continue
|
||||
lines = chunk.split("\n", 1)
|
||||
header = lines[0].split(FIELD_SEP)
|
||||
if len(header) < 5:
|
||||
continue
|
||||
sha, author, email, date, subject = header[:5]
|
||||
files = [ln.strip() for ln in lines[1].splitlines() if ln.strip()] if len(lines) > 1 else []
|
||||
commits.append(Commit(sha=sha, author=author, email=email,
|
||||
date=date, subject=subject, files=files))
|
||||
return commits
|
||||
|
||||
|
||||
def commits_to_leafs(commits: list[Commit], repo: str) -> list[dict]:
|
||||
"""Map commits to the leaf shape bin/kb/index expects (source/repo/...)."""
|
||||
out: list[dict] = []
|
||||
for c in commits:
|
||||
out.append({
|
||||
"source": f"{repo}@{c.sha}",
|
||||
"repo": repo,
|
||||
"heading": f"commit {c.sha[:12]} — {c.subject}",
|
||||
"text": c.leaf_text(repo),
|
||||
"type": "commit",
|
||||
"status": "current",
|
||||
"related": ",".join(c.files),
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
GIT_SCHEMA = (
|
||||
"CREATE NODE TABLE IF NOT EXISTS Commit (id STRING, repo STRING, subject STRING, "
|
||||
"author STRING, email STRING, date STRING, PRIMARY KEY(id))",
|
||||
"CREATE NODE TABLE IF NOT EXISTS Person (id STRING, name STRING, email STRING, PRIMARY KEY(id))",
|
||||
"CREATE REL TABLE IF NOT EXISTS HAS_VERSION (FROM File TO Commit)",
|
||||
"CREATE REL TABLE IF NOT EXISTS AUTHORED (FROM Commit TO Person)",
|
||||
)
|
||||
|
||||
|
||||
def ensure_git_schema(conn) -> None:
|
||||
for stmt in GIT_SCHEMA:
|
||||
conn.execute(stmt)
|
||||
|
||||
|
||||
def index_commits(conn, commits: list[Commit], repo: str) -> int:
|
||||
"""Write Commit/File/Person nodes + edges, one per commit (idempotent by sha)."""
|
||||
ensure_git_schema(conn)
|
||||
for c in commits:
|
||||
conn.execute(
|
||||
"MERGE (c:Commit {id:$sha}) SET c.repo=$repo, c.subject=$subject, "
|
||||
"c.author=$author, c.email=$email, c.date=$date",
|
||||
parameters={"sha": c.sha, "repo": repo, "subject": c.subject,
|
||||
"author": c.author, "email": c.email, "date": c.date},
|
||||
)
|
||||
conn.execute(
|
||||
"MERGE (p:Person {id:$email}) SET p.name=$name, p.email=$email",
|
||||
parameters={"email": c.email, "name": c.author},
|
||||
)
|
||||
conn.execute("MATCH (c:Commit {id:$sha}), (p:Person {id:$email}) "
|
||||
"MERGE (c)-[:AUTHORED]->(p)",
|
||||
parameters={"sha": c.sha, "email": c.email})
|
||||
for path in c.files:
|
||||
conn.execute(
|
||||
"MERGE (f:File {id:$fid}) SET f.path=$path, f.repo=$repo",
|
||||
parameters={"fid": f"{repo}:{path}", "path": path, "repo": repo},
|
||||
)
|
||||
conn.execute("MATCH (f:File {id:$fid}), (c:Commit {id:$sha}) "
|
||||
"MERGE (f)-[:HAS_VERSION]->(c)",
|
||||
parameters={"fid": f"{repo}:{path}", "sha": c.sha})
|
||||
return len(commits)
|
||||
@@ -0,0 +1,195 @@
|
||||
"""kblib - the 2dph brain core over LadybugDB.
|
||||
|
||||
Single embedded graph `var/kb.lbug`. Two roots: facts (assertions backed by
|
||||
>=2 independent sources) and info (narrative leafs). Hybrid retrieval: BM25
|
||||
(FTS extension) + HNSW cosine (VECTOR extension) + Cypher graph hops.
|
||||
|
||||
All access is read-only unless `--rebuild` is passed to kb/index.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
import ladybug
|
||||
|
||||
MODEL = "minishlab/potion-multilingual-128M"
|
||||
EMBED_DIM = 256
|
||||
ROOT_FACTS = "facts"
|
||||
ROOT_INFO = "info"
|
||||
CONF_CONFIRMED = "confirmed"
|
||||
|
||||
def _repo_root() -> Path:
|
||||
p = Path(__file__).resolve().parent
|
||||
while True:
|
||||
if (p / "var").is_dir() or (p / ".git").is_dir() or (p / "pyproject.toml").is_file():
|
||||
return p
|
||||
if p.parent == p:
|
||||
return Path(__file__).resolve().parents[2]
|
||||
p = p.parent
|
||||
|
||||
|
||||
VAR = _repo_root() / "var"
|
||||
DB_PATH = VAR / "kb.lbug"
|
||||
|
||||
|
||||
def sha256_b64(text: str) -> str:
|
||||
return hashlib.sha256(text.encode()).hexdigest()
|
||||
|
||||
|
||||
def _load_extension(conn: ladybug.Connection, name: str) -> None:
|
||||
"""Install (download once) and load a ladybug extension."""
|
||||
try:
|
||||
conn.execute(f"INSTALL {name}")
|
||||
except Exception:
|
||||
pass # already installed / offline-ok when present
|
||||
conn.execute(f"LOAD EXTENSION {name}")
|
||||
|
||||
|
||||
def connect(path: Path | str | None = None, read_only: bool = True) -> tuple[ladybug.Database, ladybug.Connection]:
|
||||
db = ladybug.Database(str(path or DB_PATH), read_only=read_only)
|
||||
conn = ladybug.Connection(db)
|
||||
_load_extension(conn, "FTS")
|
||||
_load_extension(conn, "VECTOR")
|
||||
return db, conn
|
||||
|
||||
|
||||
def init_schema(conn: ladybug.Connection) -> None:
|
||||
conn.execute(
|
||||
"CREATE NODE TABLE IF NOT EXISTS Leaf ("
|
||||
" id STRING, text STRING, root STRING, confidence STRING, "
|
||||
" sha256 STRING, source STRING, source_rev STRING, observed_at STRING, "
|
||||
" how STRING, loc STRING, type STRING, embedding FLOAT[256], "
|
||||
" PRIMARY KEY(id))"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE NODE TABLE IF NOT EXISTS File ("
|
||||
" id STRING, path STRING, repo STRING, mtime STRING, PRIMARY KEY(id))"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE REL TABLE IF NOT EXISTS FROM_FILE (FROM Leaf TO File)"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE NODE TABLE IF NOT EXISTS Host (id STRING, hostname STRING, user STRING, PRIMARY KEY(id))"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE REL TABLE IF NOT EXISTS RUNS_ON (FROM Leaf TO Host)"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE NODE TABLE IF NOT EXISTS Commit (id STRING, repo STRING, subject STRING, "
|
||||
"author STRING, email STRING, date STRING, PRIMARY KEY(id))"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE NODE TABLE IF NOT EXISTS Person (id STRING, name STRING, email STRING, PRIMARY KEY(id))"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE REL TABLE IF NOT EXISTS HAS_VERSION (FROM File TO Commit)"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE REL TABLE IF NOT EXISTS AUTHORED (FROM Commit TO Person)"
|
||||
)
|
||||
|
||||
|
||||
def leaf_id(text: str, source: str) -> str:
|
||||
return sha256_b64(f"{source}\0{text}")[:24]
|
||||
|
||||
|
||||
def upsert_leaf(conn: ladybug.Connection, *, text: str, root: str, confidence: str,
|
||||
source: str, source_rev: str, how: str, loc: str, type_: str,
|
||||
embedding: list[float] | None) -> str:
|
||||
lid = leaf_id(text, source)
|
||||
obs = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
conn.execute(
|
||||
"MERGE (l:Leaf {id:$id}) "
|
||||
"SET l.text=$text, l.root=$root, l.confidence=$confidence, "
|
||||
" l.sha256=$sha, l.source=$source, l.source_rev=$rev, l.observed_at=$obs, "
|
||||
" l.how=$how, l.loc=$location, l.type=$type"
|
||||
+ (", l.embedding=$emb" if embedding else ""),
|
||||
parameters={
|
||||
"id": lid, "text": text, "root": root, "confidence": confidence,
|
||||
"sha": sha256_b64(text), "source": source, "rev": source_rev,
|
||||
"obs": obs, "how": how, "location": loc, "type": type_,
|
||||
"emb": (embedding if embedding else None),
|
||||
},
|
||||
)
|
||||
return lid
|
||||
|
||||
|
||||
def create_fts_and_vector(conn: ladybug.Connection, force: bool = False) -> None:
|
||||
if force:
|
||||
conn.execute("DROP INDEX IF EXISTS Leaf.Leaf_fts")
|
||||
conn.execute("DROP INDEX IF EXISTS Leaf.Leaf_vec")
|
||||
try:
|
||||
conn.execute("CALL CREATE_FTS_INDEX('Leaf', 'id', ['text'])")
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
conn.execute("CALL CREATE_VECTOR_INDEX('Leaf', 'Leaf_vec', 'embedding', metric := 'cosine')")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def drop_indexes(conn: ladybug.Connection) -> None:
|
||||
"""Drop FTS + vector indexes so bulk MERGEs don't corrupt them.
|
||||
|
||||
Ladybug's FTS index goes inconsistent when rows are inserted while the
|
||||
index exists ("document for node offset N is missing during delete").
|
||||
Importers that add many leafs must drop indexes first, write, then
|
||||
recreate via create_fts_and_vector().
|
||||
"""
|
||||
conn.execute("DROP INDEX IF EXISTS Leaf.Leaf_fts")
|
||||
conn.execute("DROP INDEX IF EXISTS Leaf.Leaf_vec")
|
||||
|
||||
|
||||
def query_fts(conn: ladybug.Connection, text: str, limit: int = 10) -> list[dict]:
|
||||
r = conn.execute(
|
||||
"CALL QUERY_FTS_INDEX('Leaf', 'id', $q) "
|
||||
"RETURN node.id, node.text, node.root, score ORDER BY score DESC LIMIT $n",
|
||||
parameters={"q": text, "n": limit},
|
||||
)
|
||||
return [{"id": row[0], "text": row[1], "root": row[2], "score": row[3]} for row in r.get_all()]
|
||||
|
||||
|
||||
def query_vector(conn: ladybug.Connection, embedding: list[float], limit: int = 10) -> list[dict]:
|
||||
r = conn.execute(
|
||||
"CALL QUERY_VECTOR_INDEX('Leaf', 'Leaf_vec', $q, $n) "
|
||||
"RETURN node.id, node.text, node.root, distance ORDER BY distance LIMIT $n",
|
||||
parameters={"q": embedding, "n": limit},
|
||||
)
|
||||
out = []
|
||||
for row in r.get_all():
|
||||
# distance -> similarity reasonable for cosine
|
||||
score = 1.0 - row[3] if row[3] is not None else 0.0
|
||||
out.append({"id": row[0], "text": row[1], "root": row[2], "score": score})
|
||||
return out
|
||||
|
||||
|
||||
def hybrid_search(conn: ladybug.Connection, embedding: list[float], fts_hits: list[dict],
|
||||
limit: int = 10) -> list[dict]:
|
||||
"""Merge FTS + vector by reciprocal rank fusion."""
|
||||
fused: dict[str, dict] = {}
|
||||
for rank, hit in enumerate(fts_hits):
|
||||
fused.setdefault(hit["id"], {**hit, "rrf": 0.0})["rrf"] = 1.0 / (60 + rank + 1)
|
||||
for rank, hit in enumerate(query_vector(conn, embedding, limit * 3)):
|
||||
entry = fused.setdefault(hit["id"], {**hit, "rrf": 0.0})
|
||||
entry["rrf"] += 1.0 / (60 + rank + 1)
|
||||
entry.setdefault("score", hit.get("score", 0.0))
|
||||
ranked = sorted(fused.values(), key=lambda h: h.get("rrf", 0.0), reverse=True)
|
||||
return ranked[:limit]
|
||||
|
||||
|
||||
def stats(conn: ladybug.Connection) -> dict:
|
||||
r = conn.execute("MATCH (l:Leaf) RETURN l.root, count(*)")
|
||||
rows = {row[0]: row[1] for row in r.get_all()}
|
||||
total = conn.execute("MATCH (l:Leaf) RETURN count(*)").get_all()[0][0]
|
||||
return {"total": total, "by_root": rows, "db": str(DB_PATH), "model": MODEL}
|
||||
|
||||
|
||||
def open_readonly() -> tuple[ladybug.Database, ladybug.Connection]:
|
||||
if not DB_PATH.exists():
|
||||
raise FileNotFoundError(f"{DB_PATH} missing - run bin/kb/index first")
|
||||
db, conn = connect(read_only=True)
|
||||
return db, conn
|
||||
@@ -0,0 +1,84 @@
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import mistune
|
||||
|
||||
|
||||
def extract_frontmatter(text: str) -> tuple[dict, str]:
|
||||
"""Return (frontmatter dict, body). Accepts leading --- yaml ---."""
|
||||
if not text.startswith("---"):
|
||||
return {}, text
|
||||
end = text.find("\n---", 3)
|
||||
if end == -1:
|
||||
return {}, text
|
||||
fm = text[3:end].strip()
|
||||
body = text[end + 4 :]
|
||||
meta: dict = {}
|
||||
for line in fm.splitlines():
|
||||
if ":" in line:
|
||||
key, _, value = line.partition(":")
|
||||
meta[key.strip()] = value.strip().strip("\"'")
|
||||
return meta, body
|
||||
|
||||
|
||||
def split_leafs(meta: dict, body: str) -> list[dict]:
|
||||
"""Split a markdown body into leaf chunks on H2 (##) boundaries.
|
||||
|
||||
Each leaf keeps the document-level frontmatter (type, related) and gets
|
||||
its own heading + text. H1 is treated as document title, prepended to the
|
||||
first chunk.
|
||||
"""
|
||||
title = ""
|
||||
lines = body.splitlines()
|
||||
headers: list[tuple[str, int]] = []
|
||||
for i, line in enumerate(lines):
|
||||
if re.match(r"^# \S", line):
|
||||
title = line.lstrip("#").strip()
|
||||
elif re.match(r"^## \S", line):
|
||||
headers.append((line.lstrip("##").strip(), i))
|
||||
if not headers:
|
||||
text = "\n".join(l for l in lines if l.strip())
|
||||
return [{"heading": title, "text": text.strip()}]
|
||||
|
||||
leafs: list[dict] = []
|
||||
for idx, (heading, start) in enumerate(headers):
|
||||
end = headers[idx + 1][1] if idx + 1 < len(headers) else len(lines)
|
||||
chunk = "\n".join(l for l in lines[start:end] if l.strip())
|
||||
text = chunk
|
||||
if idx == 0 and title:
|
||||
text = f"{title}\n\n{chunk}"
|
||||
leafs.append({"heading": heading, "text": text.strip()})
|
||||
return leafs
|
||||
|
||||
|
||||
def to_all(text: str, path: str | Path, repo: str = "") -> list[dict]:
|
||||
meta, body = extract_frontmatter(text)
|
||||
meta.setdefault("type", "reference")
|
||||
meta.setdefault("status", "current")
|
||||
path = str(path)
|
||||
leafs = split_leafs(meta, body)
|
||||
out = []
|
||||
for lf in leafs:
|
||||
out.append({
|
||||
"source": path,
|
||||
"repo": repo,
|
||||
"heading": lf["heading"],
|
||||
"text": lf["text"],
|
||||
"type": meta.get("type", "reference"),
|
||||
"status": meta.get("status", "current"),
|
||||
"related": meta.get("related", ""),
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def read_markdown(path: Path) -> str:
|
||||
return path.read_text(encoding="utf-8", errors="replace")
|
||||
|
||||
|
||||
def walk_markdown(root: Path) -> list[Path]:
|
||||
return sorted(p for p in root.rglob("*") if p.suffix.lower() in (".md", ".markdown"))
|
||||
|
||||
|
||||
def leaves_to_json(leaves: list[dict]) -> str:
|
||||
return json.dumps(leaves, ensure_ascii=False, indent=2)
|
||||
@@ -0,0 +1,33 @@
|
||||
"""semver logic shared by bin/ci/semver and its tests. No git IO here."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
BREAKING_MARKERS = ("BREAKING CHANGE", "breaking-change")
|
||||
BUMP_PATCH_TYPES = ("fix", "perf", "refactor", "build", "ci", "docs", "chore", "test", "style", "revert")
|
||||
BUMP_MINOR_TYPE = "feat"
|
||||
|
||||
|
||||
def bump_type(subjects: list[str]) -> str:
|
||||
if not subjects:
|
||||
return "none"
|
||||
for subject in subjects:
|
||||
text = subject.lower()
|
||||
if any(m.lower() in text for m in BREAKING_MARKERS):
|
||||
return "major"
|
||||
if "!" in subject.split(":")[0]:
|
||||
return "major"
|
||||
for subject in subjects:
|
||||
if subject.startswith(f"{BUMP_MINOR_TYPE}:"):
|
||||
return "minor"
|
||||
return "patch"
|
||||
|
||||
|
||||
def bump_version(current: str | None, bump: str) -> str | None:
|
||||
if bump == "none":
|
||||
return None
|
||||
major, minor, patch = [int(n) for n in (current or "0.0.0").lstrip("v").split(".")]
|
||||
if bump == "major":
|
||||
return f"v{major + 1}.0.0"
|
||||
if bump == "minor":
|
||||
return f"v{major}.{minor + 1}.0"
|
||||
return f"v{major}.{minor}.{patch + 1}"
|
||||
@@ -0,0 +1,46 @@
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
import crmfacts # noqa: E402
|
||||
|
||||
FM = """\
|
||||
schema: 2
|
||||
meta:
|
||||
title: x
|
||||
orgs:
|
||||
- id: produktor
|
||||
label: ProProdukt SL / produktor.io
|
||||
kind: own
|
||||
period: 2006–present
|
||||
website: https://produktor.io
|
||||
- id: dyvenia
|
||||
label: Dyvenia
|
||||
kind: employer
|
||||
period: 2023–2025
|
||||
clients:
|
||||
- name: One
|
||||
- name: Two
|
||||
timeline:
|
||||
- start: 2001
|
||||
"""
|
||||
|
||||
|
||||
class CorpusOrgsTest(unittest.TestCase):
|
||||
def test_parses_label_kind_period(self):
|
||||
orgs = crmfacts.corpus_orgs(FM)
|
||||
self.assertEqual(orgs["produktor"]["label"], "ProProdukt SL / produktor.io")
|
||||
self.assertEqual(orgs["produktor"]["kind"], "own")
|
||||
self.assertEqual(orgs["dyvenia"]["kind"], "employer")
|
||||
|
||||
def test_does_not_leak_clients_into_orgs(self):
|
||||
orgs = crmfacts.corpus_orgs(FM)
|
||||
self.assertNotIn("One", orgs)
|
||||
self.assertNotIn("Two", orgs)
|
||||
self.assertNotIn("timeline", orgs)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,71 @@
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
import kblib # noqa: E402
|
||||
import gitimport # noqa: E402
|
||||
|
||||
SAMPLE = (
|
||||
"\x1e" + "a1b2c3d" + "\x1f" + "Ada Lovelace" + "\x1f" + "ada@example.com"
|
||||
+ "\x1f" + "2026-08-10T12:00:00+01:00" + "\x1f" + "feat: first commit"
|
||||
+ "\n\nREADME.md\nsrc/main.c\n"
|
||||
)
|
||||
|
||||
COMMIT_PERSON_SCHEMA = (
|
||||
"CREATE NODE TABLE IF NOT EXISTS Commit (id STRING, repo STRING, subject STRING, "
|
||||
"author STRING, email STRING, date STRING, PRIMARY KEY(id))"
|
||||
)
|
||||
PERSON_SCHEMA = (
|
||||
"CREATE NODE TABLE IF NOT EXISTS Person (id STRING, name STRING, email STRING, PRIMARY KEY(id))"
|
||||
)
|
||||
HAS_VERSION_SCHEMA = "CREATE REL TABLE IF NOT EXISTS HAS_VERSION (FROM File TO Commit)"
|
||||
AUTHORED_SCHEMA = "CREATE REL TABLE IF NOT EXISTS AUTHORED (FROM Commit TO Person)"
|
||||
|
||||
|
||||
class GitGraphTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.dir = tempfile.mkdtemp()
|
||||
self.dbpath = os.path.join(self.dir, "kb.lbug")
|
||||
self.db, self.conn = kblib.connect(self.dbpath, read_only=False)
|
||||
kblib.init_schema(self.conn)
|
||||
self.conn.execute(COMMIT_PERSON_SCHEMA)
|
||||
self.conn.execute(PERSON_SCHEMA)
|
||||
self.conn.execute(HAS_VERSION_SCHEMA)
|
||||
self.conn.execute(AUTHORED_SCHEMA)
|
||||
|
||||
def tearDown(self):
|
||||
self.conn.close()
|
||||
self.db.close()
|
||||
|
||||
def test_index_commits_creates_nodes_and_edges(self):
|
||||
cs = gitimport.parse_log(SAMPLE)
|
||||
gitimport.index_commits(self.conn, cs, "sample-repo")
|
||||
rp = self.conn.execute("MATCH (p:Person) RETURN p.name, p.email").get_all()
|
||||
self.assertEqual([tuple(r) for r in rp], [("Ada Lovelace", "ada@example.com")])
|
||||
rc = self.conn.execute("MATCH (c:Commit) RETURN c.id, c.repo").get_all()
|
||||
self.assertEqual(len(rc), 1)
|
||||
self.assertEqual(rc[0][1], "sample-repo")
|
||||
# File -[:HAS_VERSION]-> Commit -[:AUTHORED]-> Person
|
||||
rf = self.conn.execute(
|
||||
"MATCH (f:File)-[:HAS_VERSION]->(c:Commit)-[:AUTHORED]->(p:Person) "
|
||||
"RETURN f.path, c.id, p.email").get_all()
|
||||
paths = sorted(r[0] for r in rf)
|
||||
self.assertEqual(paths, ["README.md", "src/main.c"])
|
||||
self.assertTrue(all(r[2] == "ada@example.com" for r in rf))
|
||||
|
||||
def test_index_commits_idempotent(self):
|
||||
cs = gitimport.parse_log(SAMPLE)
|
||||
gitimport.index_commits(self.conn, cs, "sample-repo")
|
||||
gitimport.index_commits(self.conn, cs, "sample-repo")
|
||||
n = self.conn.execute("MATCH (c:Commit) RETURN count(*)").get_all()[0][0]
|
||||
self.assertEqual(n, 1)
|
||||
p = self.conn.execute("MATCH (p:Person) RETURN count(*)").get_all()[0][0]
|
||||
self.assertEqual(p, 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,57 @@
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
import gitimport # noqa: E402
|
||||
|
||||
SAMPLE = (
|
||||
"\x1e" + "a1b2c3d" + "\x1f" + "Ada Lovelace" + "\x1f" + "ada@example.com"
|
||||
+ "\x1f" + "2026-08-10T12:00:00+01:00" + "\x1f" + "feat: first commit"
|
||||
+ "\n\nREADME.md\nsrc/main.c\n"
|
||||
+ "\x1e" + "e4f5a6b" + "\x1f" + "Bob Babbage" + "\x1f" + "bob@example.com"
|
||||
+ "\x1f" + "2026-08-11T09:30:00+01:00" + "\x1f" + "fix: typo"
|
||||
+ "\n\ndocs/notes.md"
|
||||
)
|
||||
|
||||
|
||||
class GitparseTest(unittest.TestCase):
|
||||
def test_parses_records(self):
|
||||
cs = gitimport.parse_log(SAMPLE)
|
||||
self.assertEqual(len(cs), 2)
|
||||
|
||||
def test_parses_commit_fields(self):
|
||||
cs = gitimport.parse_log(SAMPLE)
|
||||
c = cs[0]
|
||||
self.assertEqual(c.sha, "a1b2c3d")
|
||||
self.assertEqual(c.author, "Ada Lovelace")
|
||||
self.assertEqual(c.email, "ada@example.com")
|
||||
self.assertEqual(c.date, "2026-08-10T12:00:00+01:00")
|
||||
self.assertEqual(c.subject, "feat: first commit")
|
||||
|
||||
def test_parses_changed_files(self):
|
||||
cs = gitimport.parse_log(SAMPLE)
|
||||
self.assertEqual(cs[0].files, ["README.md", "src/main.c"])
|
||||
self.assertEqual(cs[1].files, ["docs/notes.md"])
|
||||
|
||||
def test_ignores_empty(self):
|
||||
self.assertEqual(gitimport.parse_log(""), [])
|
||||
|
||||
def test_skip_malformed_record(self):
|
||||
self.assertEqual(gitimport.parse_log("\x1eweird\x1e"), [])
|
||||
|
||||
def test_commit_leaf_shape(self):
|
||||
leafs = gitimport.commits_to_leafs(gitimport.parse_log(SAMPLE), "sample-repo")
|
||||
self.assertEqual(len(leafs), 2)
|
||||
lf = leafs[0]
|
||||
self.assertEqual(lf["type"], "commit")
|
||||
self.assertEqual(lf["repo"], "sample-repo")
|
||||
self.assertEqual(lf["source"], "sample-repo@a1b2c3d")
|
||||
self.assertIn("Ada Lovelace", lf["text"])
|
||||
self.assertIn("README.md", lf["related"])
|
||||
self.assertIn("feat: first commit", lf["heading"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,73 @@
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
import kblib # noqa: E402
|
||||
|
||||
|
||||
def make_emb(value: float) -> list[float]:
|
||||
vec = [0.0] * kblib.EMBED_DIM
|
||||
vec[0] = value
|
||||
return vec
|
||||
|
||||
|
||||
class KblibTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.dir = tempfile.mkdtemp()
|
||||
self.dbpath = os.path.join(self.dir, "kb.lbug")
|
||||
self.db, self.conn = kblib.connect(self.dbpath, read_only=False)
|
||||
kblib.init_schema(self.conn)
|
||||
|
||||
def tearDown(self):
|
||||
self.conn.close()
|
||||
self.db.close()
|
||||
|
||||
def test_leaf_id_is_stable(self):
|
||||
self.assertEqual(kblib.leaf_id("abc", "src"), kblib.leaf_id("abc", "src"))
|
||||
self.assertNotEqual(kblib.leaf_id("abc", "src"), kblib.leaf_id("abd", "src"))
|
||||
|
||||
def test_upsert_roundtrip(self):
|
||||
kblib.upsert_leaf(self.conn, text="the quick brown fox", root="info",
|
||||
confidence="confirmed", source="s", source_rev="r1",
|
||||
how="test", loc="/tmp", type_="reference",
|
||||
embedding=make_emb(1.0))
|
||||
kblib.create_fts_and_vector(self.conn, force=True)
|
||||
hits = kblib.query_fts(self.conn, "fox", 5)
|
||||
self.assertEqual(len(hits), 1)
|
||||
self.assertEqual(hits[0]["root"], "info")
|
||||
|
||||
def test_hybrid_ranks_vector_match(self):
|
||||
kblib.upsert_leaf(self.conn, text="the quick brown fox", root="info",
|
||||
confidence="confirmed", source="s", source_rev="r1",
|
||||
how="test", loc="/tmp", type_="reference",
|
||||
embedding=make_emb(1.0))
|
||||
kblib.upsert_leaf(self.conn, text="a lazy dog sleeps", root="info",
|
||||
confidence="confirmed", source="s", source_rev="r1",
|
||||
how="test", loc="/tmp", type_="reference",
|
||||
embedding=make_emb(0.0))
|
||||
kblib.create_fts_and_vector(self.conn, force=True)
|
||||
result = kblib.hybrid_search(self.conn, make_emb(1.0), [], 5)
|
||||
self.assertTrue(result)
|
||||
self.assertIn("rrf", result[0])
|
||||
self.assertEqual(result[0]["text"], "the quick brown fox")
|
||||
|
||||
def test_stats_counts_roots(self):
|
||||
kblib.upsert_leaf(self.conn, text="a fact leaf", root="facts",
|
||||
confidence="confirmed", source="s", source_rev="r1",
|
||||
how="test", loc="/tmp", type_="reference",
|
||||
embedding=make_emb(0.5))
|
||||
kblib.upsert_leaf(self.conn, text="an info leaf", root="info",
|
||||
confidence="confirmed", source="s", source_rev="r1",
|
||||
how="test", loc="/tmp", type_="reference",
|
||||
embedding=make_emb(0.5))
|
||||
stats = kblib.stats(self.conn)
|
||||
self.assertEqual(stats["total"], 2)
|
||||
self.assertEqual(stats["by_root"], {"facts": 1, "info": 1})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1 @@
|
||||
{"query": "Pflegegrad Test 4", "results": [], "answers": [], "corrections": [], "infoboxes": [], "suggestions": [], "unresponsive_engines": [["brave", "Suspended: too many requests"], ["duckduckgo", "CAPTCHA"], ["startpage", "Suspended: CAPTCHA"]]}
|
||||
@@ -0,0 +1,110 @@
|
||||
"""Tests for the SearXNG client. No network: two recorded responses stand in.
|
||||
|
||||
Run: python3 -m unittest discover -s tools -t .
|
||||
"""
|
||||
import sys
|
||||
import json
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
import websearch as ws
|
||||
|
||||
FIXTURES = Path(__file__).resolve().parent / "fixtures"
|
||||
HEALTHY = json.loads((FIXTURES / "healthy.json").read_text())
|
||||
THROTTLED = json.loads((FIXTURES / "throttled.json").read_text())
|
||||
|
||||
|
||||
class Classify(unittest.TestCase):
|
||||
"""An empty result set is not evidence of absence.
|
||||
|
||||
The instance answers 200 with `results: []` when it throttles us, so calling
|
||||
that "no matches" would make an agent conclude something false.
|
||||
"""
|
||||
|
||||
def test_healthy_response_is_ok(self):
|
||||
self.assertEqual(ws.classify(HEALTHY), "ok")
|
||||
|
||||
def test_empty_response_is_throttled_not_empty(self):
|
||||
self.assertEqual(ws.classify(THROTTLED), "throttled")
|
||||
|
||||
def test_status_is_never_the_word_empty(self):
|
||||
self.assertNotIn(ws.classify(THROTTLED), ("empty", "no_results"))
|
||||
|
||||
|
||||
class Project(unittest.TestCase):
|
||||
def test_keeps_only_the_fields_worth_context(self):
|
||||
out = ws.project(HEALTHY, limit=3)
|
||||
self.assertEqual(out["status"], "ok")
|
||||
self.assertEqual(len(out["results"]), 3)
|
||||
self.assertEqual(set(out["results"][0]), {"rank", "title", "url", "snippet", "engine"})
|
||||
|
||||
def test_snippet_is_trimmed(self):
|
||||
out = ws.project(HEALTHY, limit=5, snippet_chars=40)
|
||||
self.assertTrue(all(len(r["snippet"]) <= 43 for r in out["results"]))
|
||||
|
||||
def test_projection_is_far_cheaper_than_the_raw_payload(self):
|
||||
raw = len(json.dumps(HEALTHY))
|
||||
small = len(json.dumps(ws.project(HEALTHY, limit=5)))
|
||||
self.assertLess(small * 3, raw)
|
||||
|
||||
def test_throttled_projection_carries_the_engine_reasons(self):
|
||||
out = ws.project(THROTTLED, limit=5)
|
||||
self.assertEqual(out["status"], "throttled")
|
||||
self.assertEqual(out["results"], [])
|
||||
self.assertTrue(out["unresponsive"])
|
||||
|
||||
|
||||
class CacheKey(unittest.TestCase):
|
||||
def test_same_question_same_key(self):
|
||||
self.assertEqual(ws.cache_key("Pflegegrad", {}), ws.cache_key("Pflegegrad", {}))
|
||||
|
||||
def test_case_and_padding_do_not_matter(self):
|
||||
self.assertEqual(ws.cache_key(" Pflegegrad ", {}), ws.cache_key("pflegegrad", {}))
|
||||
|
||||
def test_parameters_change_the_key(self):
|
||||
self.assertNotEqual(ws.cache_key("x", {"lang": "de"}), ws.cache_key("x", {}))
|
||||
|
||||
def test_parameter_order_does_not_change_the_key(self):
|
||||
self.assertEqual(ws.cache_key("x", {"a": "1", "b": "2"}),
|
||||
ws.cache_key("x", {"b": "2", "a": "1"}))
|
||||
|
||||
|
||||
class PhiGuard(unittest.TestCase):
|
||||
"""The query leaves this host, so client data must never reach it."""
|
||||
|
||||
def test_plain_technical_query_passes(self):
|
||||
self.assertIsNone(ws.phi_reason("Pflegegrad SGB XI Einstufung"))
|
||||
self.assertIsNone(ws.phi_reason("site:ticket.detective.de Toureffizienz"))
|
||||
|
||||
def test_long_digit_run_is_refused(self):
|
||||
self.assertIsNotNone(ws.phi_reason("Kunde 4711220385 Adresse"))
|
||||
|
||||
def test_insurance_number_is_refused(self):
|
||||
self.assertIsNotNone(ws.phi_reason("KV-Nr A123456789"))
|
||||
|
||||
def test_street_with_house_number_is_refused(self):
|
||||
self.assertIsNotNone(ws.phi_reason("Hauptstraße 14 Berlin"))
|
||||
self.assertIsNotNone(ws.phi_reason("Lindenstr. 7"))
|
||||
|
||||
def test_personalnummer_is_refused(self):
|
||||
self.assertIsNotNone(ws.phi_reason("Personalnummer 12"))
|
||||
|
||||
def test_short_numbers_are_fine(self):
|
||||
self.assertIsNone(ws.phi_reason("SGB XI Paragraph 45b"))
|
||||
|
||||
|
||||
class Throttle(unittest.TestCase):
|
||||
def test_waits_the_remainder_of_the_interval(self):
|
||||
self.assertAlmostEqual(ws.wait_for(last=100.0, now=104.0, interval=10.0), 6.0)
|
||||
|
||||
def test_no_wait_once_the_interval_passed(self):
|
||||
self.assertEqual(ws.wait_for(last=100.0, now=130.0, interval=10.0), 0.0)
|
||||
|
||||
def test_no_wait_on_a_first_call(self):
|
||||
self.assertEqual(ws.wait_for(last=None, now=130.0, interval=10.0), 0.0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,161 @@
|
||||
"""SearXNG client that is safe for agents to share.
|
||||
|
||||
Three things make this more than a curl wrapper:
|
||||
|
||||
* An empty result set from this instance usually means "throttled", not "no
|
||||
matches". Reporting it as absence would make an agent state something false,
|
||||
so `classify` never returns a word that sounds like a negative finding.
|
||||
* Queries leave the host, so `phi_reason` refuses anything that smells like
|
||||
client data before it reaches an external engine.
|
||||
* Results are cached and calls are serialised, because the instance suspends
|
||||
engines under load.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
SNIPPET_CHARS = 150
|
||||
DEFAULT_LIMIT = 5
|
||||
MIN_INTERVAL = 10.0
|
||||
CACHE_TTL = 7 * 24 * 3600
|
||||
RETRY_BACKOFF = (20.0, 60.0)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# response handling
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def classify(payload: dict) -> str:
|
||||
"""`ok` when at least one engine answered, `throttled` otherwise."""
|
||||
return "ok" if payload.get("results") else "throttled"
|
||||
|
||||
|
||||
def project(payload: dict, limit: int = DEFAULT_LIMIT,
|
||||
snippet_chars: int = SNIPPET_CHARS) -> dict:
|
||||
"""Keep the few fields worth spending context on."""
|
||||
status = classify(payload)
|
||||
results = []
|
||||
for rank, item in enumerate(payload.get("results", [])[:limit], start=1):
|
||||
snippet = re.sub(r"\s+", " ", item.get("content") or "").strip()
|
||||
if len(snippet) > snippet_chars:
|
||||
snippet = snippet[:snippet_chars].rstrip() + "..."
|
||||
results.append({
|
||||
"rank": rank,
|
||||
"title": item.get("title", ""),
|
||||
"url": item.get("url", ""),
|
||||
"snippet": snippet,
|
||||
"engine": item.get("engine", ""),
|
||||
})
|
||||
out = {
|
||||
"query": payload.get("query", ""),
|
||||
"status": status,
|
||||
"results": results,
|
||||
}
|
||||
unresponsive = [f"{name}: {reason}" for name, reason in
|
||||
payload.get("unresponsive_engines", [])]
|
||||
if unresponsive:
|
||||
out["unresponsive"] = unresponsive
|
||||
if status == "throttled":
|
||||
out["note"] = ("no engine answered - this is a throttled instance, "
|
||||
"not evidence that nothing exists")
|
||||
return out
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# cache key and throttling
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def cache_key(query: str, params: dict) -> str:
|
||||
norm = " ".join(query.lower().split())
|
||||
stable = json.dumps(params, sort_keys=True, ensure_ascii=False)
|
||||
return hashlib.sha256(f"{norm}\x00{stable}".encode()).hexdigest()
|
||||
|
||||
|
||||
def wait_for(last: float | None, now: float, interval: float = MIN_INTERVAL) -> float:
|
||||
"""Seconds to sleep so that calls stay `interval` apart."""
|
||||
if last is None:
|
||||
return 0.0
|
||||
return max(0.0, interval - (now - last))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# PII guard
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
PII_PATTERNS = [
|
||||
(re.compile(r"\d{6,}"), "a run of six or more digits looks like an ID"),
|
||||
(re.compile(r"\bpersonalnummer\b", re.I), "Personalnummer is staff data"),
|
||||
(re.compile(r"\bkv[-\s]?nr\b", re.I), "KV-Nr is an insurance number"),
|
||||
(re.compile(r"\bversichertennummer\b", re.I), "insurance number"),
|
||||
(re.compile(r"\b[A-Za-zÄÖÜäöüß]+(?:stra(?:ss|ß)e|str\.)\s*\d+", re.I),
|
||||
"a street with a house number looks like an address"),
|
||||
(re.compile(r"\bgeb(?:urtsdatum)?\.?\s*\d{1,2}[./]\d{1,2}[./]\d{2,4}", re.I),
|
||||
"a date of birth"),
|
||||
]
|
||||
|
||||
|
||||
def phi_reason(query: str) -> str | None:
|
||||
"""Why this query must not be sent, or None when it is safe."""
|
||||
for pattern, reason in PII_PATTERNS:
|
||||
if pattern.search(query):
|
||||
return reason
|
||||
return None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# cache storage
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
CACHE_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS responses (
|
||||
key TEXT PRIMARY KEY,
|
||||
fetched REAL NOT NULL,
|
||||
payload TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value REAL NOT NULL
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
def open_cache(path: Path) -> sqlite3.Connection:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(path, timeout=30)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.executescript(CACHE_SCHEMA)
|
||||
return conn
|
||||
|
||||
|
||||
def cache_get(conn: sqlite3.Connection, key: str, ttl: float = CACHE_TTL,
|
||||
now: float | None = None) -> dict | None:
|
||||
now = time.time() if now is None else now
|
||||
row = conn.execute("SELECT fetched, payload FROM responses WHERE key = ?",
|
||||
(key,)).fetchone()
|
||||
if row is None or now - row["fetched"] > ttl:
|
||||
return None
|
||||
return json.loads(row["payload"])
|
||||
|
||||
|
||||
def cache_put(conn: sqlite3.Connection, key: str, payload: dict,
|
||||
now: float | None = None) -> None:
|
||||
now = time.time() if now is None else now
|
||||
conn.execute("INSERT OR REPLACE INTO responses (key, fetched, payload) VALUES (?, ?, ?)",
|
||||
(key, now, json.dumps(payload, ensure_ascii=False)))
|
||||
conn.commit()
|
||||
|
||||
|
||||
def last_call(conn: sqlite3.Connection) -> float | None:
|
||||
row = conn.execute("SELECT value FROM meta WHERE key = 'last_call'").fetchone()
|
||||
return row["value"] if row else None
|
||||
|
||||
|
||||
def mark_call(conn: sqlite3.Connection, now: float | None = None) -> None:
|
||||
now = time.time() if now is None else now
|
||||
conn.execute("INSERT OR REPLACE INTO meta (key, value) VALUES ('last_call', ?)", (now,))
|
||||
conn.commit()
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Minimal YAML emitter.
|
||||
|
||||
Agents read YAML more cheaply than JSON and the output stays diffable. This is
|
||||
deliberately tiny: it emits the shapes these tools produce, nothing more.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
|
||||
def to_yaml(node, indent: int = 0) -> str:
|
||||
pad = " " * indent
|
||||
if isinstance(node, dict):
|
||||
if not node:
|
||||
return f"{pad}{{}}\n"
|
||||
out = ""
|
||||
for key, value in node.items():
|
||||
if isinstance(value, (dict, list)) and value:
|
||||
out += f"{pad}{key}:\n{to_yaml(value, indent + 1)}"
|
||||
elif isinstance(value, (dict, list)):
|
||||
out += f"{pad}{key}: {'{}' if isinstance(value, dict) else '[]'}\n"
|
||||
else:
|
||||
out += f"{pad}{key}: {scalar(value)}\n"
|
||||
return out
|
||||
if isinstance(node, list):
|
||||
out = ""
|
||||
for item in node:
|
||||
if isinstance(item, dict):
|
||||
out += f"{pad}-\n{to_yaml(item, indent + 1)}"
|
||||
else:
|
||||
out += f"{pad}- {scalar(item)}\n"
|
||||
return out
|
||||
return f"{pad}{scalar(node)}\n"
|
||||
|
||||
|
||||
def scalar(value) -> str:
|
||||
if value is None:
|
||||
return "null"
|
||||
if isinstance(value, bool):
|
||||
return "true" if value else "false"
|
||||
if isinstance(value, (int, float)):
|
||||
return str(value)
|
||||
text = str(value)
|
||||
if "\n" in text:
|
||||
return json.dumps(text, ensure_ascii=False)
|
||||
if text == "" or any(ch in text for ch in ":#'\"[]{}&*!|>%@`") or text != text.strip():
|
||||
return json.dumps(text, ensure_ascii=False)
|
||||
return text
|
||||
@@ -0,0 +1,105 @@
|
||||
// Package watch polls corpus directories for changes and re-runs bin/kb/index.
|
||||
//
|
||||
// Port of the former bin/kb-watch bash script to an importable, testable Go
|
||||
// package. Polls file mtimes (no inotify deps); cheap and reliable.
|
||||
package watch
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Options controls the polling loop. Zero value uses defaults.
|
||||
type Options struct {
|
||||
Dirs []string
|
||||
Interval time.Duration
|
||||
// IndexCmd is the kb/index command template. %s is replaced by the repo
|
||||
// root (from KB_ROOT). Defaults to `python3 <root>/bin/kb/index`.
|
||||
IndexCmd string
|
||||
}
|
||||
|
||||
// Run blocks forever polling Dirs (defaults: KB_WATCH_DIRS or /corpus) every
|
||||
// Interval (default 30s) and re-indexing when files change. KB_ROOT names the
|
||||
// repo root used to locate bin/kb/index.
|
||||
func Run(args []string) {
|
||||
opts := fromEnv(args)
|
||||
root, _ := os.Getwd()
|
||||
if r := os.Getenv("KB_ROOT"); r != "" {
|
||||
root = r
|
||||
}
|
||||
log.Printf("watch: dirs=%v interval=%s root=%s", opts.Dirs, opts.Interval, root)
|
||||
var last string
|
||||
for {
|
||||
if flag := Stamp(opts.Dirs); flag != "" && flag != last {
|
||||
last = flag
|
||||
reindex(opts.IndexCmd, root)
|
||||
}
|
||||
time.Sleep(opts.Interval)
|
||||
}
|
||||
}
|
||||
|
||||
func fromEnv(args []string) Options {
|
||||
opts := Options{Interval: 30 * time.Second}
|
||||
if raw := os.Getenv("KB_WATCH_INTERVAL"); raw != "" {
|
||||
if n, err := strconv.Atoi(raw); err == nil && n > 0 {
|
||||
opts.Interval = time.Duration(n) * time.Second
|
||||
}
|
||||
}
|
||||
defDirs := "/corpus"
|
||||
if raw := os.Getenv("KB_WATCH_DIRS"); raw != "" {
|
||||
defDirs = raw
|
||||
}
|
||||
if len(args) > 0 {
|
||||
opts.Dirs = args
|
||||
} else {
|
||||
for _, d := range strings.Split(defDirs, " ") {
|
||||
if d != "" {
|
||||
opts.Dirs = append(opts.Dirs, d)
|
||||
}
|
||||
}
|
||||
}
|
||||
pys := os.Getenv("KB_PY")
|
||||
if pys == "" {
|
||||
pys = "python3"
|
||||
}
|
||||
opts.IndexCmd = pys + " <root>/bin/kb/index"
|
||||
return opts
|
||||
}
|
||||
|
||||
// Stamp returns a rolling fingerprint (newest mtime under dirs) that changes
|
||||
// whenever any corpus file is touched. Empty when no files found.
|
||||
func Stamp(dirs []string) string {
|
||||
var newest time.Time
|
||||
for _, dir := range dirs {
|
||||
_ = filepath.WalkDir(dir, func(path string, _ os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if info, e := os.Stat(path); e == nil && info.ModTime().After(newest) {
|
||||
newest = info.ModTime()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if newest.IsZero() {
|
||||
return ""
|
||||
}
|
||||
return strconv.FormatInt(newest.UnixNano(), 10)
|
||||
}
|
||||
|
||||
func reindex(template, root string) {
|
||||
cmd := strings.ReplaceAll(template, "<root>", root)
|
||||
parts := strings.Fields(cmd)
|
||||
c := exec.Command(parts[0], parts[1:]...)
|
||||
out, err := c.CombinedOutput()
|
||||
if err != nil {
|
||||
log.Printf("watch: index failed: %v\n%s", err, out)
|
||||
} else {
|
||||
log.Printf("watch: re-indexed")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package watch
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestStampChangesWhenFileTouched(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
a := filepath.Join(dir, "a.md")
|
||||
if err := os.WriteFile(a, []byte("x"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s1 := Stamp([]string{dir})
|
||||
if s1 == "" {
|
||||
t.Fatal("stamp empty for a dir with a file")
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
if err := os.WriteFile(a, []byte("y"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s2 := Stamp([]string{dir}); s2 == s1 {
|
||||
t.Fatal("stamp did not change after the file was modified")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStampEmptyForMissingDir(t *testing.T) {
|
||||
if s := Stamp([]string{filepath.Join(t.TempDir(), "nope")}); s != "" {
|
||||
t.Fatalf("stamp = %q, want empty for missing dir", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFromEnvDefaults(t *testing.T) {
|
||||
t.Setenv("KB_WATCH_INTERVAL", "")
|
||||
t.Setenv("KB_WATCH_DIRS", "")
|
||||
t.Setenv("KB_PY", "")
|
||||
opts := fromEnv(nil)
|
||||
if len(opts.Dirs) == 0 || opts.Dirs[0] != "/corpus" {
|
||||
t.Fatalf("default dirs = %v, want [/corpus]", opts.Dirs)
|
||||
}
|
||||
if opts.Interval != 30*time.Second {
|
||||
t.Fatalf("default interval = %s, want 30s", opts.Interval)
|
||||
}
|
||||
if opts.IndexCmd == "" {
|
||||
t.Fatal("default index cmd is empty")
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -25,7 +25,7 @@ import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
TOOLS = Path(__file__).resolve().parents[1].parent / "tools"
|
||||
TOOLS = Path(__file__).resolve().parents[1] / "tools"
|
||||
sys.path.insert(0, str(TOOLS))
|
||||
sys.path.insert(0, str(TOOLS / "web-search"))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user