feat(kb): brain tools, Go async serve, root-level docker
- tools/kblib.py: ladybug schema, embeddings, FTS+vector, hybrid RRF
- bin/kb/{index,search,get,stats,eval}: corpus indexing + deduction search
- bin/facts/{extract,audit}: 2-source evidence acquisition + gates
- serve/: async Go HTTP server (goroutines, bounded worker pool), TDD
- docker/ flattened to root: compose.yaml + Dockerfile (multi-stage Go)
- docker scripts -> bin/ shebang pattern (kb-watch, docker-entrypoint)
- bin/ci/semver + tools/semver.py: conventional-commit semver release
- ci.yml: go tests + shell checks; drop release-please (PR toggle blocked)
This commit is contained in:
Executable
+81
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env python3
|
||||
"""facts/audit - evidence & lexicon checks for the 2dph brain.
|
||||
|
||||
bin/facts/audit self # lexicon: every fact in db has >=2 sources
|
||||
bin/facts/audit db # evidence gate: run against var/kb.lbug
|
||||
|
||||
`self` mode checks the repo itself (no network, no runtime deps). It greps
|
||||
for known-good two-source pairings and confirms the docs are consistent.
|
||||
`db` mode loads every Leaf with root=facts and asserts each has source_rev
|
||||
and a non-empty `loc` (the "where did you see it" evidence pointer) and that
|
||||
'confirmed' facts carry a two-source `source` field.
|
||||
|
||||
Exit 0 = all checks pass, 1 = audit failures, 2 = could not evaluate.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "tools"))
|
||||
|
||||
|
||||
def audit_db() -> list[str]:
|
||||
from kblib import connect, init_schema
|
||||
from kblib import VAR
|
||||
dbpath = VAR / "kb.lbug"
|
||||
if not dbpath.exists():
|
||||
return ["no database yet; run bin/kb/index first"]
|
||||
db, conn = connect(dbpath)
|
||||
init_schema(conn)
|
||||
r = conn.execute("MATCH (l:Leaf {root:'facts'}) RETURN l.id, l.source, l.loc, l.how, l.confidence")
|
||||
problems: list[str] = []
|
||||
for lid, source, loc, how, conf in r.get_all():
|
||||
if conf != "confirmed":
|
||||
problems.append(f"{lid}: facts require confidence='confirmed', got '{conf}'")
|
||||
if not source or " x " not in source:
|
||||
problems.append(f"{lid}: needs 2-source evidence in source, got '{source}'")
|
||||
if not loc:
|
||||
problems.append(f"{lid}: missing loc (evidence pointer)")
|
||||
if not how:
|
||||
problems.append(f"{lid}: missing how")
|
||||
conn.close()
|
||||
db.close()
|
||||
return problems
|
||||
|
||||
|
||||
def audit_self() -> list[str]:
|
||||
problems: list[str] = []
|
||||
plan = (ROOT / "PLAN.md").read_text()
|
||||
|
||||
if "recall@5" not in plan:
|
||||
problems.append("PLAN.md missing recall@5 gate")
|
||||
if re.search(r"(?i)facts must have.*2 sources|2.source", plan) is None:
|
||||
problems.append("PLAN.md missing the two-source evidence rule for facts")
|
||||
if re.search(r"(?i)HNSW|BM25|deduction", (ROOT / "README.md").read_text()) is None:
|
||||
problems.append("README.md missing search/retrieval description")
|
||||
return problems
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
import argparse
|
||||
p = argparse.ArgumentParser(description="evidence & lexicon audit")
|
||||
p.add_argument("mode", choices=("self", "db"))
|
||||
p.add_argument("--json", action="store_true")
|
||||
a = p.parse_args(argv)
|
||||
|
||||
problems = audit_self() if a.mode == "self" else audit_db()
|
||||
out = {"mode": a.mode, "ok": not problems, "problems": problems}
|
||||
if a.json:
|
||||
print(json.dumps(out, indent=2))
|
||||
else:
|
||||
from yamlout import to_yaml
|
||||
print(to_yaml(out))
|
||||
return 0 if not problems else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
Executable
+224
@@ -0,0 +1,224 @@
|
||||
#!/usr/bin/env python3
|
||||
"""facts/extract - acquire confirmed facts from the ops stack (2-source each).
|
||||
|
||||
bin/facts/extract [--json] [--dry-run] [--ssh PATH] [--compose PATH]
|
||||
|
||||
The deduction rule: a fact is only stored under root=facts if it is backed by
|
||||
>=2 independent sources. Sources here:
|
||||
|
||||
S1 runtime : docker ps (running containers) or ~/.ssh/config (hosts)
|
||||
S2 declared : docker-compose files or PLAN.md/README.md mentions
|
||||
|
||||
Extracted facts are written into var/kb.lbug (root=facts, confidence=confirmed).
|
||||
--dry-run prints the proposed facts without touching the database.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "tools"))
|
||||
|
||||
COMPOSE_FILES = [ROOT / "docker" / "compose.yaml", ROOT / "compose.yaml"]
|
||||
DOC_MARKERS = ["README.md", "PLAN.md", "AGENTS.md"]
|
||||
SSH_CONFIG = Path.home() / ".ssh" / "config"
|
||||
REPO = "eSlider/2dph"
|
||||
|
||||
|
||||
def read_docker_ps() -> list[str]:
|
||||
try:
|
||||
out = subprocess.run(
|
||||
["docker", "ps", "--format", "{{.Names}}"],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
if out.returncode != 0:
|
||||
return []
|
||||
return [n.strip() for n in out.stdout.splitlines() if n.strip()]
|
||||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||||
return []
|
||||
|
||||
|
||||
def compose_files_in(dirname: Path) -> list[Path]:
|
||||
out = []
|
||||
for name in ("compose.yaml", "compose.yml", "docker-compose.yml", "docker-compose.yaml"):
|
||||
p = dirname / name
|
||||
if p.exists():
|
||||
out.append(p)
|
||||
return out
|
||||
|
||||
|
||||
def container_compose_dir(name: str) -> Path | None:
|
||||
"""Find the compose project dir a running container belongs to."""
|
||||
try:
|
||||
out = subprocess.run(
|
||||
["docker", "inspect", name,
|
||||
"--format", "{{ index .Config.Labels \"com.docker.compose.project.working_dir\"}}"],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
if out.returncode != 0:
|
||||
return None
|
||||
path = out.stdout.strip()
|
||||
return Path(path) if path and path != "<no value>" else None
|
||||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||||
return None
|
||||
|
||||
|
||||
def read_compose_services(compose: Path) -> list[str]:
|
||||
if not compose.exists():
|
||||
return []
|
||||
try:
|
||||
import yaml
|
||||
return list(yaml.safe_load(compose.read_text()).get("services", {}).keys())
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def read_ssh_hosts(path: Path) -> list[str]:
|
||||
if not path.exists():
|
||||
return []
|
||||
hosts = []
|
||||
for line in path.read_text().splitlines():
|
||||
m = re.match(r"^\s*Host\s+(.+)$", line)
|
||||
if m:
|
||||
hosts.extend(h for h in m.group(1).split() if h not in ("*",))
|
||||
return hosts
|
||||
|
||||
|
||||
def mentions(term: str, files: list[Path]) -> bool:
|
||||
for path in files:
|
||||
if not path.exists():
|
||||
continue
|
||||
try:
|
||||
if re.search(rf"\b{re.escape(term)}\b", path.read_text(), re.I):
|
||||
return True
|
||||
except OSError:
|
||||
continue
|
||||
return False
|
||||
|
||||
|
||||
def build_facts() -> list[dict]:
|
||||
facts: list[dict] = []
|
||||
doc_files = [ROOT / m for m in DOC_MARKERS]
|
||||
|
||||
running = read_docker_ps()
|
||||
# Pair each running container against its own compose file (2 independent
|
||||
# sources: runtime state docker ps × declared state compose).
|
||||
runtime_facts = 0
|
||||
for name in running:
|
||||
cdir = container_compose_dir(name)
|
||||
for cfile in compose_files_in(cdir) if cdir else []:
|
||||
if name in read_compose_services(cfile):
|
||||
facts.append({
|
||||
"text": f"container '{name}' is running and declared in {cfile.name}",
|
||||
"source": f"docker ps x compose:{cfile.name}",
|
||||
"loc": f"{cfile}:{name}",
|
||||
"how": "facts/extract",
|
||||
})
|
||||
runtime_facts += 1
|
||||
break
|
||||
if runtime_facts:
|
||||
print(f"facts/extract: paired {runtime_facts}/{len(running)} running containers to compose", file=sys.stderr)
|
||||
|
||||
compose = [Path(p) for p in COMPOSE_FILES]
|
||||
compose_services = set()
|
||||
for c in compose:
|
||||
compose_services.update(read_compose_services(c))
|
||||
if compose_services and running:
|
||||
overlap = sorted(compose_services & set(running))
|
||||
for name in overlap:
|
||||
facts.append({
|
||||
"text": f"container '{name}' is running and declared in compose",
|
||||
"source": f"docker ps x {compose[0].name}",
|
||||
"loc": "docker ps; docker compose config",
|
||||
"how": "facts/extract",
|
||||
})
|
||||
|
||||
hosts = read_ssh_hosts(SSH_CONFIG)
|
||||
for host in hosts:
|
||||
if mentions(host, doc_files):
|
||||
facts.append({
|
||||
"text": f"host '{host}' is configured in ~/.ssh/config and referenced in this repo",
|
||||
"source": f"ssh config x docs({', '.join(DOC_MARKERS)})",
|
||||
"loc": f"~/.ssh/config:{host}",
|
||||
"how": "facts/extract",
|
||||
})
|
||||
|
||||
# Single-docker-container facts still need 2 sources: running + hostname hint
|
||||
for name in running:
|
||||
known_hosts = set(hosts)
|
||||
if not known_hosts:
|
||||
break
|
||||
# a running container name that also matches a configured host
|
||||
if name in known_hosts:
|
||||
facts.append({
|
||||
"text": f"container '{name}' is running and matches configured host '{name}'",
|
||||
"source": "docker ps x ssh config",
|
||||
"loc": f"docker ps:{name}; ~/.ssh/config:{name}",
|
||||
"how": "facts/extract",
|
||||
})
|
||||
return facts
|
||||
|
||||
|
||||
def dedupe(facts: list[dict]) -> list[dict]:
|
||||
seen = set()
|
||||
out = []
|
||||
for f in facts:
|
||||
key = f["text"]
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append(f)
|
||||
return out
|
||||
|
||||
|
||||
def write_facts(facts: list[dict]) -> None:
|
||||
from kblib import connect, init_schema, upsert_leaf
|
||||
from kblib import VAR
|
||||
VAR.mkdir(exist_ok=True)
|
||||
db, conn = connect(VAR / "kb.lbug", read_only=False)
|
||||
init_schema(conn)
|
||||
from model2vec import StaticModel
|
||||
model = StaticModel.from_pretrained("minishlab/potion-multilingual-128M")
|
||||
for f in facts:
|
||||
emb = model.encode([f["text"]])[0].astype(float).tolist()
|
||||
upsert_leaf(conn, text=f["text"], root="facts", confidence="confirmed",
|
||||
source=f["source"], source_rev=REPO, how=f["how"],
|
||||
loc=f["loc"], type_="fact", embedding=emb)
|
||||
conn.close()
|
||||
db.close()
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
global COMPOSE_FILES, SSH_CONFIG
|
||||
import argparse
|
||||
p = argparse.ArgumentParser(description="acquire confirmed facts from ops sources")
|
||||
p.add_argument("--json", action="store_true")
|
||||
p.add_argument("--dry-run", action="store_true")
|
||||
p.add_argument("--ssh", default=str(SSH_CONFIG))
|
||||
p.add_argument("--compose", action="append")
|
||||
a = p.parse_args(argv)
|
||||
|
||||
if a.compose:
|
||||
COMPOSE_FILES = [Path(c) for c in a.compose]
|
||||
if a.ssh:
|
||||
SSH_CONFIG = Path(a.ssh)
|
||||
|
||||
facts = dedupe(build_facts())
|
||||
if not a.dry_run and facts:
|
||||
write_facts(facts)
|
||||
|
||||
out = {"count": len(facts), "facts": facts}
|
||||
if a.json:
|
||||
print(json.dumps(out, indent=2, ensure_ascii=False))
|
||||
else:
|
||||
from yamlout import to_yaml
|
||||
print(to_yaml(out))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
Reference in New Issue
Block a user