feat: D24 fact intervals (--as-of) and bin/stack assistant helpers.
Store valid_from/valid_to on leafs and filter search by calendar day without overloading D16 source staleness; stack start/start-assistant wires brain + PicoClaw.
This commit is contained in:
+76
-14
@@ -62,7 +62,9 @@ def init_schema(conn: ladybug.Connection) -> None:
|
||||
"CREATE NODE TABLE IF NOT EXISTS Leaf ("
|
||||
" id STRING, text STRING, root STRING, confidence STRING, "
|
||||
" sha256 STRING, source STRING, source_rev STRING, observed_at STRING, "
|
||||
" how STRING, loc STRING, type STRING, embedding FLOAT[256], "
|
||||
" how STRING, loc STRING, type STRING, "
|
||||
" valid_from STRING, valid_to STRING, "
|
||||
" embedding FLOAT[256], "
|
||||
" PRIMARY KEY(id))"
|
||||
)
|
||||
conn.execute(
|
||||
@@ -91,6 +93,46 @@ def init_schema(conn: ladybug.Connection) -> None:
|
||||
conn.execute(
|
||||
"CREATE REL TABLE IF NOT EXISTS AUTHORED (FROM Commit TO Person)"
|
||||
)
|
||||
ensure_interval_columns(conn)
|
||||
|
||||
|
||||
def ensure_interval_columns(conn: ladybug.Connection) -> None:
|
||||
"""D24: add valid_from/valid_to on older Leaf tables (idempotent ALTER)."""
|
||||
for col in ("valid_from", "valid_to"):
|
||||
try:
|
||||
conn.execute(f"ALTER TABLE Leaf ADD {col} STRING")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def normalize_day(s: str) -> str:
|
||||
s = (s or "").strip()
|
||||
if len(s) >= 10 and s[4] == "-" and s[7] == "-":
|
||||
return s[:10]
|
||||
return s
|
||||
|
||||
|
||||
def active_at(valid_from: str, valid_to: str, as_of: str) -> bool:
|
||||
"""D24: fact interval of truth. Empty ends = always; empty as_of = no filter."""
|
||||
as_of = normalize_day(as_of)
|
||||
if not as_of:
|
||||
return True
|
||||
fro = normalize_day(valid_from)
|
||||
to = normalize_day(valid_to)
|
||||
if fro and as_of < fro:
|
||||
return False
|
||||
if to and as_of > to:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def filter_as_of(hits: list[dict], as_of: str) -> list[dict]:
|
||||
if not as_of:
|
||||
return hits
|
||||
return [
|
||||
h for h in hits
|
||||
if active_at(str(h.get("valid_from") or ""), str(h.get("valid_to") or ""), as_of)
|
||||
]
|
||||
|
||||
|
||||
def leaf_id(text: str, source: str) -> str:
|
||||
@@ -99,19 +141,24 @@ def leaf_id(text: str, source: str) -> str:
|
||||
|
||||
def upsert_leaf(conn: ladybug.Connection, *, text: str, root: str, confidence: str,
|
||||
source: str, source_rev: str, how: str, loc: str, type_: str,
|
||||
embedding: list[float] | None) -> str:
|
||||
embedding: list[float] | None,
|
||||
valid_from: str = "", valid_to: str = "") -> str:
|
||||
lid = leaf_id(text, source)
|
||||
obs = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
vf = normalize_day(valid_from)
|
||||
vt = normalize_day(valid_to)
|
||||
conn.execute(
|
||||
"MERGE (l:Leaf {id:$id}) "
|
||||
"SET l.text=$text, l.root=$root, l.confidence=$confidence, "
|
||||
" l.sha256=$sha, l.source=$source, l.source_rev=$rev, l.observed_at=$obs, "
|
||||
" l.how=$how, l.loc=$location, l.type=$type"
|
||||
" l.how=$how, l.loc=$location, l.type=$type, "
|
||||
" l.valid_from=$vf, l.valid_to=$vt"
|
||||
+ (", l.embedding=$emb" if embedding else ""),
|
||||
parameters={
|
||||
"id": lid, "text": text, "root": root, "confidence": confidence,
|
||||
"sha": sha256_b64(text), "source": source, "rev": source_rev,
|
||||
"obs": obs, "how": how, "location": loc, "type": type_,
|
||||
"vf": vf, "vt": vt,
|
||||
"emb": (embedding if embedding else None),
|
||||
},
|
||||
)
|
||||
@@ -121,9 +168,10 @@ def upsert_leaf(conn: ladybug.Connection, *, text: str, root: str, confidence: s
|
||||
def add_leafs(conn: ladybug.Connection, leafs: list[dict]) -> list[str]:
|
||||
"""Write facts+info leafs in one transaction. Safe while FTS/HNSW exist.
|
||||
|
||||
Each leaf dict: text, source, optional root/confidence/source_rev/how/loc/type/embedding.
|
||||
Does not delete the database file. Measured on Ladybug 0.19: MERGE of new
|
||||
ids (and updates) stays FTS+HNSW queryable; DROP INDEX is the fatal path.
|
||||
Each leaf dict: text, source, optional root/confidence/source_rev/how/loc/type/
|
||||
embedding/valid_from/valid_to. Does not delete the database file. Measured on
|
||||
Ladybug 0.19: MERGE of new ids (and updates) stays FTS+HNSW queryable; DROP
|
||||
INDEX is the fatal path.
|
||||
"""
|
||||
if not leafs:
|
||||
return []
|
||||
@@ -148,6 +196,8 @@ def add_leafs(conn: ladybug.Connection, leafs: list[dict]) -> list[str]:
|
||||
loc=str(lf.get("loc") or lf.get("source") or ""),
|
||||
type_=str(lf.get("type") or lf.get("type_") or "reference"),
|
||||
embedding=lf.get("embedding"),
|
||||
valid_from=str(lf.get("valid_from") or ""),
|
||||
valid_to=str(lf.get("valid_to") or ""),
|
||||
)
|
||||
)
|
||||
if started:
|
||||
@@ -285,29 +335,39 @@ def drop_indexes(conn: ladybug.Connection) -> None:
|
||||
def query_fts(conn: ladybug.Connection, text: str, limit: int = 10) -> list[dict]:
|
||||
r = conn.execute(
|
||||
"CALL QUERY_FTS_INDEX('Leaf', 'id', $q) "
|
||||
"RETURN node.id, node.text, node.root, score ORDER BY score DESC LIMIT $n",
|
||||
"RETURN node.id, node.text, node.root, score, node.valid_from, node.valid_to "
|
||||
"ORDER BY score DESC LIMIT $n",
|
||||
parameters={"q": text, "n": limit},
|
||||
)
|
||||
return [{"id": row[0], "text": row[1], "root": row[2], "score": row[3]} for row in r.get_all()]
|
||||
return [
|
||||
{
|
||||
"id": row[0], "text": row[1], "root": row[2], "score": row[3],
|
||||
"valid_from": row[4] or "", "valid_to": row[5] or "",
|
||||
}
|
||||
for row in r.get_all()
|
||||
]
|
||||
|
||||
|
||||
def query_vector(conn: ladybug.Connection, embedding: list[float], limit: int = 10) -> list[dict]:
|
||||
r = conn.execute(
|
||||
"CALL QUERY_VECTOR_INDEX('Leaf', 'Leaf_vec', $q, $n) "
|
||||
"RETURN node.id, node.text, node.root, distance ORDER BY distance LIMIT $n",
|
||||
"RETURN node.id, node.text, node.root, distance, node.valid_from, node.valid_to "
|
||||
"ORDER BY distance LIMIT $n",
|
||||
parameters={"q": embedding, "n": limit},
|
||||
)
|
||||
out = []
|
||||
for row in r.get_all():
|
||||
# distance -> similarity reasonable for cosine
|
||||
score = 1.0 - row[3] if row[3] is not None else 0.0
|
||||
out.append({"id": row[0], "text": row[1], "root": row[2], "score": score})
|
||||
out.append({
|
||||
"id": row[0], "text": row[1], "root": row[2], "score": score,
|
||||
"valid_from": row[4] or "", "valid_to": row[5] or "",
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def hybrid_search(conn: ladybug.Connection, embedding: list[float], fts_hits: list[dict],
|
||||
limit: int = 10) -> list[dict]:
|
||||
"""Merge FTS + vector by reciprocal rank fusion."""
|
||||
limit: int = 10, as_of: str = "") -> list[dict]:
|
||||
"""Merge FTS + vector by reciprocal rank fusion; optional D24 as-of filter."""
|
||||
fused: dict[str, dict] = {}
|
||||
for rank, hit in enumerate(fts_hits):
|
||||
fused.setdefault(hit["id"], {**hit, "rrf": 0.0})["rrf"] = 1.0 / (60 + rank + 1)
|
||||
@@ -315,8 +375,10 @@ def hybrid_search(conn: ladybug.Connection, embedding: list[float], fts_hits: li
|
||||
entry = fused.setdefault(hit["id"], {**hit, "rrf": 0.0})
|
||||
entry["rrf"] += 1.0 / (60 + rank + 1)
|
||||
entry.setdefault("score", hit.get("score", 0.0))
|
||||
entry.setdefault("valid_from", hit.get("valid_from") or "")
|
||||
entry.setdefault("valid_to", hit.get("valid_to") or "")
|
||||
ranked = sorted(fused.values(), key=lambda h: h.get("rrf", 0.0), reverse=True)
|
||||
return ranked[:limit]
|
||||
return filter_as_of(ranked, as_of)[:limit]
|
||||
|
||||
|
||||
def stats(conn: ladybug.Connection) -> dict:
|
||||
|
||||
@@ -194,6 +194,38 @@ class KblibTest(unittest.TestCase):
|
||||
self.assertEqual(person["name"], "Ada Lovelace")
|
||||
self.assertEqual(person["depth"], 3)
|
||||
|
||||
def test_as_of_keeps_x_drops_y(self) -> None:
|
||||
"""OQ5/#36: as of 2025-01-01 → works-at-X, not works-at-Y."""
|
||||
kblib.upsert_leaf(
|
||||
self.conn, text="Andrey works at X", root="facts",
|
||||
confidence="confirmed", source="crm.md x contract.md",
|
||||
source_rev="r1", how="test", loc="/tmp", type_="fact",
|
||||
embedding=make_emb(0.5),
|
||||
valid_from="2024-03-01", valid_to="2025-07-15",
|
||||
)
|
||||
kblib.upsert_leaf(
|
||||
self.conn, text="Andrey works at Y", root="facts",
|
||||
confidence="confirmed", source="offer.md x payroll.md",
|
||||
source_rev="r1", how="test", loc="/tmp", type_="fact",
|
||||
embedding=make_emb(0.6),
|
||||
valid_from="2025-07-16", valid_to="",
|
||||
)
|
||||
kblib.ensure_indexes(self.conn)
|
||||
hits = kblib.query_fts(self.conn, "Andrey works", 10)
|
||||
kept = kblib.filter_as_of(hits, "2025-01-01")
|
||||
texts = [h["text"] for h in kept]
|
||||
self.assertTrue(any("works at X" in t for t in texts), texts)
|
||||
self.assertFalse(any("works at Y" in t for t in texts), texts)
|
||||
later = kblib.filter_as_of(hits, "2025-08-01")
|
||||
later_texts = [h["text"] for h in later]
|
||||
self.assertTrue(any("works at Y" in t for t in later_texts), later_texts)
|
||||
self.assertFalse(any("works at X" in t for t in later_texts), later_texts)
|
||||
|
||||
def test_active_at_pure(self) -> None:
|
||||
self.assertTrue(kblib.active_at("2024-03-01", "2025-07-15", "2025-01-01"))
|
||||
self.assertFalse(kblib.active_at("2025-07-16", "", "2025-01-01"))
|
||||
self.assertTrue(kblib.active_at("", "", "2025-01-01"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -186,3 +186,21 @@ class PublishedDocsTest(unittest.TestCase):
|
||||
self.assertIn("epic #16", index)
|
||||
agents = (ROOT / "AGENTS.md").read_text()
|
||||
self.assertIn("roadmap.md", agents)
|
||||
|
||||
def test_oq5_intervals_are_not_d16_stale(self) -> None:
|
||||
plan = (ROOT / "PLAN.md").read_text()
|
||||
design = (ROOT / "docs" / "design.md").read_text()
|
||||
road = (ROOT / "docs" / "roadmap.md").read_text()
|
||||
contradict = (ROOT / "internal" / "facts" / "contradict.go").read_text()
|
||||
interval = (ROOT / "internal" / "facts" / "interval.go").read_text()
|
||||
self.assertIn("OQ5", plan)
|
||||
self.assertIn("D24", plan)
|
||||
self.assertIn("issues/36", plan)
|
||||
self.assertIn("valid_from", plan)
|
||||
self.assertIn("--as-of", design)
|
||||
self.assertIn("issues/36", road)
|
||||
self.assertIn("**in**", road[road.index("OQ5"):road.index("OQ5") + 80])
|
||||
self.assertIn("temporal_freshness", contradict)
|
||||
self.assertNotIn("valid_from", contradict)
|
||||
self.assertIn("ActiveAt", interval)
|
||||
self.assertIn("NormalizeDay", interval)
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
"""bin/stack/{start,start-assistant,stop,status} — offline contract + fake PATH."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
METHODS = ("start", "start-assistant", "stop", "status")
|
||||
|
||||
|
||||
class StackLayoutTest(unittest.TestCase):
|
||||
def test_methods_are_bash_with_usage_comment(self) -> None:
|
||||
lib = ROOT / "bin" / "stack" / "lib.sh"
|
||||
self.assertTrue(lib.is_file(), "missing bin/stack/lib.sh")
|
||||
for name in METHODS:
|
||||
p = ROOT / "bin" / "stack" / name
|
||||
self.assertTrue(p.is_file(), f"missing bin/stack/{name}")
|
||||
self.assertTrue(os.access(p, os.X_OK), f"bin/stack/{name} must be executable")
|
||||
lines = p.read_text().splitlines()
|
||||
self.assertEqual(lines[0], "#!/usr/bin/env bash", name)
|
||||
self.assertTrue(lines[1].startswith("# bin/stack/"), name)
|
||||
text = "\n".join(lines)
|
||||
self.assertIn("lib.sh", text, name)
|
||||
self.assertNotIn("/mnt/", text, name)
|
||||
self.assertNotIn("/home/", text, name)
|
||||
|
||||
def test_lib_has_no_host_paths_or_secrets(self) -> None:
|
||||
lib = (ROOT / "bin" / "stack" / "lib.sh").read_text()
|
||||
self.assertIn("stack_start", lib)
|
||||
self.assertIn("stack_start_assistant", lib)
|
||||
self.assertIn("stack_stop", lib)
|
||||
self.assertIn("stack_status", lib)
|
||||
self.assertIn("qwen3.5:9b", lib)
|
||||
self.assertIn("picoclaw agent", lib)
|
||||
self.assertIn("--no-deps", lib)
|
||||
self.assertIn("tools/list", lib)
|
||||
self.assertNotIn("/mnt/", lib)
|
||||
self.assertNotIn("/home/", lib)
|
||||
self.assertNotIn("password", lib.lower())
|
||||
self.assertNotIn("GITEA_TOKEN", lib)
|
||||
|
||||
def test_start_does_not_launch_picoclaw(self) -> None:
|
||||
start = (ROOT / "bin" / "stack" / "start").read_text()
|
||||
self.assertIn("stack_start", start)
|
||||
self.assertNotIn("stack_start_assistant", start)
|
||||
self.assertNotIn("picoclaw agent", start)
|
||||
|
||||
def test_start_assistant_attaches_agent(self) -> None:
|
||||
src = (ROOT / "bin" / "stack" / "start-assistant").read_text()
|
||||
self.assertIn("stack_start_assistant", src)
|
||||
self.assertIn("--no-attach", src)
|
||||
|
||||
def test_stop_does_not_down_volumes(self) -> None:
|
||||
lib = (ROOT / "bin" / "stack" / "lib.sh").read_text()
|
||||
self.assertIn(" compose ", lib)
|
||||
self.assertRegex(lib, r"\bstop\b")
|
||||
self.assertNotIn(" compose down", lib)
|
||||
self.assertNotIn("compose down", lib)
|
||||
|
||||
def test_docs_name_stack_commands(self) -> None:
|
||||
runbook = (ROOT / "docs" / "runbook.md").read_text()
|
||||
pico = (ROOT / "docs" / "picoclaw.md").read_text()
|
||||
agents = (ROOT / "AGENTS.md").read_text()
|
||||
for text in (runbook, pico, agents):
|
||||
self.assertIn("bin/stack/start", text)
|
||||
self.assertIn("bin/stack/start-assistant", text)
|
||||
self.assertIn("bin/stack/status", text)
|
||||
self.assertIn("bin/stack/stop", text)
|
||||
|
||||
|
||||
def test_help_prints_comments_not_source(self) -> None:
|
||||
r = subprocess.run(
|
||||
[str(ROOT / "bin" / "stack" / "start"), "--help"],
|
||||
cwd=str(ROOT),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
self.assertIn("bin/stack/start", r.stdout)
|
||||
self.assertNotIn("set -euo pipefail", r.stdout)
|
||||
self.assertNotIn("source ", r.stdout)
|
||||
|
||||
|
||||
class StackFakePathTest(unittest.TestCase):
|
||||
def _fake_bin(self, tmp: Path, *, health_ok: bool) -> Path:
|
||||
bindir = tmp / "bin"
|
||||
bindir.mkdir()
|
||||
curl = bindir / "curl"
|
||||
docker = bindir / "docker"
|
||||
log = tmp / "docker.log"
|
||||
curl.write_text(
|
||||
f"""#!/usr/bin/env bash
|
||||
url=""
|
||||
for a in "$@"; do
|
||||
case "$a" in http*) url=$a ;;
|
||||
esac
|
||||
done
|
||||
if [ "{int(health_ok)}" = "0" ] && [[ "$url" == */health ]]; then
|
||||
echo '{{"status":"down"}}'
|
||||
exit 7
|
||||
fi
|
||||
case "$url" in
|
||||
*/mcp)
|
||||
echo '{{"jsonrpc":"2.0","id":1,"result":{{"tools":[{{"name":"search"}},{{"name":"get"}},{{"name":"audit"}}]}}}}'
|
||||
;;
|
||||
*/api/tags)
|
||||
echo '{{"models":[{{"name":"qwen3.5:9b"}}]}}'
|
||||
;;
|
||||
*/api/pull)
|
||||
echo '{{"status":"success"}}'
|
||||
;;
|
||||
*)
|
||||
echo '{{"status":"ok"}}'
|
||||
;;
|
||||
esac
|
||||
"""
|
||||
)
|
||||
docker.write_text(
|
||||
f"""#!/usr/bin/env bash
|
||||
echo "$*" >> "{log}"
|
||||
exit 0
|
||||
"""
|
||||
)
|
||||
curl.chmod(curl.stat().st_mode | stat.S_IEXEC)
|
||||
docker.chmod(docker.stat().st_mode | stat.S_IEXEC)
|
||||
return bindir
|
||||
|
||||
def _env(self, bindir: Path) -> dict[str, str]:
|
||||
env = os.environ.copy()
|
||||
env["PATH"] = f"{bindir}:{env.get('PATH', '')}"
|
||||
env["STACK_WAIT_SECS"] = "1"
|
||||
env["STACK_WAIT_INTERVAL"] = "0"
|
||||
return env
|
||||
|
||||
def test_start_skips_compose_when_brain_healthy(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as raw:
|
||||
tmp = Path(raw)
|
||||
bindir = self._fake_bin(tmp, health_ok=True)
|
||||
log = tmp / "docker.log"
|
||||
r = subprocess.run(
|
||||
[str(ROOT / "bin" / "stack" / "start")],
|
||||
cwd=str(ROOT),
|
||||
env=self._env(bindir),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
self.assertFalse(log.exists(), "healthy brain must not docker compose up")
|
||||
|
||||
def test_start_ups_brain_when_unhealthy(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as raw:
|
||||
tmp = Path(raw)
|
||||
bindir = self._fake_bin(tmp, health_ok=False)
|
||||
log = tmp / "docker.log"
|
||||
r = subprocess.run(
|
||||
[str(ROOT / "bin" / "stack" / "start")],
|
||||
cwd=str(ROOT),
|
||||
env=self._env(bindir),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
self.assertNotEqual(r.returncode, 0, "unhealthy brain without recovering compose must fail")
|
||||
self.assertTrue(log.exists(), r.stderr)
|
||||
logged = log.read_text()
|
||||
self.assertIn("up -d", logged)
|
||||
self.assertIn("brain", logged)
|
||||
self.assertNotIn("picoclaw", logged)
|
||||
|
||||
def test_stop_stops_named_services(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as raw:
|
||||
tmp = Path(raw)
|
||||
bindir = self._fake_bin(tmp, health_ok=True)
|
||||
log = tmp / "docker.log"
|
||||
r = subprocess.run(
|
||||
[str(ROOT / "bin" / "stack" / "stop")],
|
||||
cwd=str(ROOT),
|
||||
env=self._env(bindir),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
logged = log.read_text()
|
||||
self.assertIn("stop", logged)
|
||||
self.assertNotIn(" down", logged)
|
||||
for svc in ("brain", "brain-mcp", "reasoner", "picoclaw"):
|
||||
self.assertIn(svc, logged)
|
||||
|
||||
def test_start_assistant_no_attach_starts_picoclaw(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as raw:
|
||||
tmp = Path(raw)
|
||||
bindir = self._fake_bin(tmp, health_ok=True)
|
||||
log = tmp / "docker.log"
|
||||
r = subprocess.run(
|
||||
[str(ROOT / "bin" / "stack" / "start-assistant"), "--no-attach"],
|
||||
cwd=str(ROOT),
|
||||
env=self._env(bindir),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
self.assertEqual(r.returncode, 0, r.stderr + r.stdout)
|
||||
logged = log.read_text() if log.exists() else ""
|
||||
self.assertIn("picoclaw", logged)
|
||||
self.assertIn("--no-deps", logged)
|
||||
self.assertNotIn("picoclaw agent", logged)
|
||||
Reference in New Issue
Block a user