feat: D24 fact intervals (--as-of) and bin/stack assistant helpers.
Tests / Test (push) Skipped
Tests / OCR (tesseract fixture) (push) Skipped
Tests / Release (semver) (push) Skipped
Tests / Test (pull_request) Failing after 5s
Tests / OCR (tesseract fixture) (pull_request) Failing after 4s
Tests / Release (semver) (pull_request) Skipped

Store valid_from/valid_to on leafs and filter search by calendar day without
overloading D16 source staleness; stack start/start-assistant wires brain + PicoClaw.
This commit is contained in:
2026-08-14 15:45:09 +01:00
parent 0c4bb87001
commit 094e2cb9df
34 changed files with 991 additions and 47 deletions
+9
View File
@@ -85,9 +85,18 @@ func openWithSandbox(epsv string) error {
closeBrain()
return fmt.Errorf("LOAD EXTENSION VECTOR: %w", err)
}
migrateIntervalColumns()
return nil
}
// migrateIntervalColumns adds D24 valid_from/valid_to on existing Leaf tables.
// Fresh CREATE already has them; ALTER is a no-op when the column exists.
func migrateIntervalColumns() {
for _, col := range []string{"valid_from", "valid_to"} {
_, _ = conn.Query("ALTER TABLE Leaf ADD " + col + " STRING")
}
}
func closeBrain() {
if conn != nil {
conn.Close()
+3 -3
View File
@@ -21,8 +21,8 @@ func Ready() error {
// HTTP is the in-process API used by bin/brain/serve.go.
type HTTP struct{}
func (HTTP) Search(ctx context.Context, query string, limit int) ([]byte, error) {
hits, err := searchHits(query, "", "", limit)
func (HTTP) Search(ctx context.Context, query string, limit int, asOf string) ([]byte, error) {
hits, err := searchHits(query, "", "", limit, asOf)
if err != nil {
return nil, err
}
@@ -41,7 +41,7 @@ func (HTTP) Search(ctx context.Context, query string, limit int) ([]byte, error)
var buf bytes.Buffer
enc := json.NewEncoder(&buf)
enc.SetEscapeHTML(false)
if err := enc.Encode(toJSONOut(hits, query, "", webOut)); err != nil {
if err := enc.Encode(toJSONOut(hits, query, "", asOf, webOut)); err != nil {
return nil, err
}
return buf.Bytes(), nil
+11 -1
View File
@@ -5,10 +5,11 @@ import (
"strconv"
"github.com/eSlider/2dph/internal/cli"
"github.com/eSlider/2dph/internal/facts"
"github.com/integrii/flaggy"
)
const Usage = `usage: bin/brain/search.go "query" [--root facts|info] [--repo REPO] [-n N] [--hop N] [--json] [--no-web]
const Usage = `usage: bin/brain/search.go "query" [--root facts|info] [--repo REPO] [-n N] [--hop N] [--as-of YYYY-MM-DD] [--json] [--no-web]
bin/brain/search.go serve [port]
bin/brain/search.go --list-model
source <(./bin/cli/complete.go bash)`
@@ -19,6 +20,7 @@ type Options struct {
Repo string
Limit int
Hop int
AsOf string
JSONOut bool
ListModel bool
NoWeb bool
@@ -35,6 +37,7 @@ func NewParser(opt *Options) *flaggy.Parser {
p.String(&opt.Repo, "", "repo", "filter by repo")
p.Int(&opt.Limit, "n", "n", "max hits")
p.Int(&opt.Hop, "", "hop", "walk FROM_FILE depth 1-3")
p.String(&opt.AsOf, "", "as-of", "keep facts active on YYYY-MM-DD (D24)")
p.Bool(&opt.JSONOut, "", "json", "JSON output")
p.Bool(&opt.NoWeb, "", "no-web", "stay local")
p.Bool(&opt.ListModel, "", "list-model", "print embedding model")
@@ -64,6 +67,13 @@ func ParseArgs(args []string) (Options, error) {
if opt.Hop > 3 {
return opt, fmt.Errorf("--hop max is 3 (File → Commit → Person)")
}
if opt.AsOf != "" {
day := facts.NormalizeDay(opt.AsOf)
if len(day) != 10 || day[4] != '-' || day[7] != '-' {
return opt, fmt.Errorf("--as-of must be YYYY-MM-DD, got %q", opt.AsOf)
}
opt.AsOf = day
}
if opt.Query == "" && !opt.ListModel {
return opt, fmt.Errorf("no query given")
}
+34
View File
@@ -0,0 +1,34 @@
package rank
import "testing"
func TestFilterAsOfKeepsXDropsY(t *testing.T) {
hits := []Hit{
{ID: "x", Text: "Andrey works at X", ValidFrom: "2024-03-01", ValidTo: "2025-07-15"},
{ID: "y", Text: "Andrey works at Y", ValidFrom: "2025-07-16", ValidTo: ""},
{ID: "legacy", Text: "always true claim", ValidFrom: "", ValidTo: ""},
}
out := FilterAsOf(hits, "2025-01-01")
if len(out) != 2 {
t.Fatalf("len=%d want 2: %+v", len(out), out)
}
if out[0].ID != "x" || out[1].ID != "legacy" {
t.Fatalf("got %+v", out)
}
if FilterAsOf(hits, "") == nil || len(FilterAsOf(hits, "")) != 3 {
t.Fatal("empty as-of must keep all")
}
}
func TestParseArgsAsOf(t *testing.T) {
opt, err := ParseArgs([]string{"who works where", "--as-of", "2025-01-01", "--json"})
if err != nil {
t.Fatal(err)
}
if opt.AsOf != "2025-01-01" {
t.Fatalf("AsOf=%q", opt.AsOf)
}
if _, err := ParseArgs([]string{"q", "--as-of", "not-a-date"}); err == nil {
t.Fatal("expected bad as-of error")
}
}
+4 -2
View File
@@ -3,10 +3,12 @@ package rank
// BM25 ranks best-first, so the top hits are the *highest* scores; cosine
// distance ranks best-first ascending. Both mirror kblib.py.
const FTSStmt = "CALL QUERY_FTS_INDEX('Leaf', 'id', $q) " +
"RETURN node.id, node.text, node.root, node.source, score, node.confidence ORDER BY score DESC LIMIT $n"
"RETURN node.id, node.text, node.root, node.source, score, node.confidence, " +
"node.valid_from, node.valid_to ORDER BY score DESC LIMIT $n"
const VecStmt = "CALL QUERY_VECTOR_INDEX('Leaf', 'Leaf_vec', $q, $n) " +
"RETURN node.id, node.text, node.root, node.source, distance, node.confidence ORDER BY distance LIMIT $n"
"RETURN node.id, node.text, node.root, node.source, distance, node.confidence, " +
"node.valid_from, node.valid_to ORDER BY distance LIMIT $n"
// HopStmt is the Cypher walk from a search hit. Depth 1 = File, 2 = Commit, 3 = Person.
func HopStmt(depth int) string {
+31 -4
View File
@@ -5,6 +5,8 @@ package rank
import (
"sort"
"strings"
"github.com/eSlider/2dph/internal/facts"
)
type HopNode struct {
@@ -23,17 +25,24 @@ type Hit struct {
Source string `json:"-"`
Score float64 `json:"score"`
Snippet string `json:"snippet,omitempty"`
ValidFrom string `json:"valid_from,omitempty"`
ValidTo string `json:"valid_to,omitempty"`
Hops []HopNode `json:"hops,omitempty"`
}
// rrfK dampens the contribution of low ranks; same constant as kblib.py.
const rrfK = 60
// RankAndFilter fuses the two hit lists, applies --root/--repo, then cuts to
// limit. Cutting first dropped every matching leaf ranked below the cut, so
// `--root facts` came back empty whenever info leafs filled the top N.
// limit <= 0 keeps everything.
// RankAndFilter fuses the two hit lists, applies --root/--repo/--as-of, then
// cuts to limit. Cutting first dropped every matching leaf ranked below the
// cut, so `--root facts` came back empty whenever info leafs filled the top N.
// limit <= 0 keeps everything. asOf empty skips interval filter (D24).
func RankAndFilter(fts, vec []Hit, root, repo string, limit int) []Hit {
return RankAndFilterAsOf(fts, vec, root, repo, "", limit)
}
// RankAndFilterAsOf is RankAndFilter with D24 fact-interval filter.
func RankAndFilterAsOf(fts, vec []Hit, root, repo, asOf string, limit int) []Hit {
out := Hybrid(fts, vec, 0)
if root != "" {
out = FilterRoot(out, root)
@@ -41,12 +50,30 @@ func RankAndFilter(fts, vec []Hit, root, repo string, limit int) []Hit {
if repo != "" {
out = FilterRepo(out, repo)
}
if asOf != "" {
out = FilterAsOf(out, asOf)
}
if limit > 0 && len(out) > limit {
out = out[:limit]
}
return out
}
// FilterAsOf keeps hits whose [valid_from, valid_to] covers asOf (D24).
// Empty intervals stay (legacy leafs). Empty asOf keeps all.
func FilterAsOf(hits []Hit, asOf string) []Hit {
if asOf == "" {
return hits
}
var out []Hit
for _, h := range hits {
if facts.ActiveAt(h.ValidFrom, h.ValidTo, asOf) {
out = append(out, h)
}
}
return out
}
// Hybrid merges FTS and vector hits by reciprocal rank fusion.
// limit <= 0 returns the full fused list.
func Hybrid(fts, vec []Hit, limit int) []Hit {
+38 -6
View File
@@ -55,7 +55,7 @@ func runSearch(args []string) int {
}
defer closeBrain()
hits, err := searchHits(query, root, repo, limit)
hits, err := searchHits(query, root, repo, limit, opt.AsOf)
if err != nil {
fmt.Fprintf(os.Stderr, "search: %v\n", err)
return 1
@@ -85,6 +85,7 @@ func runSearch(args []string) int {
out := Dict{
{"query", query},
{"root_filter", root},
{"as_of", opt.AsOf},
{"count", len(results)},
{"results", resultsToDicts(results)},
}
@@ -96,13 +97,13 @@ func runSearch(args []string) int {
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
enc.SetEscapeHTML(false)
return b2i(enc.Encode(toJSONOut(results, query, root, webOut)))
return b2i(enc.Encode(toJSONOut(results, query, root, opt.AsOf, webOut)))
}
fmt.Print(toYAML(out, 0))
return 0
}
func searchHits(query, root, repo string, limit int) ([]Hit, error) {
func searchHits(query, root, repo string, limit int, asOf string) ([]Hit, error) {
emb, err := embedQuery(query)
if err != nil {
return nil, fmt.Errorf("embed: %w", err)
@@ -115,7 +116,7 @@ func searchHits(query, root, repo string, limit int) ([]Hit, error) {
if vec, err = queryVector(emb, limit*3); err != nil {
fmt.Fprintf(os.Stderr, "vec: %v\n", err)
}
return rank.RankAndFilter(fts, vec, root, repo, limit), nil
return rank.RankAndFilterAsOf(fts, vec, root, repo, asOf, limit), nil
}
func attachHops(hits []Hit, n int) error {
@@ -220,15 +221,35 @@ func rowsToHits(res *lbug.QueryResult) ([]Hit, error) {
if len(vals) >= 6 {
conf = fmt.Sprint(vals[5])
}
hits = append(hits, Hit{ID: id, Text: text, Root: root, Source: source, Score: score, Confidence: conf})
vf, vt := "", ""
if len(vals) >= 8 {
vf = nullStr(vals[6])
vt = nullStr(vals[7])
}
hits = append(hits, Hit{
ID: id, Text: text, Root: root, Source: source, Score: score,
Confidence: conf, ValidFrom: vf, ValidTo: vt,
})
}
return hits, nil
}
func nullStr(v any) string {
if v == nil {
return ""
}
s := fmt.Sprint(v)
if s == "<nil>" {
return ""
}
return s
}
// JSON output types
type jsonOut struct {
Query string `json:"query"`
RootFilter string `json:"root_filter"`
AsOf string `json:"as_of,omitempty"`
Count int `json:"count"`
Results []jsonHit `json:"results"`
Web *rank.SecondSource `json:"web,omitempty"`
@@ -241,10 +262,12 @@ type jsonHit struct {
Confidence string `json:"confidence,omitempty"`
Score float64 `json:"score"`
Snippet string `json:"snippet,omitempty"`
ValidFrom string `json:"valid_from,omitempty"`
ValidTo string `json:"valid_to,omitempty"`
Hops []rank.HopNode `json:"hops,omitempty"`
}
func toJSONOut(hits []Hit, query, rootFilter string, web *rank.SecondSource) *jsonOut {
func toJSONOut(hits []Hit, query, rootFilter, asOf string, web *rank.SecondSource) *jsonOut {
out := make([]jsonHit, len(hits))
for i, h := range hits {
out[i] = jsonHit{
@@ -254,12 +277,15 @@ func toJSONOut(hits []Hit, query, rootFilter string, web *rank.SecondSource) *js
Confidence: h.Confidence,
Score: h.Score,
Snippet: h.Snippet,
ValidFrom: h.ValidFrom,
ValidTo: h.ValidTo,
Hops: h.Hops,
}
}
return &jsonOut{
Query: query,
RootFilter: rootFilter,
AsOf: asOf,
Count: len(hits),
Results: out,
Web: web,
@@ -278,6 +304,12 @@ func resultsToDicts(hits []Hit) []any {
if h.Confidence != "" {
d = append(d, KV{"confidence", h.Confidence})
}
if h.ValidFrom != "" {
d = append(d, KV{"valid_from", h.ValidFrom})
}
if h.ValidTo != "" {
d = append(d, KV{"valid_to", h.ValidTo})
}
if h.Snippet != "" {
d = append(d, KV{"snippet", h.Snippet})
}
+44
View File
@@ -0,0 +1,44 @@
package facts
// Interval of truth for a fact leaf (D24 / OQ5). Not D16 source staleness.
//
// Empty valid_from and valid_to means "always" (legacy leafs). Empty asOf
// means "do not filter". Dates compare as YYYY-MM-DD (lexicographic).
// NormalizeDay keeps the calendar day from ISO-8601 or bare dates.
func NormalizeDay(s string) string {
s = trimSpace(s)
if len(s) >= 10 && s[4] == '-' && s[7] == '-' {
return s[:10]
}
return s
}
func trimSpace(s string) string {
i, j := 0, len(s)
for i < j && (s[i] == ' ' || s[i] == '\t' || s[i] == '\n' || s[i] == '\r') {
i++
}
for j > i && (s[j-1] == ' ' || s[j-1] == '\t' || s[j-1] == '\n' || s[j-1] == '\r') {
j--
}
return s[i:j]
}
// ActiveAt reports whether a fact with [validFrom, validTo] holds at asOf.
// validTo empty = open-ended. Both ends inclusive.
func ActiveAt(validFrom, validTo, asOf string) bool {
asOf = NormalizeDay(asOf)
if asOf == "" {
return true
}
from := NormalizeDay(validFrom)
to := NormalizeDay(validTo)
if from != "" && asOf < from {
return false
}
if to != "" && asOf > to {
return false
}
return true
}
+73
View File
@@ -0,0 +1,73 @@
package facts
import "testing"
func TestActiveAtOpenEnded(t *testing.T) {
// works at Y from 2025-07-16, no end
if !ActiveAt("2025-07-16", "", "2025-07-16") {
t.Fatal("inclusive valid_from")
}
if !ActiveAt("2025-07-16", "", "2026-01-01") {
t.Fatal("open-ended valid_to")
}
if ActiveAt("2025-07-16", "", "2025-07-15") {
t.Fatal("before valid_from must be inactive")
}
}
func TestActiveAtClosedInterval(t *testing.T) {
// works at X 2024-03-01 .. 2025-07-15
if !ActiveAt("2024-03-01", "2025-07-15", "2025-01-01") {
t.Fatal("mid interval")
}
if !ActiveAt("2024-03-01", "2025-07-15", "2024-03-01") {
t.Fatal("inclusive start")
}
if !ActiveAt("2024-03-01", "2025-07-15", "2025-07-15") {
t.Fatal("inclusive end")
}
if ActiveAt("2024-03-01", "2025-07-15", "2025-07-16") {
t.Fatal("day after end")
}
if ActiveAt("2024-03-01", "2025-07-15", "2024-02-28") {
t.Fatal("day before start")
}
}
func TestActiveAtEmptyIntervalAlwaysTrue(t *testing.T) {
// legacy leafs without intervals stay visible for any as-of
if !ActiveAt("", "", "2025-01-01") {
t.Fatal("empty interval must remain active")
}
if !ActiveAt("", "", "") {
t.Fatal("no as-of means all active")
}
}
func TestActiveAtEmptyAsOfKeepsAll(t *testing.T) {
if !ActiveAt("2099-01-01", "2099-12-31", "") {
t.Fatal("empty as-of must not filter")
}
}
func TestAsOfPickXNotY(t *testing.T) {
// Acceptance from #36: as of 2025-01-01 → X, not Y
xFrom, xTo := "2024-03-01", "2025-07-15"
yFrom, yTo := "2025-07-16", ""
asOf := "2025-01-01"
if !ActiveAt(xFrom, xTo, asOf) {
t.Fatal("X must be active as of 2025-01-01")
}
if ActiveAt(yFrom, yTo, asOf) {
t.Fatal("Y must be inactive as of 2025-01-01")
}
}
func TestNormalizeDayTrimsTime(t *testing.T) {
if NormalizeDay("2025-01-01T12:00:00Z") != "2025-01-01" {
t.Fatalf("got %q", NormalizeDay("2025-01-01T12:00:00Z"))
}
if NormalizeDay("2025-01-01") != "2025-01-01" {
t.Fatalf("got %q", NormalizeDay("2025-01-01"))
}
}
+8 -1
View File
@@ -105,11 +105,18 @@ func (s *Server) mcpCall(r *http.Request, params json.RawMessage) (any, error) {
if limit < 1 || limit > 100 {
return mcpText(`{"error":"n must be int 1..100"}`, true), nil
}
asOf := ""
if raw, ok := p.Arguments["as_of"]; ok {
asOf = strings.TrimSpace(fmt.Sprint(raw))
if asOf == "<nil>" {
asOf = ""
}
}
if !s.tryAcquire(r) {
return nil, fmt.Errorf("cancelled")
}
defer s.release()
body, err = s.api.Search(r.Context(), q, limit)
body, err = s.api.Search(r.Context(), q, limit, asOf)
case "get":
id := strings.TrimSpace(fmt.Sprint(p.Arguments["id"]))
if id == "" || id == "<nil>" {
+10 -4
View File
@@ -24,7 +24,7 @@ import (
// API is the in-process brain surface. Production serve.go wires internal/brain.
type API interface {
Search(ctx context.Context, query string, limit int) ([]byte, error)
Search(ctx context.Context, query string, limit int, asOf string) ([]byte, error)
Get(ctx context.Context, id string, body bool) ([]byte, error)
Stats(ctx context.Context) ([]byte, error)
Audit(ctx context.Context) ([]byte, error)
@@ -85,11 +85,12 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) {
}
limit = n
}
asOf := strings.TrimSpace(r.URL.Query().Get("as_of"))
if !s.acquire(w, r) {
return
}
defer s.release()
body, err := s.api.Search(r.Context(), q, limit)
body, err := s.api.Search(r.Context(), q, limit, asOf)
writeAPI(w, body, err)
}
@@ -187,13 +188,18 @@ type ExecSearcher struct {
Timeout time.Duration
}
func (b ExecSearcher) Search(ctx context.Context, query string, limit int) ([]byte, error) {
func (b ExecSearcher) Search(ctx context.Context, query string, limit int, asOf string) ([]byte, error) {
if b.Timeout == 0 {
b.Timeout = 60 * time.Second
}
ctx, cancel := context.WithTimeout(ctx, b.Timeout)
defer cancel()
cmd := exec.CommandContext(ctx, b.CmdPath, "--json", "-n", strconv.Itoa(limit), query)
args := []string{"--json", "-n", strconv.Itoa(limit)}
if asOf != "" {
args = append(args, "--as-of", asOf)
}
args = append(args, query)
cmd := exec.CommandContext(ctx, b.CmdPath, args...)
out, err := cmd.Output()
if err != nil {
var exitErr *exec.ExitError
+5 -5
View File
@@ -21,10 +21,10 @@ type fakeSearcher struct {
calls int
active atomic.Int32
maxSeen atomic.Int32
callback func(q string, limit int) ([]byte, error)
callback func(q string, limit int, asOf string) ([]byte, error)
}
func (f *fakeSearcher) Search(ctx context.Context, query string, limit int) ([]byte, error) {
func (f *fakeSearcher) Search(ctx context.Context, query string, limit int, asOf string) ([]byte, error) {
f.mu.Lock()
f.calls++
f.mu.Unlock()
@@ -44,7 +44,7 @@ func (f *fakeSearcher) Search(ctx context.Context, query string, limit int) ([]b
}
}
if f.callback != nil {
return f.callback(query, limit)
return f.callback(query, limit, asOf)
}
return []byte(`{"query":"` + query + `","count":0,"results":[]}`), nil
}
@@ -109,7 +109,7 @@ func TestSearchMissingQuery(t *testing.T) {
}
func TestSearchReturnsSearcherResult(t *testing.T) {
fs := &fakeSearcher{callback: func(q string, limit int) ([]byte, error) {
fs := &fakeSearcher{callback: func(q string, limit int, asOf string) ([]byte, error) {
return []byte(`{"query":"` + q + `","count":1,"results":[{"id":"x"}]}`), nil
}}
h := NewServer(fs, 1)
@@ -168,7 +168,7 @@ func TestSearchRejectsBadLimit(t *testing.T) {
}
func TestGetLeaf(t *testing.T) {
fs := &fakeSearcher{callback: func(q string, limit int) ([]byte, error) {
fs := &fakeSearcher{callback: func(q string, limit int, asOf string) ([]byte, error) {
return []byte(`{}`), nil
}}
h := NewServer(fs, 1)
+3
View File
@@ -35,6 +35,7 @@ var Ops = []Op{
Params: []Param{
{Name: "q", In: "query", Type: "string", Description: "search query", Required: true},
{Name: "n", In: "query", Type: "integer", Description: "hit limit 1..100 (default 10)"},
{Name: "as_of", In: "query", Type: "string", Description: "YYYY-MM-DD; keep facts active on that day (D24)"},
},
},
{
@@ -54,6 +55,8 @@ var Ops = []Op{
{Name: "text", In: "query", Type: "string", Description: "leaf text (omit for CLI hint)"},
{Name: "root", In: "query", Type: "string", Description: "facts or info (default info)"},
{Name: "source", In: "query", Type: "string", Description: "evidence pointer; facts need two sources"},
{Name: "valid_from", In: "query", Type: "string", Description: "fact interval start YYYY-MM-DD (D24)"},
{Name: "valid_to", In: "query", Type: "string", Description: "fact interval end YYYY-MM-DD inclusive (D24)"},
},
},
{Path: PathOpenAPI, Method: "get", ID: "openapi", Summary: "OpenAPI 3 document for this server"},