feat: D24 fact intervals (--as-of) and bin/stack assistant helpers.
Tests / Test (push) Skipped
Tests / OCR (tesseract fixture) (push) Skipped
Tests / Release (semver) (push) Skipped
Tests / Test (pull_request) Failing after 5s
Tests / OCR (tesseract fixture) (pull_request) Failing after 4s
Tests / Release (semver) (pull_request) Skipped

Store valid_from/valid_to on leafs and filter search by calendar day without
overloading D16 source staleness; stack start/start-assistant wires brain + PicoClaw.
This commit is contained in:
2026-08-14 15:45:09 +01:00
parent 0c4bb87001
commit 094e2cb9df
34 changed files with 991 additions and 47 deletions
+9
View File
@@ -85,9 +85,18 @@ func openWithSandbox(epsv string) error {
closeBrain()
return fmt.Errorf("LOAD EXTENSION VECTOR: %w", err)
}
migrateIntervalColumns()
return nil
}
// migrateIntervalColumns adds D24 valid_from/valid_to on existing Leaf tables.
// Fresh CREATE already has them; ALTER is a no-op when the column exists.
func migrateIntervalColumns() {
for _, col := range []string{"valid_from", "valid_to"} {
_, _ = conn.Query("ALTER TABLE Leaf ADD " + col + " STRING")
}
}
func closeBrain() {
if conn != nil {
conn.Close()
+3 -3
View File
@@ -21,8 +21,8 @@ func Ready() error {
// HTTP is the in-process API used by bin/brain/serve.go.
type HTTP struct{}
func (HTTP) Search(ctx context.Context, query string, limit int) ([]byte, error) {
hits, err := searchHits(query, "", "", limit)
func (HTTP) Search(ctx context.Context, query string, limit int, asOf string) ([]byte, error) {
hits, err := searchHits(query, "", "", limit, asOf)
if err != nil {
return nil, err
}
@@ -41,7 +41,7 @@ func (HTTP) Search(ctx context.Context, query string, limit int) ([]byte, error)
var buf bytes.Buffer
enc := json.NewEncoder(&buf)
enc.SetEscapeHTML(false)
if err := enc.Encode(toJSONOut(hits, query, "", webOut)); err != nil {
if err := enc.Encode(toJSONOut(hits, query, "", asOf, webOut)); err != nil {
return nil, err
}
return buf.Bytes(), nil
+11 -1
View File
@@ -5,10 +5,11 @@ import (
"strconv"
"github.com/eSlider/2dph/internal/cli"
"github.com/eSlider/2dph/internal/facts"
"github.com/integrii/flaggy"
)
const Usage = `usage: bin/brain/search.go "query" [--root facts|info] [--repo REPO] [-n N] [--hop N] [--json] [--no-web]
const Usage = `usage: bin/brain/search.go "query" [--root facts|info] [--repo REPO] [-n N] [--hop N] [--as-of YYYY-MM-DD] [--json] [--no-web]
bin/brain/search.go serve [port]
bin/brain/search.go --list-model
source <(./bin/cli/complete.go bash)`
@@ -19,6 +20,7 @@ type Options struct {
Repo string
Limit int
Hop int
AsOf string
JSONOut bool
ListModel bool
NoWeb bool
@@ -35,6 +37,7 @@ func NewParser(opt *Options) *flaggy.Parser {
p.String(&opt.Repo, "", "repo", "filter by repo")
p.Int(&opt.Limit, "n", "n", "max hits")
p.Int(&opt.Hop, "", "hop", "walk FROM_FILE depth 1-3")
p.String(&opt.AsOf, "", "as-of", "keep facts active on YYYY-MM-DD (D24)")
p.Bool(&opt.JSONOut, "", "json", "JSON output")
p.Bool(&opt.NoWeb, "", "no-web", "stay local")
p.Bool(&opt.ListModel, "", "list-model", "print embedding model")
@@ -64,6 +67,13 @@ func ParseArgs(args []string) (Options, error) {
if opt.Hop > 3 {
return opt, fmt.Errorf("--hop max is 3 (File → Commit → Person)")
}
if opt.AsOf != "" {
day := facts.NormalizeDay(opt.AsOf)
if len(day) != 10 || day[4] != '-' || day[7] != '-' {
return opt, fmt.Errorf("--as-of must be YYYY-MM-DD, got %q", opt.AsOf)
}
opt.AsOf = day
}
if opt.Query == "" && !opt.ListModel {
return opt, fmt.Errorf("no query given")
}
+34
View File
@@ -0,0 +1,34 @@
package rank
import "testing"
func TestFilterAsOfKeepsXDropsY(t *testing.T) {
hits := []Hit{
{ID: "x", Text: "Andrey works at X", ValidFrom: "2024-03-01", ValidTo: "2025-07-15"},
{ID: "y", Text: "Andrey works at Y", ValidFrom: "2025-07-16", ValidTo: ""},
{ID: "legacy", Text: "always true claim", ValidFrom: "", ValidTo: ""},
}
out := FilterAsOf(hits, "2025-01-01")
if len(out) != 2 {
t.Fatalf("len=%d want 2: %+v", len(out), out)
}
if out[0].ID != "x" || out[1].ID != "legacy" {
t.Fatalf("got %+v", out)
}
if FilterAsOf(hits, "") == nil || len(FilterAsOf(hits, "")) != 3 {
t.Fatal("empty as-of must keep all")
}
}
func TestParseArgsAsOf(t *testing.T) {
opt, err := ParseArgs([]string{"who works where", "--as-of", "2025-01-01", "--json"})
if err != nil {
t.Fatal(err)
}
if opt.AsOf != "2025-01-01" {
t.Fatalf("AsOf=%q", opt.AsOf)
}
if _, err := ParseArgs([]string{"q", "--as-of", "not-a-date"}); err == nil {
t.Fatal("expected bad as-of error")
}
}
+4 -2
View File
@@ -3,10 +3,12 @@ package rank
// BM25 ranks best-first, so the top hits are the *highest* scores; cosine
// distance ranks best-first ascending. Both mirror kblib.py.
const FTSStmt = "CALL QUERY_FTS_INDEX('Leaf', 'id', $q) " +
"RETURN node.id, node.text, node.root, node.source, score, node.confidence ORDER BY score DESC LIMIT $n"
"RETURN node.id, node.text, node.root, node.source, score, node.confidence, " +
"node.valid_from, node.valid_to ORDER BY score DESC LIMIT $n"
const VecStmt = "CALL QUERY_VECTOR_INDEX('Leaf', 'Leaf_vec', $q, $n) " +
"RETURN node.id, node.text, node.root, node.source, distance, node.confidence ORDER BY distance LIMIT $n"
"RETURN node.id, node.text, node.root, node.source, distance, node.confidence, " +
"node.valid_from, node.valid_to ORDER BY distance LIMIT $n"
// HopStmt is the Cypher walk from a search hit. Depth 1 = File, 2 = Commit, 3 = Person.
func HopStmt(depth int) string {
+31 -4
View File
@@ -5,6 +5,8 @@ package rank
import (
"sort"
"strings"
"github.com/eSlider/2dph/internal/facts"
)
type HopNode struct {
@@ -23,17 +25,24 @@ type Hit struct {
Source string `json:"-"`
Score float64 `json:"score"`
Snippet string `json:"snippet,omitempty"`
ValidFrom string `json:"valid_from,omitempty"`
ValidTo string `json:"valid_to,omitempty"`
Hops []HopNode `json:"hops,omitempty"`
}
// rrfK dampens the contribution of low ranks; same constant as kblib.py.
const rrfK = 60
// RankAndFilter fuses the two hit lists, applies --root/--repo, then cuts to
// limit. Cutting first dropped every matching leaf ranked below the cut, so
// `--root facts` came back empty whenever info leafs filled the top N.
// limit <= 0 keeps everything.
// RankAndFilter fuses the two hit lists, applies --root/--repo/--as-of, then
// cuts to limit. Cutting first dropped every matching leaf ranked below the
// cut, so `--root facts` came back empty whenever info leafs filled the top N.
// limit <= 0 keeps everything. asOf empty skips interval filter (D24).
func RankAndFilter(fts, vec []Hit, root, repo string, limit int) []Hit {
return RankAndFilterAsOf(fts, vec, root, repo, "", limit)
}
// RankAndFilterAsOf is RankAndFilter with D24 fact-interval filter.
func RankAndFilterAsOf(fts, vec []Hit, root, repo, asOf string, limit int) []Hit {
out := Hybrid(fts, vec, 0)
if root != "" {
out = FilterRoot(out, root)
@@ -41,12 +50,30 @@ func RankAndFilter(fts, vec []Hit, root, repo string, limit int) []Hit {
if repo != "" {
out = FilterRepo(out, repo)
}
if asOf != "" {
out = FilterAsOf(out, asOf)
}
if limit > 0 && len(out) > limit {
out = out[:limit]
}
return out
}
// FilterAsOf keeps hits whose [valid_from, valid_to] covers asOf (D24).
// Empty intervals stay (legacy leafs). Empty asOf keeps all.
func FilterAsOf(hits []Hit, asOf string) []Hit {
if asOf == "" {
return hits
}
var out []Hit
for _, h := range hits {
if facts.ActiveAt(h.ValidFrom, h.ValidTo, asOf) {
out = append(out, h)
}
}
return out
}
// Hybrid merges FTS and vector hits by reciprocal rank fusion.
// limit <= 0 returns the full fused list.
func Hybrid(fts, vec []Hit, limit int) []Hit {
+38 -6
View File
@@ -55,7 +55,7 @@ func runSearch(args []string) int {
}
defer closeBrain()
hits, err := searchHits(query, root, repo, limit)
hits, err := searchHits(query, root, repo, limit, opt.AsOf)
if err != nil {
fmt.Fprintf(os.Stderr, "search: %v\n", err)
return 1
@@ -85,6 +85,7 @@ func runSearch(args []string) int {
out := Dict{
{"query", query},
{"root_filter", root},
{"as_of", opt.AsOf},
{"count", len(results)},
{"results", resultsToDicts(results)},
}
@@ -96,13 +97,13 @@ func runSearch(args []string) int {
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
enc.SetEscapeHTML(false)
return b2i(enc.Encode(toJSONOut(results, query, root, webOut)))
return b2i(enc.Encode(toJSONOut(results, query, root, opt.AsOf, webOut)))
}
fmt.Print(toYAML(out, 0))
return 0
}
func searchHits(query, root, repo string, limit int) ([]Hit, error) {
func searchHits(query, root, repo string, limit int, asOf string) ([]Hit, error) {
emb, err := embedQuery(query)
if err != nil {
return nil, fmt.Errorf("embed: %w", err)
@@ -115,7 +116,7 @@ func searchHits(query, root, repo string, limit int) ([]Hit, error) {
if vec, err = queryVector(emb, limit*3); err != nil {
fmt.Fprintf(os.Stderr, "vec: %v\n", err)
}
return rank.RankAndFilter(fts, vec, root, repo, limit), nil
return rank.RankAndFilterAsOf(fts, vec, root, repo, asOf, limit), nil
}
func attachHops(hits []Hit, n int) error {
@@ -220,15 +221,35 @@ func rowsToHits(res *lbug.QueryResult) ([]Hit, error) {
if len(vals) >= 6 {
conf = fmt.Sprint(vals[5])
}
hits = append(hits, Hit{ID: id, Text: text, Root: root, Source: source, Score: score, Confidence: conf})
vf, vt := "", ""
if len(vals) >= 8 {
vf = nullStr(vals[6])
vt = nullStr(vals[7])
}
hits = append(hits, Hit{
ID: id, Text: text, Root: root, Source: source, Score: score,
Confidence: conf, ValidFrom: vf, ValidTo: vt,
})
}
return hits, nil
}
func nullStr(v any) string {
if v == nil {
return ""
}
s := fmt.Sprint(v)
if s == "<nil>" {
return ""
}
return s
}
// JSON output types
type jsonOut struct {
Query string `json:"query"`
RootFilter string `json:"root_filter"`
AsOf string `json:"as_of,omitempty"`
Count int `json:"count"`
Results []jsonHit `json:"results"`
Web *rank.SecondSource `json:"web,omitempty"`
@@ -241,10 +262,12 @@ type jsonHit struct {
Confidence string `json:"confidence,omitempty"`
Score float64 `json:"score"`
Snippet string `json:"snippet,omitempty"`
ValidFrom string `json:"valid_from,omitempty"`
ValidTo string `json:"valid_to,omitempty"`
Hops []rank.HopNode `json:"hops,omitempty"`
}
func toJSONOut(hits []Hit, query, rootFilter string, web *rank.SecondSource) *jsonOut {
func toJSONOut(hits []Hit, query, rootFilter, asOf string, web *rank.SecondSource) *jsonOut {
out := make([]jsonHit, len(hits))
for i, h := range hits {
out[i] = jsonHit{
@@ -254,12 +277,15 @@ func toJSONOut(hits []Hit, query, rootFilter string, web *rank.SecondSource) *js
Confidence: h.Confidence,
Score: h.Score,
Snippet: h.Snippet,
ValidFrom: h.ValidFrom,
ValidTo: h.ValidTo,
Hops: h.Hops,
}
}
return &jsonOut{
Query: query,
RootFilter: rootFilter,
AsOf: asOf,
Count: len(hits),
Results: out,
Web: web,
@@ -278,6 +304,12 @@ func resultsToDicts(hits []Hit) []any {
if h.Confidence != "" {
d = append(d, KV{"confidence", h.Confidence})
}
if h.ValidFrom != "" {
d = append(d, KV{"valid_from", h.ValidFrom})
}
if h.ValidTo != "" {
d = append(d, KV{"valid_to", h.ValidTo})
}
if h.Snippet != "" {
d = append(d, KV{"snippet", h.Snippet})
}