feat: D24 fact intervals (--as-of) and bin/stack assistant helpers. (#37)
Tests / Test (push) Skipped
Tests / OCR (tesseract fixture) (push) Skipped
Tests / Release (semver) (push) Skipped
Tests / Test (push) Skipped
Tests / OCR (tesseract fixture) (push) Skipped
Tests / Release (semver) (push) Skipped
Store valid_from/valid_to on leafs and filter search by calendar day without overloading D16 source staleness; stack start/start-assistant wires brain + PicoClaw.
This commit is contained in:
@@ -85,9 +85,18 @@ func openWithSandbox(epsv string) error {
|
||||
closeBrain()
|
||||
return fmt.Errorf("LOAD EXTENSION VECTOR: %w", err)
|
||||
}
|
||||
migrateIntervalColumns()
|
||||
return nil
|
||||
}
|
||||
|
||||
// migrateIntervalColumns adds D24 valid_from/valid_to on existing Leaf tables.
|
||||
// Fresh CREATE already has them; ALTER is a no-op when the column exists.
|
||||
func migrateIntervalColumns() {
|
||||
for _, col := range []string{"valid_from", "valid_to"} {
|
||||
_, _ = conn.Query("ALTER TABLE Leaf ADD " + col + " STRING")
|
||||
}
|
||||
}
|
||||
|
||||
func closeBrain() {
|
||||
if conn != nil {
|
||||
conn.Close()
|
||||
|
||||
@@ -21,8 +21,8 @@ func Ready() error {
|
||||
// HTTP is the in-process API used by bin/brain/serve.go.
|
||||
type HTTP struct{}
|
||||
|
||||
func (HTTP) Search(ctx context.Context, query string, limit int) ([]byte, error) {
|
||||
hits, err := searchHits(query, "", "", limit)
|
||||
func (HTTP) Search(ctx context.Context, query string, limit int, asOf string) ([]byte, error) {
|
||||
hits, err := searchHits(query, "", "", limit, asOf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -41,7 +41,7 @@ func (HTTP) Search(ctx context.Context, query string, limit int) ([]byte, error)
|
||||
var buf bytes.Buffer
|
||||
enc := json.NewEncoder(&buf)
|
||||
enc.SetEscapeHTML(false)
|
||||
if err := enc.Encode(toJSONOut(hits, query, "", webOut)); err != nil {
|
||||
if err := enc.Encode(toJSONOut(hits, query, "", asOf, webOut)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
|
||||
@@ -5,10 +5,11 @@ import (
|
||||
"strconv"
|
||||
|
||||
"github.com/eSlider/2dph/internal/cli"
|
||||
"github.com/eSlider/2dph/internal/facts"
|
||||
"github.com/integrii/flaggy"
|
||||
)
|
||||
|
||||
const Usage = `usage: bin/brain/search.go "query" [--root facts|info] [--repo REPO] [-n N] [--hop N] [--json] [--no-web]
|
||||
const Usage = `usage: bin/brain/search.go "query" [--root facts|info] [--repo REPO] [-n N] [--hop N] [--as-of YYYY-MM-DD] [--json] [--no-web]
|
||||
bin/brain/search.go serve [port]
|
||||
bin/brain/search.go --list-model
|
||||
source <(./bin/cli/complete.go bash)`
|
||||
@@ -19,6 +20,7 @@ type Options struct {
|
||||
Repo string
|
||||
Limit int
|
||||
Hop int
|
||||
AsOf string
|
||||
JSONOut bool
|
||||
ListModel bool
|
||||
NoWeb bool
|
||||
@@ -35,6 +37,7 @@ func NewParser(opt *Options) *flaggy.Parser {
|
||||
p.String(&opt.Repo, "", "repo", "filter by repo")
|
||||
p.Int(&opt.Limit, "n", "n", "max hits")
|
||||
p.Int(&opt.Hop, "", "hop", "walk FROM_FILE depth 1-3")
|
||||
p.String(&opt.AsOf, "", "as-of", "keep facts active on YYYY-MM-DD (D24)")
|
||||
p.Bool(&opt.JSONOut, "", "json", "JSON output")
|
||||
p.Bool(&opt.NoWeb, "", "no-web", "stay local")
|
||||
p.Bool(&opt.ListModel, "", "list-model", "print embedding model")
|
||||
@@ -64,6 +67,13 @@ func ParseArgs(args []string) (Options, error) {
|
||||
if opt.Hop > 3 {
|
||||
return opt, fmt.Errorf("--hop max is 3 (File → Commit → Person)")
|
||||
}
|
||||
if opt.AsOf != "" {
|
||||
day := facts.NormalizeDay(opt.AsOf)
|
||||
if len(day) != 10 || day[4] != '-' || day[7] != '-' {
|
||||
return opt, fmt.Errorf("--as-of must be YYYY-MM-DD, got %q", opt.AsOf)
|
||||
}
|
||||
opt.AsOf = day
|
||||
}
|
||||
if opt.Query == "" && !opt.ListModel {
|
||||
return opt, fmt.Errorf("no query given")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
package rank
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestFilterAsOfKeepsXDropsY(t *testing.T) {
|
||||
hits := []Hit{
|
||||
{ID: "x", Text: "Andrey works at X", ValidFrom: "2024-03-01", ValidTo: "2025-07-15"},
|
||||
{ID: "y", Text: "Andrey works at Y", ValidFrom: "2025-07-16", ValidTo: ""},
|
||||
{ID: "legacy", Text: "always true claim", ValidFrom: "", ValidTo: ""},
|
||||
}
|
||||
out := FilterAsOf(hits, "2025-01-01")
|
||||
if len(out) != 2 {
|
||||
t.Fatalf("len=%d want 2: %+v", len(out), out)
|
||||
}
|
||||
if out[0].ID != "x" || out[1].ID != "legacy" {
|
||||
t.Fatalf("got %+v", out)
|
||||
}
|
||||
if FilterAsOf(hits, "") == nil || len(FilterAsOf(hits, "")) != 3 {
|
||||
t.Fatal("empty as-of must keep all")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseArgsAsOf(t *testing.T) {
|
||||
opt, err := ParseArgs([]string{"who works where", "--as-of", "2025-01-01", "--json"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if opt.AsOf != "2025-01-01" {
|
||||
t.Fatalf("AsOf=%q", opt.AsOf)
|
||||
}
|
||||
if _, err := ParseArgs([]string{"q", "--as-of", "not-a-date"}); err == nil {
|
||||
t.Fatal("expected bad as-of error")
|
||||
}
|
||||
}
|
||||
@@ -3,10 +3,12 @@ package rank
|
||||
// BM25 ranks best-first, so the top hits are the *highest* scores; cosine
|
||||
// distance ranks best-first ascending. Both mirror kblib.py.
|
||||
const FTSStmt = "CALL QUERY_FTS_INDEX('Leaf', 'id', $q) " +
|
||||
"RETURN node.id, node.text, node.root, node.source, score, node.confidence ORDER BY score DESC LIMIT $n"
|
||||
"RETURN node.id, node.text, node.root, node.source, score, node.confidence, " +
|
||||
"node.valid_from, node.valid_to ORDER BY score DESC LIMIT $n"
|
||||
|
||||
const VecStmt = "CALL QUERY_VECTOR_INDEX('Leaf', 'Leaf_vec', $q, $n) " +
|
||||
"RETURN node.id, node.text, node.root, node.source, distance, node.confidence ORDER BY distance LIMIT $n"
|
||||
"RETURN node.id, node.text, node.root, node.source, distance, node.confidence, " +
|
||||
"node.valid_from, node.valid_to ORDER BY distance LIMIT $n"
|
||||
|
||||
// HopStmt is the Cypher walk from a search hit. Depth 1 = File, 2 = Commit, 3 = Person.
|
||||
func HopStmt(depth int) string {
|
||||
|
||||
@@ -5,6 +5,8 @@ package rank
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/eSlider/2dph/internal/facts"
|
||||
)
|
||||
|
||||
type HopNode struct {
|
||||
@@ -23,17 +25,24 @@ type Hit struct {
|
||||
Source string `json:"-"`
|
||||
Score float64 `json:"score"`
|
||||
Snippet string `json:"snippet,omitempty"`
|
||||
ValidFrom string `json:"valid_from,omitempty"`
|
||||
ValidTo string `json:"valid_to,omitempty"`
|
||||
Hops []HopNode `json:"hops,omitempty"`
|
||||
}
|
||||
|
||||
// rrfK dampens the contribution of low ranks; same constant as kblib.py.
|
||||
const rrfK = 60
|
||||
|
||||
// RankAndFilter fuses the two hit lists, applies --root/--repo, then cuts to
|
||||
// limit. Cutting first dropped every matching leaf ranked below the cut, so
|
||||
// `--root facts` came back empty whenever info leafs filled the top N.
|
||||
// limit <= 0 keeps everything.
|
||||
// RankAndFilter fuses the two hit lists, applies --root/--repo/--as-of, then
|
||||
// cuts to limit. Cutting first dropped every matching leaf ranked below the
|
||||
// cut, so `--root facts` came back empty whenever info leafs filled the top N.
|
||||
// limit <= 0 keeps everything. asOf empty skips interval filter (D24).
|
||||
func RankAndFilter(fts, vec []Hit, root, repo string, limit int) []Hit {
|
||||
return RankAndFilterAsOf(fts, vec, root, repo, "", limit)
|
||||
}
|
||||
|
||||
// RankAndFilterAsOf is RankAndFilter with D24 fact-interval filter.
|
||||
func RankAndFilterAsOf(fts, vec []Hit, root, repo, asOf string, limit int) []Hit {
|
||||
out := Hybrid(fts, vec, 0)
|
||||
if root != "" {
|
||||
out = FilterRoot(out, root)
|
||||
@@ -41,12 +50,30 @@ func RankAndFilter(fts, vec []Hit, root, repo string, limit int) []Hit {
|
||||
if repo != "" {
|
||||
out = FilterRepo(out, repo)
|
||||
}
|
||||
if asOf != "" {
|
||||
out = FilterAsOf(out, asOf)
|
||||
}
|
||||
if limit > 0 && len(out) > limit {
|
||||
out = out[:limit]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FilterAsOf keeps hits whose [valid_from, valid_to] covers asOf (D24).
|
||||
// Empty intervals stay (legacy leafs). Empty asOf keeps all.
|
||||
func FilterAsOf(hits []Hit, asOf string) []Hit {
|
||||
if asOf == "" {
|
||||
return hits
|
||||
}
|
||||
var out []Hit
|
||||
for _, h := range hits {
|
||||
if facts.ActiveAt(h.ValidFrom, h.ValidTo, asOf) {
|
||||
out = append(out, h)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Hybrid merges FTS and vector hits by reciprocal rank fusion.
|
||||
// limit <= 0 returns the full fused list.
|
||||
func Hybrid(fts, vec []Hit, limit int) []Hit {
|
||||
|
||||
@@ -55,7 +55,7 @@ func runSearch(args []string) int {
|
||||
}
|
||||
defer closeBrain()
|
||||
|
||||
hits, err := searchHits(query, root, repo, limit)
|
||||
hits, err := searchHits(query, root, repo, limit, opt.AsOf)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "search: %v\n", err)
|
||||
return 1
|
||||
@@ -85,6 +85,7 @@ func runSearch(args []string) int {
|
||||
out := Dict{
|
||||
{"query", query},
|
||||
{"root_filter", root},
|
||||
{"as_of", opt.AsOf},
|
||||
{"count", len(results)},
|
||||
{"results", resultsToDicts(results)},
|
||||
}
|
||||
@@ -96,13 +97,13 @@ func runSearch(args []string) int {
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
enc.SetEscapeHTML(false)
|
||||
return b2i(enc.Encode(toJSONOut(results, query, root, webOut)))
|
||||
return b2i(enc.Encode(toJSONOut(results, query, root, opt.AsOf, webOut)))
|
||||
}
|
||||
fmt.Print(toYAML(out, 0))
|
||||
return 0
|
||||
}
|
||||
|
||||
func searchHits(query, root, repo string, limit int) ([]Hit, error) {
|
||||
func searchHits(query, root, repo string, limit int, asOf string) ([]Hit, error) {
|
||||
emb, err := embedQuery(query)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("embed: %w", err)
|
||||
@@ -115,7 +116,7 @@ func searchHits(query, root, repo string, limit int) ([]Hit, error) {
|
||||
if vec, err = queryVector(emb, limit*3); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "vec: %v\n", err)
|
||||
}
|
||||
return rank.RankAndFilter(fts, vec, root, repo, limit), nil
|
||||
return rank.RankAndFilterAsOf(fts, vec, root, repo, asOf, limit), nil
|
||||
}
|
||||
|
||||
func attachHops(hits []Hit, n int) error {
|
||||
@@ -220,15 +221,35 @@ func rowsToHits(res *lbug.QueryResult) ([]Hit, error) {
|
||||
if len(vals) >= 6 {
|
||||
conf = fmt.Sprint(vals[5])
|
||||
}
|
||||
hits = append(hits, Hit{ID: id, Text: text, Root: root, Source: source, Score: score, Confidence: conf})
|
||||
vf, vt := "", ""
|
||||
if len(vals) >= 8 {
|
||||
vf = nullStr(vals[6])
|
||||
vt = nullStr(vals[7])
|
||||
}
|
||||
hits = append(hits, Hit{
|
||||
ID: id, Text: text, Root: root, Source: source, Score: score,
|
||||
Confidence: conf, ValidFrom: vf, ValidTo: vt,
|
||||
})
|
||||
}
|
||||
return hits, nil
|
||||
}
|
||||
|
||||
func nullStr(v any) string {
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
s := fmt.Sprint(v)
|
||||
if s == "<nil>" {
|
||||
return ""
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// JSON output types
|
||||
type jsonOut struct {
|
||||
Query string `json:"query"`
|
||||
RootFilter string `json:"root_filter"`
|
||||
AsOf string `json:"as_of,omitempty"`
|
||||
Count int `json:"count"`
|
||||
Results []jsonHit `json:"results"`
|
||||
Web *rank.SecondSource `json:"web,omitempty"`
|
||||
@@ -241,10 +262,12 @@ type jsonHit struct {
|
||||
Confidence string `json:"confidence,omitempty"`
|
||||
Score float64 `json:"score"`
|
||||
Snippet string `json:"snippet,omitempty"`
|
||||
ValidFrom string `json:"valid_from,omitempty"`
|
||||
ValidTo string `json:"valid_to,omitempty"`
|
||||
Hops []rank.HopNode `json:"hops,omitempty"`
|
||||
}
|
||||
|
||||
func toJSONOut(hits []Hit, query, rootFilter string, web *rank.SecondSource) *jsonOut {
|
||||
func toJSONOut(hits []Hit, query, rootFilter, asOf string, web *rank.SecondSource) *jsonOut {
|
||||
out := make([]jsonHit, len(hits))
|
||||
for i, h := range hits {
|
||||
out[i] = jsonHit{
|
||||
@@ -254,12 +277,15 @@ func toJSONOut(hits []Hit, query, rootFilter string, web *rank.SecondSource) *js
|
||||
Confidence: h.Confidence,
|
||||
Score: h.Score,
|
||||
Snippet: h.Snippet,
|
||||
ValidFrom: h.ValidFrom,
|
||||
ValidTo: h.ValidTo,
|
||||
Hops: h.Hops,
|
||||
}
|
||||
}
|
||||
return &jsonOut{
|
||||
Query: query,
|
||||
RootFilter: rootFilter,
|
||||
AsOf: asOf,
|
||||
Count: len(hits),
|
||||
Results: out,
|
||||
Web: web,
|
||||
@@ -278,6 +304,12 @@ func resultsToDicts(hits []Hit) []any {
|
||||
if h.Confidence != "" {
|
||||
d = append(d, KV{"confidence", h.Confidence})
|
||||
}
|
||||
if h.ValidFrom != "" {
|
||||
d = append(d, KV{"valid_from", h.ValidFrom})
|
||||
}
|
||||
if h.ValidTo != "" {
|
||||
d = append(d, KV{"valid_to", h.ValidTo})
|
||||
}
|
||||
if h.Snippet != "" {
|
||||
d = append(d, KV{"snippet", h.Snippet})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user