diff --git a/AGENTS.md b/AGENTS.md index 0dd5be5..039e101 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -81,6 +81,7 @@ bin/facts/audit ["self"|"facts"|"info"|"stale"] # 2-source + staleness gate bin/facts/crm [--dry-run] # proof person↔company/company↔project (ooCRM × corpus SoT) bin/kb/search "query" [--repo X] # deprecated wrapper → bin/brain/search.go bin/brain/search.go "query" [--root facts|info] # deduction search → YAML +bin/brain/search.go "query" --no-web # local graph only bin/brain/get.go [--body] bin/markdown/import.go [dir] # mistune leaves → YAML bin/git/import.go [REPO] [--json] [--limit N] # go-git history → commit leafs diff --git a/PLAN.md b/PLAN.md index 44a26d2..b448abc 100644 --- a/PLAN.md +++ b/PLAN.md @@ -40,7 +40,7 @@ detective method: **a fact needs ≥2 independent sources or it is | D14 | tooling style | `bin/{subject}/{method}.go` shebang (e.g. `bin/brain/search.go`). Shared code in `internal/`. One root `go.mod` + `go.work`. No `bin/*/main.go`, no nested modules. | | D15 | repo | Gitea [`eSlider/2dph`](https://git.produktor.io/eSlider/2dph) is origin + [issues](https://git.produktor.io/eSlider/2dph/issues). GitHub `eSlider/2dph` is the public clone (PRs + Actions CI). No direct `main` pushes. TDD → PR → CI green → merge. | | D16 | contradictions | ≥2 yes vs ≥2 no → unrelated sources conflict → hypothesis → `(not confirmed)`. Resolution (authority, staleness adjudication) = **v2**, tracked as open question. | -| D17 | assertion gate | Fact-check every *claim* (facts → info → live sources → web), not every edit. Missing graph ≠ “does not exist”. | +| D17 | assertion gate | Fact-check every *claim* (facts → info → live → web), not every edit. `bin/brain/search.go` adds a `web` block when there is no facts hit (`throttled`/`skipped`/`refused` ≠ absence). `--root` and `--no-web` stay local. Missing graph ≠ “does not exist”. | | D18 | reasoner | Pluggable OpenAI-compatible URL. RAM: Qwen3.5-9B. Quality: Bonsai-27B or Qwen3.6-27B. No official Qwen3.6-9B. | | D19 | git history | [go-git](https://github.com/go-git/go-git) via `bin/git/import.go`. No subprocess of the git binary. Conversion prints commit leafs; brain write is `bin/brain/index.go`. | diff --git a/README.md b/README.md index 8480efa..55f0e4f 100644 --- a/README.md +++ b/README.md @@ -85,9 +85,10 @@ fact; conflicting sources or a single source → `hypothesis` → `(not confirme ## Deduction search ```bash -bin/brain/search.go "Matrix federation over HTTPS" # facts → info → web-search +bin/brain/search.go "Matrix federation over HTTPS" # facts → info → web bin/brain/search.go "onlyoffice postgres" --root facts bin/brain/search.go "where is cs-lexicon" --json | yq '.' +bin/brain/search.go "upstream flag" --no-web # local graph only bin/brain/get.go --body # full chunk on demand bin/brain/stats.go # index health bin/brain/eval.go # recall@5 gate diff --git a/bin/brain/search.go b/bin/brain/search.go index 78ce932..3206ebe 100755 --- a/bin/brain/search.go +++ b/bin/brain/search.go @@ -3,7 +3,7 @@ // // bin/brain/search.go - deduction search over the 2dph brain. // -// ./bin/brain/search.go "query" [--root facts|info] [--repo P] [-n N] [--json] +// ./bin/brain/search.go "query" [--root facts|info] [--repo P] [-n N] [--json] [--no-web] // ./bin/brain/search.go serve [port] // ./bin/brain/search.go --list-model // diff --git a/bin/tools/test_published_docs.py b/bin/tools/test_published_docs.py index 345af9b..68b884f 100644 --- a/bin/tools/test_published_docs.py +++ b/bin/tools/test_published_docs.py @@ -59,6 +59,13 @@ class PublishedDocsTest(unittest.TestCase): self.assertNotIn("password", settings.lower()) self.assertIn("json", settings) + def test_readme_search_escalates_web(self) -> None: + text = (ROOT / "README.md").read_text() + self.assertIn("--no-web", text) + self.assertIn("D17", (ROOT / "PLAN.md").read_text()) + skill = (ROOT / "skills" / "brain" / "SKILL.md").read_text() + self.assertIn("`web` block", skill) + def test_docs_do_not_claim_hop_walks(self) -> None: paths = [ ROOT / "README.md", diff --git a/docs/design.md b/docs/design.md index ab5f284..5acb3fa 100644 --- a/docs/design.md +++ b/docs/design.md @@ -21,7 +21,8 @@ bin/brain/search.go "question" 1. facts root — confirmed answers only → return with evidence links 2. info root — supporting narrative → snippets, marked (not confirmed) 3. web-search — second independent source → upgrade hypothesis to confirmed - (`bin/web/search.go`; status `throttled` is not evidence of absence) + (`web` block from `bin/web/search.go` when no facts hit; status `throttled` + is not evidence of absence; `--no-web` / `--root` skip it) ``` `--hop` is not implemented yet (needs File/FROM_FILE edges). The flag is an diff --git a/internal/brain/http.go b/internal/brain/http.go index fa153bc..c65a5dc 100644 --- a/internal/brain/http.go +++ b/internal/brain/http.go @@ -7,6 +7,8 @@ import ( "context" "encoding/json" "fmt" + + "github.com/eSlider/2dph/internal/brain/rank" ) // Ready opens the Ladybug file for the life of the serve process. @@ -17,7 +19,7 @@ func Ready() error { // HTTP is the in-process API used by bin/brain/serve.go. type HTTP struct{} -func (HTTP) Search(_ context.Context, query string, limit int) ([]byte, error) { +func (HTTP) Search(ctx context.Context, query string, limit int) ([]byte, error) { hits, err := searchHits(query, "", "", limit) if err != nil { return nil, err @@ -31,10 +33,13 @@ func (HTTP) Search(_ context.Context, query string, limit int) ([]byte, error) { hits[i].Snippet = string(runes) } } + webOut := rank.Deduce(hits, query, "", false, func(q string) rank.SecondSource { + return lookupWeb(ctx, q) + }) var buf bytes.Buffer enc := json.NewEncoder(&buf) enc.SetEscapeHTML(false) - if err := enc.Encode(toJSONOut(hits, query, "")); err != nil { + if err := enc.Encode(toJSONOut(hits, query, "", webOut)); err != nil { return nil, err } return buf.Bytes(), nil diff --git a/internal/brain/rank/args.go b/internal/brain/rank/args.go index dd4970f..5c62247 100644 --- a/internal/brain/rank/args.go +++ b/internal/brain/rank/args.go @@ -6,7 +6,7 @@ import ( "strings" ) -const Usage = `usage: bin/brain/search.go "query" [--root facts|info] [--repo REPO] [-n N] [--json] +const Usage = `usage: bin/brain/search.go "query" [--root facts|info] [--repo REPO] [-n N] [--json] [--no-web] bin/brain/search.go serve [port] bin/brain/search.go --list-model` @@ -17,6 +17,7 @@ type Options struct { Limit int JSONOut bool ListModel bool + NoWeb bool } // ParseArgs reads flags. Unknown flags are an error: silently dropping them @@ -54,6 +55,8 @@ func ParseArgs(args []string) (Options, error) { return opt, fmt.Errorf("--hop is not implemented yet (needs File/FROM_FILE edges)") case "--json": opt.JSONOut = true + case "--no-web": + opt.NoWeb = true case "--list-model": opt.ListModel = true default: diff --git a/internal/brain/rank/escalate.go b/internal/brain/rank/escalate.go new file mode 100644 index 0000000..62b5b11 --- /dev/null +++ b/internal/brain/rank/escalate.go @@ -0,0 +1,43 @@ +package rank + +// SecondSource is the web-search block on a deduction answer. +// Kept apart from graph hits so "ours" and "not ours" stay visible. +type SecondSource struct { + Status string `json:"status"` + Note string `json:"note,omitempty"` + Cached bool `json:"cached,omitempty"` + Results []SecondSourceHit `json:"results,omitempty"` +} + +type SecondSourceHit struct { + Rank int `json:"rank"` + Title string `json:"title"` + URL string `json:"url"` + Snippet string `json:"snippet"` + Engine string `json:"engine"` +} + +type WebFn func(query string) SecondSource + +// ShouldEscalate is true when the default deduction path has no facts hit. +// `--root facts|info` is a single-root ask: do not mix in the web. +func ShouldEscalate(hits []Hit, rootFilter string) bool { + if rootFilter != "" { + return false + } + for _, h := range hits { + if h.Root == "facts" { + return false + } + } + return true +} + +// Deduce returns the second-source block, or nil when web must not run. +func Deduce(hits []Hit, query, rootFilter string, noWeb bool, web WebFn) *SecondSource { + if noWeb || web == nil || !ShouldEscalate(hits, rootFilter) { + return nil + } + out := web(query) + return &out +} diff --git a/internal/brain/rank/escalate_test.go b/internal/brain/rank/escalate_test.go new file mode 100644 index 0000000..c5dd168 --- /dev/null +++ b/internal/brain/rank/escalate_test.go @@ -0,0 +1,78 @@ +package rank + +import ( + "strings" + "testing" +) + +func TestShouldEscalateWhenNoFacts(t *testing.T) { + if !ShouldEscalate(nil, "") { + t.Fatal("empty local graph must escalate") + } + if !ShouldEscalate([]Hit{h("i", "info", "docs/a.md")}, "") { + t.Fatal("info-only must escalate (not confirmed)") + } +} + +func TestShouldNotEscalateWhenFactsConfirm(t *testing.T) { + hits := []Hit{h("f", "facts", "docker ps x compose"), h("i", "info", "docs/a.md")} + if ShouldEscalate(hits, "") { + t.Fatal("facts hit is already confirmed; do not mix web") + } +} + +func TestShouldNotEscalateWhenRootFilterSet(t *testing.T) { + if ShouldEscalate(nil, "facts") { + t.Fatal("--root facts must stay local") + } + if ShouldEscalate([]Hit{h("i", "info", "x")}, "info") { + t.Fatal("--root info must stay local") + } +} + +func TestDeduceCallsWebOnlyWhenEscalating(t *testing.T) { + called := 0 + web := func(q string) SecondSource { + called++ + if q != "LadybugDB" { + t.Fatalf("query = %q", q) + } + return SecondSource{Status: "ok", Results: []SecondSourceHit{{Title: "t", URL: "http://example.com"}}} + } + got := Deduce([]Hit{h("i", "info", "x")}, "LadybugDB", "", false, web) + if called != 1 || got == nil || got.Status != "ok" { + t.Fatalf("got %+v called=%d", got, called) + } +} + +func TestDeduceNilWhenFactsOrNoWeb(t *testing.T) { + web := func(string) SecondSource { + t.Fatal("web must not run") + return SecondSource{} + } + if Deduce([]Hit{h("f", "facts", "x")}, "q", "", false, web) != nil { + t.Fatal("facts") + } + if Deduce([]Hit{h("i", "info", "x")}, "q", "", true, web) != nil { + t.Fatal("--no-web") + } + if Deduce(nil, "q", "facts", false, web) != nil { + t.Fatal("--root facts") + } + if Deduce(nil, "q", "", false, nil) != nil { + t.Fatal("nil web fn") + } +} + +func TestParseNoWeb(t *testing.T) { + opt, err := ParseArgs([]string{"query", "--no-web", "--json"}) + if err != nil || !opt.NoWeb || !opt.JSONOut || opt.Query != "query" { + t.Fatalf("got %+v err=%v", opt, err) + } +} + +func TestUsageNamesNoWeb(t *testing.T) { + if !strings.Contains(Usage, "--no-web") { + t.Fatalf("usage must name --no-web, got:\n%s", Usage) + } +} diff --git a/internal/brain/search.go b/internal/brain/search.go index c0d9b38..2f739bf 100644 --- a/internal/brain/search.go +++ b/internal/brain/search.go @@ -68,18 +68,25 @@ func runSearch(args []string) int { } } + webOut := rank.Deduce(results, query, root, opt.NoWeb, func(q string) rank.SecondSource { + return lookupWeb(context.Background(), q) + }) + out := Dict{ {"query", query}, {"root_filter", root}, {"count", len(results)}, {"results", resultsToDicts(results)}, } + if webOut != nil { + out = append(out, KV{"web", secondToDict(*webOut)}) + } if jsonOut { enc := json.NewEncoder(os.Stdout) enc.SetIndent("", " ") enc.SetEscapeHTML(false) - return b2i(enc.Encode(toJSONOut(results, query, root))) + return b2i(enc.Encode(toJSONOut(results, query, root, webOut))) } fmt.Print(toYAML(out, 0)) return 0 @@ -168,10 +175,11 @@ func rowsToHits(res *lbug.QueryResult) ([]Hit, error) { // JSON output types type jsonOut struct { - Query string `json:"query"` - RootFilter string `json:"root_filter"` - Count int `json:"count"` - Results []jsonHit `json:"results"` + Query string `json:"query"` + RootFilter string `json:"root_filter"` + Count int `json:"count"` + Results []jsonHit `json:"results"` + Web *rank.SecondSource `json:"web,omitempty"` } type jsonHit struct { @@ -182,7 +190,7 @@ type jsonHit struct { Snippet string `json:"snippet,omitempty"` } -func toJSONOut(hits []Hit, query, rootFilter string) *jsonOut { +func toJSONOut(hits []Hit, query, rootFilter string, web *rank.SecondSource) *jsonOut { out := make([]jsonHit, len(hits)) for i, h := range hits { out[i] = jsonHit{ @@ -198,6 +206,7 @@ func toJSONOut(hits []Hit, query, rootFilter string) *jsonOut { RootFilter: rootFilter, Count: len(hits), Results: out, + Web: web, } } diff --git a/internal/brain/web.go b/internal/brain/web.go new file mode 100644 index 0000000..a777259 --- /dev/null +++ b/internal/brain/web.go @@ -0,0 +1,56 @@ +package brain + +import ( + "context" + + "github.com/eSlider/2dph/internal/brain/rank" + "github.com/eSlider/2dph/internal/websearch" +) + +func lookupWeb(ctx context.Context, query string) rank.SecondSource { + o := websearch.Lookup(ctx, query, websearch.LookupOpt{Limit: 5}) + return toSecond(o) +} + +func toSecond(o websearch.Output) rank.SecondSource { + hits := make([]rank.SecondSourceHit, 0, len(o.Results)) + for _, h := range o.Results { + hits = append(hits, rank.SecondSourceHit{ + Rank: h.Rank, + Title: h.Title, + URL: h.URL, + Snippet: h.Snippet, + Engine: h.Engine, + }) + } + return rank.SecondSource{ + Status: o.Status, + Note: o.Note, + Cached: o.Cached, + Results: hits, + } +} + +func secondToDict(w rank.SecondSource) Dict { + d := Dict{ + {"status", w.Status}, + } + if w.Note != "" { + d = append(d, KV{"note", w.Note}) + } + if w.Cached { + d = append(d, KV{"cached", true}) + } + rows := make([]any, 0, len(w.Results)) + for _, h := range w.Results { + rows = append(rows, Dict{ + {"rank", h.Rank}, + {"title", h.Title}, + {"url", h.URL}, + {"snippet", h.Snippet}, + {"engine", h.Engine}, + }) + } + d = append(d, KV{"results", rows}) + return d +} diff --git a/internal/websearch/lookup.go b/internal/websearch/lookup.go new file mode 100644 index 0000000..7d36639 --- /dev/null +++ b/internal/websearch/lookup.go @@ -0,0 +1,122 @@ +package websearch + +import ( + "context" + "fmt" + "net/http" + "os" + "time" + + "golang.org/x/sys/unix" +) + +const ( + StatusSkipped = "skipped" + StatusRefused = "refused" +) + +type LookupOpt struct { + Limit int + Timeout time.Duration + EnvPath string + CachePath string + Client *http.Client + Now func() float64 + Sleep func(context.Context, time.Duration) error +} + +func Lookup(ctx context.Context, query string, opt LookupOpt) Output { + if ctx == nil { + ctx = context.Background() + } + if opt.Limit <= 0 { + opt.Limit = DefaultLimit + } + if opt.Timeout <= 0 { + opt.Timeout = 25 * time.Second + } + nowFn := opt.Now + if nowFn == nil { + nowFn = func() float64 { return float64(time.Now().Unix()) } + } + sleepFn := opt.Sleep + if sleepFn == nil { + sleepFn = func(ctx context.Context, d time.Duration) error { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-t.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } + } + } + + if reason := PHIReason(query); reason != "" { + return Output{Query: query, Status: StatusRefused, Note: reason} + } + + cachePath := opt.CachePath + if cachePath == "" { + cachePath = os.Getenv("BRAIN_SEARCH_CACHE") + } + if cachePath == "" { + cachePath = os.Getenv("HOME") + "/.cache/brain/web-search.sqlite" + } + cache, err := OpenCache(cachePath) + if err != nil { + return Output{Query: query, Status: StatusSkipped, Note: "cache: " + err.Error()} + } + defer cache.Close() + + key := CacheKey(query, nil) + now := nowFn() + if cached, err := cache.Get(key, CacheTTL, now); err == nil && cached != nil { + out := Project(*cached, opt.Limit, DefaultSnippetChars) + out.Cached = true + return out + } + + envPath := opt.EnvPath + if envPath == "" { + envPath = os.Getenv("BRAIN_SEARCH_ENV") + } + if envPath == "" { + envPath = os.Getenv("HOME") + "/.config/brain/search.env" + } + conf, err := LoadConfig(envPath) + if err != nil { + return Output{Query: query, Status: StatusSkipped, Note: "no BRAIN_SEARCH_URL; second source not consulted"} + } + + lock, err := os.OpenFile(cachePath+".lock", os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return Output{Query: query, Status: StatusSkipped, Note: "lock: " + err.Error()} + } + defer lock.Close() + if err := unix.Flock(int(lock.Fd()), unix.LOCK_EX); err != nil { + return Output{Query: query, Status: StatusSkipped, Note: "lock: " + err.Error()} + } + defer unix.Flock(int(lock.Fd()), unix.LOCK_UN) + + last, err := cache.LastCall() + if err != nil { + return Output{Query: query, Status: StatusSkipped, Note: "cache: " + err.Error()} + } + if delay := WaitFor(last, nowFn(), MinInterval); delay > 0 { + if err := sleepFn(ctx, time.Duration(delay*float64(time.Second))); err != nil { + return Output{Query: query, Status: StatusSkipped, Note: "cancelled"} + } + } + _ = cache.MarkCall(nowFn()) + + payload, err := Fetch(opt.Client, conf, query, nil, opt.Timeout) + if err != nil { + return Output{Query: query, Status: StatusThrottled, Note: fmt.Sprintf("request failed: %v", err)} + } + if Classify(payload) == StatusOK { + _ = cache.Put(key, payload, nowFn()) + } + return Project(payload, opt.Limit, DefaultSnippetChars) +} diff --git a/internal/websearch/lookup_test.go b/internal/websearch/lookup_test.go new file mode 100644 index 0000000..034bf3d --- /dev/null +++ b/internal/websearch/lookup_test.go @@ -0,0 +1,97 @@ +package websearch + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" +) + +func TestLookupRefusesPIIWithoutFetch(t *testing.T) { + hits := 0 + srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + hits++ + })) + defer srv.Close() + out := Lookup(context.Background(), "Personalnummer 12", LookupOpt{ + EnvPath: writeEnv(t, srv.URL), + CachePath: filepath.Join(t.TempDir(), "c.sqlite"), + Client: srv.Client(), + Sleep: func(context.Context, time.Duration) error { return nil }, + }) + if out.Status != StatusRefused { + t.Fatalf("status = %s", out.Status) + } + if hits != 0 { + t.Fatal("PII query left the host") + } +} + +func TestLookupSkipsWhenNoConfig(t *testing.T) { + out := Lookup(context.Background(), "LadybugDB", LookupOpt{ + EnvPath: filepath.Join(t.TempDir(), "missing.env"), + CachePath: filepath.Join(t.TempDir(), "c.sqlite"), + Sleep: func(context.Context, time.Duration) error { return nil }, + }) + if out.Status != StatusSkipped { + t.Fatalf("status = %s", out.Status) + } +} + +func TestLookupFetchesOnceAndCaches(t *testing.T) { + hits := 0 + payload := Payload{Query: "x", Results: []RawHit{{Title: "t", URL: "http://example.com", Content: "c", Engine: "bing"}}} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits++ + json.NewEncoder(w).Encode(payload) + })) + defer srv.Close() + opt := LookupOpt{ + EnvPath: writeEnv(t, srv.URL), + CachePath: filepath.Join(t.TempDir(), "c.sqlite"), + Client: srv.Client(), + Now: func() float64 { return 1_000 }, + Sleep: func(context.Context, time.Duration) error { return nil }, + } + a := Lookup(context.Background(), "LadybugDB", opt) + b := Lookup(context.Background(), "LadybugDB", opt) + if a.Status != StatusOK || b.Status != StatusOK { + t.Fatalf("a=%s b=%s", a.Status, b.Status) + } + if hits != 1 { + t.Fatalf("hits = %d, want 1 (second from cache)", hits) + } + if !b.Cached { + t.Fatal("second lookup not cached") + } +} + +func TestLookupEmptyIsThrottled(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"query":"x","results":[]}`)) + })) + defer srv.Close() + out := Lookup(context.Background(), "LadybugDB", LookupOpt{ + EnvPath: writeEnv(t, srv.URL), + CachePath: filepath.Join(t.TempDir(), "c.sqlite"), + Client: srv.Client(), + Now: func() float64 { return 1_000 }, + Sleep: func(context.Context, time.Duration) error { return nil }, + }) + if out.Status != StatusThrottled { + t.Fatalf("status = %s", out.Status) + } +} + +func writeEnv(t *testing.T, url string) string { + t.Helper() + p := filepath.Join(t.TempDir(), "search.env") + if err := os.WriteFile(p, []byte("BRAIN_SEARCH_URL="+url+"\n"), 0o600); err != nil { + t.Fatal(err) + } + return p +} diff --git a/skills/brain/SKILL.md b/skills/brain/SKILL.md index 0944f3b..cd33772 100644 --- a/skills/brain/SKILL.md +++ b/skills/brain/SKILL.md @@ -39,8 +39,9 @@ are not wired yet); do not treat it as a graph walk. - Search before you read. Never grep a repo for a concept the graph covers. - `--root facts` returns only confirmed evidence-linked answers. Default shows facts first, then info leafs clearly marked `(not confirmed)`. +- If there is no facts hit, `bin/brain/search.go` consults SearXNG and adds a + `web` block (kept apart from graph hits). `throttled` / `skipped` / `refused` + are not evidence of absence. `--root facts|info` and `--no-web` skip the web. - If recall looks wrong, run `bin/brain/eval.go`; it gates control questions and should stay at or above 95% recall@5. -- Escalate to `web-search` (the `web-search` skill) as the independent second - source when both local roots cannot confirm; never report an unconfirmed - single-source local answer as fact. \ No newline at end of file +- Never report an unconfirmed single-source local answer as fact. \ No newline at end of file