feat: Go SearXNG client; throttled is not absence

AGENTS.md

@@ -81,7 +81,6 @@ bin/facts/audit ["self"|"facts"|"info"|"stale"]   # 2-source + staleness gate
 bin/facts/crm [--dry-run]                         # proof person↔company/company↔project (ooCRM × corpus SoT)
 bin/kb/search "query" [--repo X]                  # deprecated wrapper → bin/brain/search.go
 bin/brain/search.go "query" [--root facts|info]   # deduction search → YAML
-bin/brain/search.go "query" --no-web              # local graph only
 bin/brain/get.go <id> [--body]
 bin/markdown/import.go [dir]                      # mistune leaves → YAML
 bin/git/import.go [REPO] [--json] [--limit N]     # go-git history → commit leafs

PLAN.md

@@ -40,7 +40,7 @@ detective method: **a fact needs ≥2 independent sources or it is
 | D14 | tooling style | `bin/{subject}/{method}.go` shebang (e.g. `bin/brain/search.go`). Shared code in `internal/`. One root `go.mod` + `go.work`. No `bin/*/main.go`, no nested modules. |
 | D15 | repo | Gitea [`eSlider/2dph`](https://git.produktor.io/eSlider/2dph) is origin + [issues](https://git.produktor.io/eSlider/2dph/issues). GitHub `eSlider/2dph` is the public clone (PRs + Actions CI). No direct `main` pushes. TDD → PR → CI green → merge. |
 | D16 | contradictions | ≥2 yes vs ≥2 no → unrelated sources conflict → hypothesis → `(not confirmed)`. Resolution (authority, staleness adjudication) = **v2**, tracked as open question. |
-| D17 | assertion gate | Fact-check every *claim* (facts → info → live → web), not every edit. `bin/brain/search.go` adds a `web` block when there is no facts hit (`throttled`/`skipped`/`refused` ≠ absence). `--root` and `--no-web` stay local. Missing graph ≠ “does not exist”. |
+| D17 | assertion gate | Fact-check every *claim* (facts → info → live sources → web), not every edit. Missing graph ≠ “does not exist”. |
 | D18 | reasoner | Pluggable OpenAI-compatible URL. RAM: Qwen3.5-9B. Quality: Bonsai-27B or Qwen3.6-27B. No official Qwen3.6-9B. |
 | D19 | git history | [go-git](https://github.com/go-git/go-git) via `bin/git/import.go`. No subprocess of the git binary. Conversion prints commit leafs; brain write is `bin/brain/index.go`. |
 

README.md

@@ -85,10 +85,9 @@ fact; conflicting sources or a single source → `hypothesis` → `(not confirme
 ## Deduction search
 
 ```bash
-bin/brain/search.go "Matrix federation over HTTPS"   # facts → info → web
+bin/brain/search.go "Matrix federation over HTTPS"   # facts → info → web-search
 bin/brain/search.go "onlyoffice postgres" --root facts
 bin/brain/search.go "where is cs-lexicon" --json | yq '.'
-bin/brain/search.go "upstream flag" --no-web         # local graph only
 bin/brain/get.go <id> --body                         # full chunk on demand
 bin/brain/stats.go                                   # index health
 bin/brain/eval.go                                    # recall@5 gate

bin/brain/search.go

@@ -3,7 +3,7 @@
 //
 // bin/brain/search.go - deduction search over the 2dph brain.
 //
-//	./bin/brain/search.go "query" [--root facts|info] [--repo P] [-n N] [--json] [--no-web]
+//	./bin/brain/search.go "query" [--root facts|info] [--repo P] [-n N] [--json]
 //	./bin/brain/search.go serve [port]
 //	./bin/brain/search.go --list-model
 //

bin/tools/test_published_docs.py

@@ -59,13 +59,6 @@ class PublishedDocsTest(unittest.TestCase):
         self.assertNotIn("password", settings.lower())
         self.assertIn("json", settings)
 
-    def test_readme_search_escalates_web(self) -> None:
-        text = (ROOT / "README.md").read_text()
-        self.assertIn("--no-web", text)
-        self.assertIn("D17", (ROOT / "PLAN.md").read_text())
-        skill = (ROOT / "skills" / "brain" / "SKILL.md").read_text()
-        self.assertIn("`web` block", skill)
-
     def test_docs_do_not_claim_hop_walks(self) -> None:
         paths = [
             ROOT / "README.md",

docs/design.md

@@ -21,8 +21,7 @@ bin/brain/search.go "question"
   1. facts root   — confirmed answers only   → return with evidence links
   2. info  root   — supporting narrative     → snippets, marked (not confirmed)
   3. web-search   — second independent source → upgrade hypothesis to confirmed
-     (`web` block from `bin/web/search.go` when no facts hit; status `throttled`
+     (`bin/web/search.go`; status `throttled` is not evidence of absence)
-     is not evidence of absence; `--no-web` / `--root` skip it)
 ```
 
 `--hop` is not implemented yet (needs File/FROM_FILE edges). The flag is an

internal/brain/http.go

@@ -7,8 +7,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-
-	"github.com/eSlider/2dph/internal/brain/rank"
 )
 
 // Ready opens the Ladybug file for the life of the serve process.
@@ -19,7 +17,7 @@ func Ready() error {
 // HTTP is the in-process API used by bin/brain/serve.go.
 type HTTP struct{}
 
-func (HTTP) Search(ctx context.Context, query string, limit int) ([]byte, error) {
+func (HTTP) Search(_ context.Context, query string, limit int) ([]byte, error) {
 	hits, err := searchHits(query, "", "", limit)
 	if err != nil {
 		return nil, err
@@ -33,13 +31,10 @@ func (HTTP) Search(ctx context.Context, query string, limit int) ([]byte, error)
 			hits[i].Snippet = string(runes)
 		}
 	}
-	webOut := rank.Deduce(hits, query, "", false, func(q string) rank.SecondSource {
-		return lookupWeb(ctx, q)
-	})
 	var buf bytes.Buffer
 	enc := json.NewEncoder(&buf)
 	enc.SetEscapeHTML(false)
-	if err := enc.Encode(toJSONOut(hits, query, "", webOut)); err != nil {
+	if err := enc.Encode(toJSONOut(hits, query, "")); err != nil {
 		return nil, err
 	}
 	return buf.Bytes(), nil

internal/brain/rank/args.go

@@ -6,7 +6,7 @@ import (
 	"strings"
 )
 
-const Usage = `usage: bin/brain/search.go "query" [--root facts|info] [--repo REPO] [-n N] [--json] [--no-web]
+const Usage = `usage: bin/brain/search.go "query" [--root facts|info] [--repo REPO] [-n N] [--json]
        bin/brain/search.go serve [port]
        bin/brain/search.go --list-model`
 
@@ -17,7 +17,6 @@ type Options struct {
 	Limit     int
 	JSONOut   bool
 	ListModel bool
-	NoWeb     bool
 }
 
 // ParseArgs reads flags. Unknown flags are an error: silently dropping them
@@ -55,8 +54,6 @@ func ParseArgs(args []string) (Options, error) {
 			return opt, fmt.Errorf("--hop is not implemented yet (needs File/FROM_FILE edges)")
 		case "--json":
 			opt.JSONOut = true
-		case "--no-web":
-			opt.NoWeb = true
 		case "--list-model":
 			opt.ListModel = true
 		default:

internal/brain/rank/escalate.go

@@ -1,43 +0,0 @@
-package rank
-
-// SecondSource is the web-search block on a deduction answer.
-// Kept apart from graph hits so "ours" and "not ours" stay visible.
-type SecondSource struct {
-	Status  string            `json:"status"`
-	Note    string            `json:"note,omitempty"`
-	Cached  bool              `json:"cached,omitempty"`
-	Results []SecondSourceHit `json:"results,omitempty"`
-}
-
-type SecondSourceHit struct {
-	Rank    int    `json:"rank"`
-	Title   string `json:"title"`
-	URL     string `json:"url"`
-	Snippet string `json:"snippet"`
-	Engine  string `json:"engine"`
-}
-
-type WebFn func(query string) SecondSource
-
-// ShouldEscalate is true when the default deduction path has no facts hit.
-// `--root facts|info` is a single-root ask: do not mix in the web.
-func ShouldEscalate(hits []Hit, rootFilter string) bool {
-	if rootFilter != "" {
-		return false
-	}
-	for _, h := range hits {
-		if h.Root == "facts" {
-			return false
-		}
-	}
-	return true
-}
-
-// Deduce returns the second-source block, or nil when web must not run.
-func Deduce(hits []Hit, query, rootFilter string, noWeb bool, web WebFn) *SecondSource {
-	if noWeb || web == nil || !ShouldEscalate(hits, rootFilter) {
-		return nil
-	}
-	out := web(query)
-	return &out
-}

internal/brain/rank/escalate_test.go

@@ -1,78 +0,0 @@
-package rank
-
-import (
-	"strings"
-	"testing"
-)
-
-func TestShouldEscalateWhenNoFacts(t *testing.T) {
-	if !ShouldEscalate(nil, "") {
-		t.Fatal("empty local graph must escalate")
-	}
-	if !ShouldEscalate([]Hit{h("i", "info", "docs/a.md")}, "") {
-		t.Fatal("info-only must escalate (not confirmed)")
-	}
-}
-
-func TestShouldNotEscalateWhenFactsConfirm(t *testing.T) {
-	hits := []Hit{h("f", "facts", "docker ps x compose"), h("i", "info", "docs/a.md")}
-	if ShouldEscalate(hits, "") {
-		t.Fatal("facts hit is already confirmed; do not mix web")
-	}
-}
-
-func TestShouldNotEscalateWhenRootFilterSet(t *testing.T) {
-	if ShouldEscalate(nil, "facts") {
-		t.Fatal("--root facts must stay local")
-	}
-	if ShouldEscalate([]Hit{h("i", "info", "x")}, "info") {
-		t.Fatal("--root info must stay local")
-	}
-}
-
-func TestDeduceCallsWebOnlyWhenEscalating(t *testing.T) {
-	called := 0
-	web := func(q string) SecondSource {
-		called++
-		if q != "LadybugDB" {
-			t.Fatalf("query = %q", q)
-		}
-		return SecondSource{Status: "ok", Results: []SecondSourceHit{{Title: "t", URL: "http://example.com"}}}
-	}
-	got := Deduce([]Hit{h("i", "info", "x")}, "LadybugDB", "", false, web)
-	if called != 1 || got == nil || got.Status != "ok" {
-		t.Fatalf("got %+v called=%d", got, called)
-	}
-}
-
-func TestDeduceNilWhenFactsOrNoWeb(t *testing.T) {
-	web := func(string) SecondSource {
-		t.Fatal("web must not run")
-		return SecondSource{}
-	}
-	if Deduce([]Hit{h("f", "facts", "x")}, "q", "", false, web) != nil {
-		t.Fatal("facts")
-	}
-	if Deduce([]Hit{h("i", "info", "x")}, "q", "", true, web) != nil {
-		t.Fatal("--no-web")
-	}
-	if Deduce(nil, "q", "facts", false, web) != nil {
-		t.Fatal("--root facts")
-	}
-	if Deduce(nil, "q", "", false, nil) != nil {
-		t.Fatal("nil web fn")
-	}
-}
-
-func TestParseNoWeb(t *testing.T) {
-	opt, err := ParseArgs([]string{"query", "--no-web", "--json"})
-	if err != nil || !opt.NoWeb || !opt.JSONOut || opt.Query != "query" {
-		t.Fatalf("got %+v err=%v", opt, err)
-	}
-}
-
-func TestUsageNamesNoWeb(t *testing.T) {
-	if !strings.Contains(Usage, "--no-web") {
-		t.Fatalf("usage must name --no-web, got:\n%s", Usage)
-	}
-}

internal/brain/search.go

@@ -68,25 +68,18 @@ func runSearch(args []string) int {
 		}
 	}
 
-	webOut := rank.Deduce(results, query, root, opt.NoWeb, func(q string) rank.SecondSource {
-		return lookupWeb(context.Background(), q)
-	})
-
 	out := Dict{
 		{"query", query},
 		{"root_filter", root},
 		{"count", len(results)},
 		{"results", resultsToDicts(results)},
 	}
-	if webOut != nil {
-		out = append(out, KV{"web", secondToDict(*webOut)})
-	}
 
 	if jsonOut {
 		enc := json.NewEncoder(os.Stdout)
 		enc.SetIndent("", "  ")
 		enc.SetEscapeHTML(false)
-		return b2i(enc.Encode(toJSONOut(results, query, root, webOut)))
+		return b2i(enc.Encode(toJSONOut(results, query, root)))
 	}
 	fmt.Print(toYAML(out, 0))
 	return 0
@@ -175,11 +168,10 @@ func rowsToHits(res *lbug.QueryResult) ([]Hit, error) {
 
 // JSON output types
 type jsonOut struct {
-	Query      string             `json:"query"`
+	Query      string    `json:"query"`
-	RootFilter string             `json:"root_filter"`
+	RootFilter string    `json:"root_filter"`
-	Count      int                `json:"count"`
+	Count      int       `json:"count"`
-	Results    []jsonHit          `json:"results"`
+	Results    []jsonHit `json:"results"`
-	Web        *rank.SecondSource `json:"web,omitempty"`
 }
 
 type jsonHit struct {
@@ -190,7 +182,7 @@ type jsonHit struct {
 	Snippet string  `json:"snippet,omitempty"`
 }
 
-func toJSONOut(hits []Hit, query, rootFilter string, web *rank.SecondSource) *jsonOut {
+func toJSONOut(hits []Hit, query, rootFilter string) *jsonOut {
 	out := make([]jsonHit, len(hits))
 	for i, h := range hits {
 		out[i] = jsonHit{
@@ -206,7 +198,6 @@ func toJSONOut(hits []Hit, query, rootFilter string, web *rank.SecondSource) *js
 		RootFilter: rootFilter,
 		Count:      len(hits),
 		Results:    out,
-		Web:        web,
 	}
 }
 

internal/brain/web.go

@@ -1,56 +0,0 @@
-package brain
-
-import (
-	"context"
-
-	"github.com/eSlider/2dph/internal/brain/rank"
-	"github.com/eSlider/2dph/internal/websearch"
-)
-
-func lookupWeb(ctx context.Context, query string) rank.SecondSource {
-	o := websearch.Lookup(ctx, query, websearch.LookupOpt{Limit: 5})
-	return toSecond(o)
-}
-
-func toSecond(o websearch.Output) rank.SecondSource {
-	hits := make([]rank.SecondSourceHit, 0, len(o.Results))
-	for _, h := range o.Results {
-		hits = append(hits, rank.SecondSourceHit{
-			Rank:    h.Rank,
-			Title:   h.Title,
-			URL:     h.URL,
-			Snippet: h.Snippet,
-			Engine:  h.Engine,
-		})
-	}
-	return rank.SecondSource{
-		Status:  o.Status,
-		Note:    o.Note,
-		Cached:  o.Cached,
-		Results: hits,
-	}
-}
-
-func secondToDict(w rank.SecondSource) Dict {
-	d := Dict{
-		{"status", w.Status},
-	}
-	if w.Note != "" {
-		d = append(d, KV{"note", w.Note})
-	}
-	if w.Cached {
-		d = append(d, KV{"cached", true})
-	}
-	rows := make([]any, 0, len(w.Results))
-	for _, h := range w.Results {
-		rows = append(rows, Dict{
-			{"rank", h.Rank},
-			{"title", h.Title},
-			{"url", h.URL},
-			{"snippet", h.Snippet},
-			{"engine", h.Engine},
-		})
-	}
-	d = append(d, KV{"results", rows})
-	return d
-}

internal/websearch/lookup.go

@@ -1,122 +0,0 @@
-package websearch
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"os"
-	"time"
-
-	"golang.org/x/sys/unix"
-)
-
-const (
-	StatusSkipped = "skipped"
-	StatusRefused = "refused"
-)
-
-type LookupOpt struct {
-	Limit     int
-	Timeout   time.Duration
-	EnvPath   string
-	CachePath string
-	Client    *http.Client
-	Now       func() float64
-	Sleep     func(context.Context, time.Duration) error
-}
-
-func Lookup(ctx context.Context, query string, opt LookupOpt) Output {
-	if ctx == nil {
-		ctx = context.Background()
-	}
-	if opt.Limit <= 0 {
-		opt.Limit = DefaultLimit
-	}
-	if opt.Timeout <= 0 {
-		opt.Timeout = 25 * time.Second
-	}
-	nowFn := opt.Now
-	if nowFn == nil {
-		nowFn = func() float64 { return float64(time.Now().Unix()) }
-	}
-	sleepFn := opt.Sleep
-	if sleepFn == nil {
-		sleepFn = func(ctx context.Context, d time.Duration) error {
-			t := time.NewTimer(d)
-			defer t.Stop()
-			select {
-			case <-t.C:
-				return nil
-			case <-ctx.Done():
-				return ctx.Err()
-			}
-		}
-	}
-
-	if reason := PHIReason(query); reason != "" {
-		return Output{Query: query, Status: StatusRefused, Note: reason}
-	}
-
-	cachePath := opt.CachePath
-	if cachePath == "" {
-		cachePath = os.Getenv("BRAIN_SEARCH_CACHE")
-	}
-	if cachePath == "" {
-		cachePath = os.Getenv("HOME") + "/.cache/brain/web-search.sqlite"
-	}
-	cache, err := OpenCache(cachePath)
-	if err != nil {
-		return Output{Query: query, Status: StatusSkipped, Note: "cache: " + err.Error()}
-	}
-	defer cache.Close()
-
-	key := CacheKey(query, nil)
-	now := nowFn()
-	if cached, err := cache.Get(key, CacheTTL, now); err == nil && cached != nil {
-		out := Project(*cached, opt.Limit, DefaultSnippetChars)
-		out.Cached = true
-		return out
-	}
-
-	envPath := opt.EnvPath
-	if envPath == "" {
-		envPath = os.Getenv("BRAIN_SEARCH_ENV")
-	}
-	if envPath == "" {
-		envPath = os.Getenv("HOME") + "/.config/brain/search.env"
-	}
-	conf, err := LoadConfig(envPath)
-	if err != nil {
-		return Output{Query: query, Status: StatusSkipped, Note: "no BRAIN_SEARCH_URL; second source not consulted"}
-	}
-
-	lock, err := os.OpenFile(cachePath+".lock", os.O_CREATE|os.O_RDWR, 0o600)
-	if err != nil {
-		return Output{Query: query, Status: StatusSkipped, Note: "lock: " + err.Error()}
-	}
-	defer lock.Close()
-	if err := unix.Flock(int(lock.Fd()), unix.LOCK_EX); err != nil {
-		return Output{Query: query, Status: StatusSkipped, Note: "lock: " + err.Error()}
-	}
-	defer unix.Flock(int(lock.Fd()), unix.LOCK_UN)
-
-	last, err := cache.LastCall()
-	if err != nil {
-		return Output{Query: query, Status: StatusSkipped, Note: "cache: " + err.Error()}
-	}
-	if delay := WaitFor(last, nowFn(), MinInterval); delay > 0 {
-		if err := sleepFn(ctx, time.Duration(delay*float64(time.Second))); err != nil {
-			return Output{Query: query, Status: StatusSkipped, Note: "cancelled"}
-		}
-	}
-	_ = cache.MarkCall(nowFn())
-
-	payload, err := Fetch(opt.Client, conf, query, nil, opt.Timeout)
-	if err != nil {
-		return Output{Query: query, Status: StatusThrottled, Note: fmt.Sprintf("request failed: %v", err)}
-	}
-	if Classify(payload) == StatusOK {
-		_ = cache.Put(key, payload, nowFn())
-	}
-	return Project(payload, opt.Limit, DefaultSnippetChars)
-}

internal/websearch/lookup_test.go

@@ -1,97 +0,0 @@
-package websearch
-
-import (
-	"context"
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"path/filepath"
-	"testing"
-	"time"
-)
-
-func TestLookupRefusesPIIWithoutFetch(t *testing.T) {
-	hits := 0
-	srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
-		hits++
-	}))
-	defer srv.Close()
-	out := Lookup(context.Background(), "Personalnummer 12", LookupOpt{
-		EnvPath: writeEnv(t, srv.URL),
-		CachePath: filepath.Join(t.TempDir(), "c.sqlite"),
-		Client:    srv.Client(),
-		Sleep:     func(context.Context, time.Duration) error { return nil },
-	})
-	if out.Status != StatusRefused {
-		t.Fatalf("status = %s", out.Status)
-	}
-	if hits != 0 {
-		t.Fatal("PII query left the host")
-	}
-}
-
-func TestLookupSkipsWhenNoConfig(t *testing.T) {
-	out := Lookup(context.Background(), "LadybugDB", LookupOpt{
-		EnvPath:   filepath.Join(t.TempDir(), "missing.env"),
-		CachePath: filepath.Join(t.TempDir(), "c.sqlite"),
-		Sleep:     func(context.Context, time.Duration) error { return nil },
-	})
-	if out.Status != StatusSkipped {
-		t.Fatalf("status = %s", out.Status)
-	}
-}
-
-func TestLookupFetchesOnceAndCaches(t *testing.T) {
-	hits := 0
-	payload := Payload{Query: "x", Results: []RawHit{{Title: "t", URL: "http://example.com", Content: "c", Engine: "bing"}}}
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		hits++
-		json.NewEncoder(w).Encode(payload)
-	}))
-	defer srv.Close()
-	opt := LookupOpt{
-		EnvPath:   writeEnv(t, srv.URL),
-		CachePath: filepath.Join(t.TempDir(), "c.sqlite"),
-		Client:    srv.Client(),
-		Now:       func() float64 { return 1_000 },
-		Sleep:     func(context.Context, time.Duration) error { return nil },
-	}
-	a := Lookup(context.Background(), "LadybugDB", opt)
-	b := Lookup(context.Background(), "LadybugDB", opt)
-	if a.Status != StatusOK || b.Status != StatusOK {
-		t.Fatalf("a=%s b=%s", a.Status, b.Status)
-	}
-	if hits != 1 {
-		t.Fatalf("hits = %d, want 1 (second from cache)", hits)
-	}
-	if !b.Cached {
-		t.Fatal("second lookup not cached")
-	}
-}
-
-func TestLookupEmptyIsThrottled(t *testing.T) {
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Write([]byte(`{"query":"x","results":[]}`))
-	}))
-	defer srv.Close()
-	out := Lookup(context.Background(), "LadybugDB", LookupOpt{
-		EnvPath:   writeEnv(t, srv.URL),
-		CachePath: filepath.Join(t.TempDir(), "c.sqlite"),
-		Client:    srv.Client(),
-		Now:       func() float64 { return 1_000 },
-		Sleep:     func(context.Context, time.Duration) error { return nil },
-	})
-	if out.Status != StatusThrottled {
-		t.Fatalf("status = %s", out.Status)
-	}
-}
-
-func writeEnv(t *testing.T, url string) string {
-	t.Helper()
-	p := filepath.Join(t.TempDir(), "search.env")
-	if err := os.WriteFile(p, []byte("BRAIN_SEARCH_URL="+url+"\n"), 0o600); err != nil {
-		t.Fatal(err)
-	}
-	return p
-}

skills/brain/SKILL.md

@@ -39,9 +39,8 @@ are not wired yet); do not treat it as a graph walk.
 - Search before you read. Never grep a repo for a concept the graph covers.
 - `--root facts` returns only confirmed evidence-linked answers. Default shows
   facts first, then info leafs clearly marked `(not confirmed)`.
-- If there is no facts hit, `bin/brain/search.go` consults SearXNG and adds a
-  `web` block (kept apart from graph hits). `throttled` / `skipped` / `refused`
-  are not evidence of absence. `--root facts|info` and `--no-web` skip the web.
 - If recall looks wrong, run `bin/brain/eval.go`; it gates control questions and
   should stay at or above 95% recall@5.
-- Never report an unconfirmed single-source local answer as fact.
+- Escalate to `web-search` (the `web-search` skill) as the independent second
+  source when both local roots cannot confirm; never report an unconfirmed
+  single-source local answer as fact.