diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50d2625..41f9f99 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,10 @@ jobs: with: fetch-depth: 0 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + - name: Install uv uses: astral-sh/setup-uv@v6 with: @@ -32,23 +36,19 @@ jobs: bash -n bin/db/psql-yq bash -n bin/db/ssh-tunnel bash -n bin/docker-entrypoint + bash -n bin/kb/search - name: Python unit tests (offline, vendored tools) run: | uv run python -m unittest discover -s bin/tools -t . - - name: Go tests (server + watch packages) + - name: Go tests (root module, no ladybug cgo) run: | go vet ./... go test ./... -count=1 - - name: kbsearch ranking tests (no cgo / no ladybug) - working-directory: bin/kbsearch - run: go test ./rank -count=1 - - - name: Go tests (chats nested module) - working-directory: bin/chats - run: go test ./... -count=1 + - name: brain ranking tests (no cgo / no ladybug) + run: go test ./internal/brain/rank -count=1 - name: facts/audit self (lexicon consistency, no network) run: | diff --git a/.gitignore b/.gitignore index defce95..e3cb108 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ __pycache__/ .env .secrets/ lib-ladybug/ +go.work.local diff --git a/AGENTS.md b/AGENTS.md index c1307fe..3ab90d6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -35,7 +35,9 @@ Read first: [PLAN](PLAN.md) → [docs](docs/). PLAN.md decisions + execution + open questions docs/ published docs skills/ in-project agent skills (vendored, no external links) -bin/ self-describing tools bin/{subject}/{method} (shebang) +bin/ self-describing tools bin/{subject}/{method}.go (shebang) +bin/brain/ search.go (deduction); libs in internal/brain +internal/ shared Go (brain/rank is cgo-free) bin/serve.go async Go HTTP server entry (self-executing go run shebang) bin/watch/ corpus watcher Go package (mtimes, no inotify deps) bin/server/ async Go HTTP server (goroutines, bounded worker pool) @@ -74,7 +76,8 @@ bin/mail/index_mail # rebuil ```bash bin/facts/audit ["self"|"facts"|"info"|"stale"] # 2-source + staleness gate bin/facts/crm [--dry-run] # proof person↔company/company↔project (ooCRM × corpus SoT) -bin/kb/search "query" [--hop N] [--repo X] # deduction search → YAML +bin/kb/search "query" [--repo X] # deprecated wrapper → bin/brain/search.go +bin/brain/search.go "query" [--root facts|info] # deduction search → YAML bin/md/tables # what the graph holds → YAML bin/brain/deduce "question" # thinking wrapper ``` diff --git a/PLAN.md b/PLAN.md index 5230b91..5f66157 100644 --- a/PLAN.md +++ b/PLAN.md @@ -37,7 +37,7 @@ detective method: **a fact needs ≥2 independent sources or it is | D11 | strong/weak | `root` column: `facts` (strong) vs `info` (weak). Answer is `confirmed` only from facts root. | | D12 | transactional | facts and info split by root but **written in the same Ladybug transaction (ACID)** on every write. | | D13 | portfolio | start graph `(Person:eslider)-[:HAS]->(Portfolio)`, associate other natural/juristic persons later. | -| D14 | tooling style | `bin/{subject}/{method}` self-describing: shebang line 1, usage comment from line 2. Go shebang: `///usr/bin/env go run "$0" "$@"; exit`. | +| D14 | tooling style | `bin/{subject}/{method}.go` shebang (e.g. `bin/brain/search.go`). Shared code in `internal/`. One root `go.mod` + `go.work`. No `bin/*/main.go`, no nested modules. | | D15 | repo | Gitea [`eSlider/2dph`](https://git.produktor.io/eSlider/2dph) is origin + [issues](https://git.produktor.io/eSlider/2dph/issues). GitHub `eSlider/2dph` is the public clone (PRs + Actions CI). No direct `main` pushes. TDD → PR → CI green → merge. | | D16 | contradictions | ≥2 yes vs ≥2 no → unrelated sources conflict → hypothesis → `(not confirmed)`. Resolution (authority, staleness adjudication) = **v2**, tracked as open question. | diff --git a/bin/brain/doc.go b/bin/brain/doc.go new file mode 100644 index 0000000..a132c92 --- /dev/null +++ b/bin/brain/doc.go @@ -0,0 +1,3 @@ +// Commands in this directory are shebang mains (search.go). +// search.go is behind the system_ladybug build tag (cgo). +package main diff --git a/bin/brain/search.go b/bin/brain/search.go new file mode 100755 index 0000000..78ce932 --- /dev/null +++ b/bin/brain/search.go @@ -0,0 +1,23 @@ +//usr/bin/env go run -tags=system_ladybug "$0" "$@"; exit +//go:build cgo && system_ladybug +// +// bin/brain/search.go - deduction search over the 2dph brain. +// +// ./bin/brain/search.go "query" [--root facts|info] [--repo P] [-n N] [--json] +// ./bin/brain/search.go serve [port] +// ./bin/brain/search.go --list-model +// +// Needs CGO + libladybug (CGO_CFLAGS/CGO_LDFLAGS). Prefer the wrapper +// bin/kb/search which sets those and builds a binary for the embed daemon. +// NOTE: never run `gofmt -w` on this file — it breaks the shebang. +package main + +import ( + "os" + + "github.com/eSlider/2dph/internal/brain" +) + +func main() { + os.Exit(brain.Main(os.Args[1:])) +} diff --git a/bin/chats/go.mod b/bin/chats/go.mod deleted file mode 100644 index 9df1766..0000000 --- a/bin/chats/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module github.com/eSlider/2dph/bin/chats - -go 1.25.0 diff --git a/bin/kb/search b/bin/kb/search index 3c24a47..5b4af3d 100755 --- a/bin/kb/search +++ b/bin/kb/search @@ -1,34 +1,37 @@ #!/usr/bin/env bash -# bin/kb/search - Go deduction search over the brain (model served by daemon). -# Builds the kbsearch binary on first run / when source changes, then execs it. +# bin/kb/search — deprecated wrapper. Use bin/brain/search.go. +# Sets CGO for ladybug, builds a binary (embed daemon needs a real executable), +# then execs it. Prints one deprecation line. set -euo pipefail -KB="$(cd "$(dirname "$0")/../.." && pwd)" -BIN="$KB/var/bin/kbsearch" -SRC="$KB/bin/kbsearch" +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +BIN="$ROOT/var/bin/brain-search" +SRC="$ROOT/internal/brain" +CMD="$ROOT/bin/brain" -mkdir -p "$KB/var/bin" +mkdir -p "$ROOT/var/bin" -# Rebuild if binary missing or any .go source newer need_build=0 if [ ! -x "$BIN" ]; then - need_build=1 + need_build=1 else - # Check if any .go in kbsearch is newer than binary - while IFS= read -r -d '' f; do - if [ "$f" -nt "$BIN" ]; then - need_build=1 - break - fi - done < <(find "$SRC" -name '*.go' -print0 2>/dev/null) + while IFS= read -r -d '' f; do + if [ "$f" -nt "$BIN" ]; then + need_build=1 + break + fi + done < <(find "$SRC" "$CMD" -name '*.go' -print0 2>/dev/null) fi if [ "$need_build" -eq 1 ]; then - echo "Building kbsearch..." >&2 - (cd "$SRC" && \ - CGO_CFLAGS="-I$KB/lib-ladybug" \ - CGO_LDFLAGS="-L$KB/lib-ladybug -Wl,-rpath,$KB/lib-ladybug" \ - go build -tags system_ladybug -o "$BIN" .) || exit 1 + echo "Building brain/search..." >&2 + ( + cd "$ROOT" && + CGO_CFLAGS="-I$ROOT/lib-ladybug" \ + CGO_LDFLAGS="-L$ROOT/lib-ladybug -Wl,-rpath,$ROOT/lib-ladybug" \ + go build -tags system_ladybug -o "$BIN" ./bin/brain + ) || exit 1 fi -exec "$BIN" "$@" \ No newline at end of file +echo "bin/kb/search is deprecated; use bin/brain/search.go" >&2 +exec "$BIN" "$@" diff --git a/bin/kbsearch/go.mod b/bin/kbsearch/go.mod deleted file mode 100644 index 5054e8e..0000000 --- a/bin/kbsearch/go.mod +++ /dev/null @@ -1,23 +0,0 @@ -module github.com/eSlider/2dph/bin/kbsearch - -go 1.26.0 - -require ( - github.com/LadybugDB/go-ladybug v0.17.0 - github.com/chewxy/math32 v1.11.2 - github.com/daulet/tokenizers v1.27.0 -) - -require ( - github.com/apache/arrow-go/v18 v18.6.0 // indirect - github.com/goccy/go-json v0.10.6 // indirect - github.com/google/flatbuffers v25.12.19+incompatible // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/klauspost/compress v1.18.5 // indirect - github.com/klauspost/cpuid/v2 v2.3.0 // indirect - github.com/pierrec/lz4/v4 v4.1.26 // indirect - github.com/shopspring/decimal v1.4.0 // indirect - github.com/zeebo/xxh3 v1.1.0 // indirect - golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect - golang.org/x/sys v0.43.0 // indirect -) diff --git a/bin/kbsearch/go.sum b/bin/kbsearch/go.sum deleted file mode 100644 index 0064c1b..0000000 --- a/bin/kbsearch/go.sum +++ /dev/null @@ -1,44 +0,0 @@ -github.com/LadybugDB/go-ladybug v0.17.0 h1:RXDbkBjrbRmLdEbhGl4CLOIEzSt09gbP0n9UbKDEfwI= -github.com/LadybugDB/go-ladybug v0.17.0/go.mod h1:GeIXmE8XyF5TFS94NAuTag7vgCC+no/HTBMRA6Rd5Cs= -github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= -github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= -github.com/apache/arrow-go/v18 v18.6.0 h1:GX/Jyd3R7mCLiECAwY9FWbbaYblie2WXBSz4Sw8fNpM= -github.com/apache/arrow-go/v18 v18.6.0/go.mod h1:gm3MiPpY82fLYK5VKPB3WoJbsiLVDfT7flD5/vHReKw= -github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc= -github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g= -github.com/chewxy/math32 v1.11.2 h1:IufN08Zwr1NKuWfY+4Tz55BcwKmyKKNdOP7KtumehnM= -github.com/chewxy/math32 v1.11.2/go.mod h1:dOB2rcuFrCn6UHrze36WSLVPKtzPMRAQvBvUwkSsLqs= -github.com/daulet/tokenizers v1.27.0 h1:MmFYAEDFz69s/nNQfHg59DWqHz3v94m99kEZ/JbL+s4= -github.com/daulet/tokenizers v1.27.0/go.mod h1:YjFY1o1HGMyWkQgbXJDghhvke/yFDp2vGdIO2hYs4MQ= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= -github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= -github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= -github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= -github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= -github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= -github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= -github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= -github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= -github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= -github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= -golang.org/x/exp v0.0.0-20260112195511-716be5621a96 h1:Z/6YuSHTLOHfNFdb8zVZomZr7cqNgTJvA8+Qz75D8gU= -golang.org/x/exp v0.0.0-20260112195511-716be5621a96/go.mod h1:nzimsREAkjBCIEFtHiYkrJyT+2uy9YZJB7H1k68CXZU= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= -gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/bin/kbsearch/main.go b/bin/kbsearch/main.go deleted file mode 100644 index b5ac729..0000000 --- a/bin/kbsearch/main.go +++ /dev/null @@ -1,46 +0,0 @@ -// bin/kbsearch - the Go implementation of bin/kb/search (nested module so the -// root `go test ./...` and CI never compile it against native ladyships). -// -// Usage (built/run by ./bin/kb/search): -// -// kbsearch "query" [--root facts|info] [--repo P] [-n N] [--json] -// kbsearch serve [port] start the embedding daemon -// kbsearch --list-model print the resolved model dir -// -// The potion-multilingual model is loaded only in `serve`; a CLI reuses the -// daemon over localhost HTTP (KBSEARCH_PORT, default 17830) and starts one in -// the background when none answers. KBSEARCH_NO_DAEMON=1 skips that and embeds -// in-process instead. -package main - -import ( - "fmt" - "log" - "os" - "strconv" -) - -func main() { - if len(os.Args) > 1 && os.Args[1] == "serve" { - port := 17830 - if len(os.Args) > 2 { - if p, err := strconv.Atoi(os.Args[2]); err == nil { - port = p - } - } - if err := serve(port); err != nil { - log.Fatalf("kbsearch serve: %v", err) - } - return - } - if len(os.Args) > 1 && os.Args[1] == "--list-model" { - dir, err := modelDir() - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - fmt.Println(dir) - return - } - os.Exit(runSearch(os.Args[1:])) -} diff --git a/bin/tools/test_bin_layout.py b/bin/tools/test_bin_layout.py new file mode 100644 index 0000000..bf7e9e7 --- /dev/null +++ b/bin/tools/test_bin_layout.py @@ -0,0 +1,36 @@ +"""D14 layout: bin/{subject}/{method}.go, libs in internal/, one go.mod.""" +from __future__ import annotations + +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + + +class BinLayoutTest(unittest.TestCase): + def test_brain_search_shebang_exists(self) -> None: + p = ROOT / "bin" / "brain" / "search.go" + self.assertTrue(p.is_file(), "missing bin/brain/search.go") + first = p.read_text().splitlines()[0] + self.assertTrue( + first.startswith("//usr/bin/env go run"), + f"shebang first line, got {first!r}", + ) + + def test_no_nested_go_mod_under_bin(self) -> None: + nested = list((ROOT / "bin").rglob("go.mod")) + self.assertEqual(nested, [], f"nested go.mod files: {nested}") + + def test_rank_lives_in_internal_brain(self) -> None: + self.assertTrue( + (ROOT / "internal" / "brain" / "rank" / "rank.go").is_file(), + "ranking must live in internal/brain/rank (cgo-free)", + ) + self.assertFalse( + (ROOT / "bin" / "kbsearch").exists(), + "bin/kbsearch nested module must be gone", + ) + + def test_no_main_go_under_bin_brain(self) -> None: + main = ROOT / "bin" / "brain" / "main.go" + self.assertFalse(main.exists(), "bin/brain/main.go is not a method") diff --git a/go.mod b/go.mod index 2d704f6..79ccecf 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,25 @@ module github.com/eSlider/2dph -go 1.25.0 +go 1.26 require ( + github.com/LadybugDB/go-ladybug v0.17.0 github.com/arran4/golang-ical v0.3.5 + github.com/chewxy/math32 v1.11.2 + github.com/daulet/tokenizers v1.27.0 golang.org/x/text v0.40.0 ) + +require ( + github.com/apache/arrow-go/v18 v18.6.0 // indirect + github.com/goccy/go-json v0.10.6 // indirect + github.com/google/flatbuffers v25.12.19+incompatible // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.18.5 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/pierrec/lz4/v4 v4.1.26 // indirect + github.com/shopspring/decimal v1.4.0 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect + golang.org/x/sys v0.43.0 // indirect +) diff --git a/go.sum b/go.sum index 5962f4f..e7bf4ab 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,50 @@ +github.com/LadybugDB/go-ladybug v0.17.0 h1:RXDbkBjrbRmLdEbhGl4CLOIEzSt09gbP0n9UbKDEfwI= +github.com/LadybugDB/go-ladybug v0.17.0/go.mod h1:GeIXmE8XyF5TFS94NAuTag7vgCC+no/HTBMRA6Rd5Cs= +github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= +github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/apache/arrow-go/v18 v18.6.0 h1:GX/Jyd3R7mCLiECAwY9FWbbaYblie2WXBSz4Sw8fNpM= +github.com/apache/arrow-go/v18 v18.6.0/go.mod h1:gm3MiPpY82fLYK5VKPB3WoJbsiLVDfT7flD5/vHReKw= +github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc= +github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g= github.com/arran4/golang-ical v0.3.5 h1:bbz6ld4dC+MmCKiFfOd6SkmIGnhNMBACZ485ULh7p9A= github.com/arran4/golang-ical v0.3.5/go.mod h1:OnguFgjN0Hmx8jzpmWcC+AkHio94ujmLHKoaef7xQh8= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/chewxy/math32 v1.11.2 h1:IufN08Zwr1NKuWfY+4Tz55BcwKmyKKNdOP7KtumehnM= +github.com/chewxy/math32 v1.11.2/go.mod h1:dOB2rcuFrCn6UHrze36WSLVPKtzPMRAQvBvUwkSsLqs= +github.com/daulet/tokenizers v1.27.0 h1:MmFYAEDFz69s/nNQfHg59DWqHz3v94m99kEZ/JbL+s4= +github.com/daulet/tokenizers v1.27.0/go.mod h1:YjFY1o1HGMyWkQgbXJDghhvke/yFDp2vGdIO2hYs4MQ= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= +github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= +github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= +github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= +github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +golang.org/x/exp v0.0.0-20260112195511-716be5621a96 h1:Z/6YuSHTLOHfNFdb8zVZomZr7cqNgTJvA8+Qz75D8gU= +golang.org/x/exp v0.0.0-20260112195511-716be5621a96/go.mod h1:nzimsREAkjBCIEFtHiYkrJyT+2uy9YZJB7H1k68CXZU= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go.work b/go.work new file mode 100644 index 0000000..f746806 --- /dev/null +++ b/go.work @@ -0,0 +1,3 @@ +go 1.26 + +use . diff --git a/bin/kbsearch/brain.go b/internal/brain/db.go similarity index 96% rename from bin/kbsearch/brain.go rename to internal/brain/db.go index caf3644..cce5e9b 100644 --- a/bin/kbsearch/brain.go +++ b/internal/brain/db.go @@ -1,5 +1,6 @@ -// Brain connection management using go-ladybug. -package main +//go:build cgo && system_ladybug + +package brain import ( "fmt" diff --git a/internal/brain/doc.go b/internal/brain/doc.go new file mode 100644 index 0000000..d5670bd --- /dev/null +++ b/internal/brain/doc.go @@ -0,0 +1,3 @@ +// Package brain is deduction search over Ladybug (FTS + HNSW). +// Query/embed code that needs cgo lives behind the system_ladybug tag. +package brain diff --git a/bin/kbsearch/model.go b/internal/brain/model.go similarity index 99% rename from bin/kbsearch/model.go rename to internal/brain/model.go index c8735fa..c16cba4 100644 --- a/bin/kbsearch/model.go +++ b/internal/brain/model.go @@ -1,9 +1,11 @@ +//go:build cgo && system_ladybug + // StaticModel wraps the potion-multilingual-128m embedding model. // // Mirrors model2vec.StaticModel: tokenizer (daulet Unigram) + safetensors matrix. // Embed(text) applies the same preprocessing: median_token_length pre-truncation, // add_special_tokens=false, drop unk (id=1), truncate to 512, mean pool, L2 normalize +1e-32. -package main +package brain import ( "encoding/json" diff --git a/bin/kbsearch/modeldir.go b/internal/brain/modeldir.go similarity index 94% rename from bin/kbsearch/modeldir.go rename to internal/brain/modeldir.go index 7913745..a075aae 100644 --- a/bin/kbsearch/modeldir.go +++ b/internal/brain/modeldir.go @@ -1,5 +1,4 @@ -// modelDir returns the resolved potion-multilingual-128m model directory. -package main +package brain import ( "fmt" diff --git a/bin/kbsearch/rank/args.go b/internal/brain/rank/args.go similarity index 90% rename from bin/kbsearch/rank/args.go rename to internal/brain/rank/args.go index 72263e6..dd4970f 100644 --- a/bin/kbsearch/rank/args.go +++ b/internal/brain/rank/args.go @@ -6,9 +6,9 @@ import ( "strings" ) -const Usage = `usage: kbsearch "query" [--root facts|info] [--repo REPO] [-n N] [--json] - kbsearch serve [port] - kbsearch --list-model` +const Usage = `usage: bin/brain/search.go "query" [--root facts|info] [--repo REPO] [-n N] [--json] + bin/brain/search.go serve [port] + bin/brain/search.go --list-model` type Options struct { Query string diff --git a/bin/kbsearch/rank/query.go b/internal/brain/rank/query.go similarity index 100% rename from bin/kbsearch/rank/query.go rename to internal/brain/rank/query.go diff --git a/bin/kbsearch/rank/rank.go b/internal/brain/rank/rank.go similarity index 96% rename from bin/kbsearch/rank/rank.go rename to internal/brain/rank/rank.go index 474836b..4cdf47b 100644 --- a/bin/kbsearch/rank/rank.go +++ b/internal/brain/rank/rank.go @@ -1,4 +1,4 @@ -// Package rank is the cgo-free ranking and CLI parsing for kbsearch. +// Package rank is the cgo-free ranking and CLI parsing for brain search. // CI can `go test ./rank` without the native ladybug library. package rank diff --git a/bin/kbsearch/rank/rank_test.go b/internal/brain/rank/rank_test.go similarity index 95% rename from bin/kbsearch/rank/rank_test.go rename to internal/brain/rank/rank_test.go index 838acfc..15e0eb1 100644 --- a/bin/kbsearch/rank/rank_test.go +++ b/internal/brain/rank/rank_test.go @@ -136,6 +136,12 @@ func TestListModelNeedsNoQuery(t *testing.T) { } } +func TestUsageNamesBrainSearch(t *testing.T) { + if !strings.Contains(Usage, "bin/brain/search.go") { + t.Fatalf("usage must name bin/brain/search.go, got:\n%s", Usage) + } +} + func TestFTSQueryOrdersByScoreDescending(t *testing.T) { if !strings.Contains(FTSStmt, "ORDER BY score DESC") { t.Fatalf("FTS query must order by score DESC, got:\n%s", FTSStmt) diff --git a/bin/kbsearch/search.go b/internal/brain/search.go similarity index 92% rename from bin/kbsearch/search.go rename to internal/brain/search.go index 3e090ab..d150186 100644 --- a/bin/kbsearch/search.go +++ b/internal/brain/search.go @@ -1,5 +1,6 @@ -// Hybrid FTS + vector search implementation, plus daemon client/server. -package main +//go:build cgo && system_ladybug + +package brain import ( "bytes" @@ -18,7 +19,7 @@ import ( "time" lbug "github.com/LadybugDB/go-ladybug" - "github.com/eSlider/2dph/bin/kbsearch/rank" + "github.com/eSlider/2dph/internal/brain/rank" ) const defaultPort = 17830 @@ -28,7 +29,7 @@ const healthPath = "/health" func runSearch(args []string) int { opt, err := rank.ParseArgs(args) if err != nil { - fmt.Fprintf(os.Stderr, "kbsearch: %v\n%s\n", err, rank.Usage) + fmt.Fprintf(os.Stderr, "brain/search: %v\n%s\n", err, rank.Usage) return 2 } root, repo, limit, query := opt.Root, opt.Repo, opt.Limit, opt.Query @@ -247,7 +248,7 @@ func serve(port int) error { }) addr := fmt.Sprintf("127.0.0.1:%d", port) - log.Printf("kbsearch daemon listening on %s", addr) + log.Printf("brain search daemon listening on %s", addr) return http.ListenAndServe(addr, mux) } @@ -357,3 +358,21 @@ func ensureDaemon(port int) error { } return fmt.Errorf("daemon failed to start on port %d", port) } + +// Main is the bin/brain/search.go entry: search, serve, or --list-model. +func Main(args []string) int { + if len(args) > 0 && args[0] == "serve" { + port := defaultPort + if len(args) > 1 { + if p, err := strconv.Atoi(args[1]); err == nil { + port = p + } + } + if err := serve(port); err != nil { + log.Printf("brain/search serve: %v", err) + return 1 + } + return 0 + } + return runSearch(args) +} diff --git a/bin/kbsearch/types.go b/internal/brain/types.go similarity index 67% rename from bin/kbsearch/types.go rename to internal/brain/types.go index 509bf1a..09409a9 100644 --- a/bin/kbsearch/types.go +++ b/internal/brain/types.go @@ -1,10 +1,9 @@ -// Common types and helpers for kbsearch. -package main +package brain import ( "os" - "github.com/eSlider/2dph/bin/kbsearch/rank" + "github.com/eSlider/2dph/internal/brain/rank" ) func eps() string { return os.Getenv("KBTEST_EPS") } diff --git a/bin/kbsearch/yaml.go b/internal/brain/yaml.go similarity index 99% rename from bin/kbsearch/yaml.go rename to internal/brain/yaml.go index 74c4f18..4bc4e59 100644 --- a/bin/kbsearch/yaml.go +++ b/internal/brain/yaml.go @@ -1,5 +1,5 @@ // YAML emitter ported from bin/kb/yamlout.py — preserves insertion order. -package main +package brain import ( "fmt"