Compare commits
20
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
639e2cc077 | ||
|
|
5990feb1f6 | ||
|
|
d27a738fee | ||
|
|
669e184cf6 | ||
|
|
ebc3f948c1 | ||
|
|
fe6a02024c | ||
|
|
4a065d9838 | ||
|
|
e3c6ef5684 | ||
|
|
98c14e23f1 | ||
|
|
ff1716de40 | ||
|
|
fec5325c7a | ||
|
|
27d9521e7f | ||
|
|
a7cb8d4c76 | ||
|
|
53cd00284d | ||
|
|
b73b4d4f97 | ||
|
|
1d1f6a90ff | ||
|
|
f220bcd95a | ||
|
|
678a1d1dba | ||
|
|
8781c0c3eb | ||
|
|
d6b17e8819 |
@@ -19,6 +19,10 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
|
|
||||||
|
- uses: actions/setup-go@v5
|
||||||
|
with:
|
||||||
|
go-version-file: go.mod
|
||||||
|
|
||||||
- name: Install uv
|
- name: Install uv
|
||||||
uses: astral-sh/setup-uv@v6
|
uses: astral-sh/setup-uv@v6
|
||||||
with:
|
with:
|
||||||
@@ -32,16 +36,20 @@ jobs:
|
|||||||
bash -n bin/db/psql-yq
|
bash -n bin/db/psql-yq
|
||||||
bash -n bin/db/ssh-tunnel
|
bash -n bin/db/ssh-tunnel
|
||||||
bash -n bin/docker-entrypoint
|
bash -n bin/docker-entrypoint
|
||||||
|
bash -n bin/kb/search
|
||||||
|
|
||||||
- name: Python unit tests (offline, vendored tools)
|
- name: Python unit tests (offline, vendored tools)
|
||||||
run: |
|
run: |
|
||||||
uv run python -m unittest discover -s bin/tools -t .
|
uv run python -m unittest discover -s bin/tools -t .
|
||||||
|
|
||||||
- name: Go tests (server + watch packages)
|
- name: Go tests (root module, no ladybug cgo)
|
||||||
run: |
|
run: |
|
||||||
go vet ./...
|
go vet ./...
|
||||||
go test ./... -count=1
|
go test ./... -count=1
|
||||||
|
|
||||||
|
- name: brain ranking tests (no cgo / no ladybug)
|
||||||
|
run: go test ./internal/brain/rank -count=1
|
||||||
|
|
||||||
- name: facts/audit self (lexicon consistency, no network)
|
- name: facts/audit self (lexicon consistency, no network)
|
||||||
run: |
|
run: |
|
||||||
./bin/facts/audit self 2>/dev/null || echo "audit: not yet implemented; gate skipped"
|
./bin/facts/audit self 2>/dev/null || echo "audit: not yet implemented; gate skipped"
|
||||||
|
|||||||
@@ -10,3 +10,4 @@ __pycache__/
|
|||||||
.env
|
.env
|
||||||
.secrets/
|
.secrets/
|
||||||
lib-ladybug/
|
lib-ladybug/
|
||||||
|
go.work.local
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ Read first: [PLAN](PLAN.md) → [docs](docs/).
|
|||||||
2. **Read-only data sources.** Ladybug `var/kb.lbug` and Postgres are opened
|
2. **Read-only data sources.** Ladybug `var/kb.lbug` and Postgres are opened
|
||||||
read-only for queries. Index rebuilds write to `var/` (gitignored).
|
read-only for queries. Index rebuilds write to `var/` (gitignored).
|
||||||
3. **PII.** `brain-test`, `cs_brain` client data is never read or quoted.
|
3. **PII.** `brain-test`, `cs_brain` client data is never read or quoted.
|
||||||
4. **No main pushes.** Feature branches + PR via `gh`; CI must be green.
|
4. **No main pushes.** Feature branches + GitHub PR (`gh`); CI (Actions) must be green. Work board: [Gitea issues](https://git.produktor.io/eSlider/2dph/issues).
|
||||||
5. **TDD.** Failing test before tool code. Unit tests run offline against
|
5. **TDD.** Failing test before tool code. Unit tests run offline against
|
||||||
fixtures; network/db calls are wrapped.
|
fixtures; network/db calls are wrapped.
|
||||||
6. **docs reflect behaviour.** Any change updates `docs/` + `PLAN.md` status.
|
6. **docs reflect behaviour.** Any change updates `docs/` + `PLAN.md` status.
|
||||||
@@ -35,7 +35,9 @@ Read first: [PLAN](PLAN.md) → [docs](docs/).
|
|||||||
PLAN.md decisions + execution + open questions
|
PLAN.md decisions + execution + open questions
|
||||||
docs/ published docs
|
docs/ published docs
|
||||||
skills/ in-project agent skills (vendored, no external links)
|
skills/ in-project agent skills (vendored, no external links)
|
||||||
bin/ self-describing tools bin/{subject}/{method} (shebang)
|
bin/ self-describing tools bin/{subject}/{method}.go (shebang)
|
||||||
|
bin/brain/ search.go (deduction); libs in internal/brain
|
||||||
|
internal/ shared Go (brain/rank is cgo-free)
|
||||||
bin/serve.go async Go HTTP server entry (self-executing go run shebang)
|
bin/serve.go async Go HTTP server entry (self-executing go run shebang)
|
||||||
bin/watch/ corpus watcher Go package (mtimes, no inotify deps)
|
bin/watch/ corpus watcher Go package (mtimes, no inotify deps)
|
||||||
bin/server/ async Go HTTP server (goroutines, bounded worker pool)
|
bin/server/ async Go HTTP server (goroutines, bounded worker pool)
|
||||||
@@ -74,7 +76,8 @@ bin/mail/index_mail # rebuil
|
|||||||
```bash
|
```bash
|
||||||
bin/facts/audit ["self"|"facts"|"info"|"stale"] # 2-source + staleness gate
|
bin/facts/audit ["self"|"facts"|"info"|"stale"] # 2-source + staleness gate
|
||||||
bin/facts/crm [--dry-run] # proof person↔company/company↔project (ooCRM × corpus SoT)
|
bin/facts/crm [--dry-run] # proof person↔company/company↔project (ooCRM × corpus SoT)
|
||||||
bin/kb/search "query" [--hop N] [--repo X] # deduction search → YAML
|
bin/kb/search "query" [--repo X] # deprecated wrapper → bin/brain/search.go
|
||||||
|
bin/brain/search.go "query" [--root facts|info] # deduction search → YAML
|
||||||
bin/md/tables # what the graph holds → YAML
|
bin/md/tables # what the graph holds → YAML
|
||||||
bin/brain/deduce "question" # thinking wrapper
|
bin/brain/deduce "question" # thinking wrapper
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -37,8 +37,8 @@ detective method: **a fact needs ≥2 independent sources or it is
|
|||||||
| D11 | strong/weak | `root` column: `facts` (strong) vs `info` (weak). Answer is `confirmed` only from facts root. |
|
| D11 | strong/weak | `root` column: `facts` (strong) vs `info` (weak). Answer is `confirmed` only from facts root. |
|
||||||
| D12 | transactional | facts and info split by root but **written in the same Ladybug transaction (ACID)** on every write. |
|
| D12 | transactional | facts and info split by root but **written in the same Ladybug transaction (ACID)** on every write. |
|
||||||
| D13 | portfolio | start graph `(Person:eslider)-[:HAS]->(Portfolio)`, associate other natural/juristic persons later. |
|
| D13 | portfolio | start graph `(Person:eslider)-[:HAS]->(Portfolio)`, associate other natural/juristic persons later. |
|
||||||
| D14 | tooling style | `bin/{subject}/{method}` self-describing: shebang line 1, usage comment from line 2. Go shebang: `///usr/bin/env go run "$0" "$@"; exit`. |
|
| D14 | tooling style | `bin/{subject}/{method}.go` shebang (e.g. `bin/brain/search.go`). Shared code in `internal/`. One root `go.mod` + `go.work`. No `bin/*/main.go`, no nested modules. |
|
||||||
| D15 | repo | GitHub `eSlider/2dph`, public (like sibling repos), push/commit via `gh`, TDD + commit every change, CI/CD. |
|
| D15 | repo | Gitea [`eSlider/2dph`](https://git.produktor.io/eSlider/2dph) is origin + [issues](https://git.produktor.io/eSlider/2dph/issues). GitHub `eSlider/2dph` is the public clone (PRs + Actions CI). No direct `main` pushes. TDD → PR → CI green → merge. |
|
||||||
| D16 | contradictions | ≥2 yes vs ≥2 no → unrelated sources conflict → hypothesis → `(not confirmed)`. Resolution (authority, staleness adjudication) = **v2**, tracked as open question. |
|
| D16 | contradictions | ≥2 yes vs ≥2 no → unrelated sources conflict → hypothesis → `(not confirmed)`. Resolution (authority, staleness adjudication) = **v2**, tracked as open question. |
|
||||||
|
|
||||||
## Architecture
|
## Architecture
|
||||||
@@ -116,11 +116,13 @@ Common props on every node/edge: `root`, `confidence`, `evidence[]`, `how`,
|
|||||||
|
|
||||||
`.github/workflows/ci.yml`:
|
`.github/workflows/ci.yml`:
|
||||||
|
|
||||||
1. go vet + go test ./... (Go tools)
|
1. go vet + go test ./... (Go tools; root module)
|
||||||
2. python -m unittest discover + pytest (Py tools)
|
2. `go test ./rank` in `bin/kbsearch` (cgo-free ranking + flag parser; nested module still needs ladybug for the rest)
|
||||||
3. bin/facts/audit self (lexicon internal consistency)
|
3. `go test ./...` in `bin/chats` (Telegram + LinkedIn parsers; nested module)
|
||||||
4. bin/kb/eval (recall@5 ≥ 0.95, gates index regressions)
|
4. python -m unittest discover (Py tools)
|
||||||
5. md-docs build/lint if docs tooling arrives.
|
5. bin/facts/audit self (lexicon internal consistency)
|
||||||
|
6. bin/kb/eval (recall@5 ≥ 0.95, gates index regressions)
|
||||||
|
7. md-docs build/lint if docs tooling arrives.
|
||||||
|
|
||||||
Feedback loop: every commit → PR → CI → green/gate → merge. Same discipline as
|
Feedback loop: every commit → PR → CI → green/gate → merge. Same discipline as
|
||||||
`db/tech-poc`: contract first where there is an OpenAPI/message shape.
|
`db/tech-poc`: contract first where there is an OpenAPI/message shape.
|
||||||
|
|||||||
@@ -148,4 +148,7 @@ docker compose up brain-watch # auto re-index on change
|
|||||||
skills (`web-search`, `db-yaml`, …) that 2dph integrates
|
skills (`web-search`, `db-yaml`, …) that 2dph integrates
|
||||||
- detective method — the two-source method
|
- detective method — the two-source method
|
||||||
|
|
||||||
|
Work board (issues): [git.produktor.io/eSlider/2dph/issues](https://git.produktor.io/eSlider/2dph/issues).
|
||||||
|
PRs and CI: GitHub [`eSlider/2dph`](https://github.com/eSlider/2dph).
|
||||||
|
|
||||||
See [PLAN.md](PLAN.md) for decisions, execution status, and v2 open questions.
|
See [PLAN.md](PLAN.md) for decisions, execution status, and v2 open questions.
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
// Commands in this directory are shebang mains (search.go).
|
||||||
|
// search.go is behind the system_ladybug build tag (cgo).
|
||||||
|
package main
|
||||||
Executable
+23
@@ -0,0 +1,23 @@
|
|||||||
|
//usr/bin/env go run -tags=system_ladybug "$0" "$@"; exit
|
||||||
|
//go:build cgo && system_ladybug
|
||||||
|
//
|
||||||
|
// bin/brain/search.go - deduction search over the 2dph brain.
|
||||||
|
//
|
||||||
|
// ./bin/brain/search.go "query" [--root facts|info] [--repo P] [-n N] [--json]
|
||||||
|
// ./bin/brain/search.go serve [port]
|
||||||
|
// ./bin/brain/search.go --list-model
|
||||||
|
//
|
||||||
|
// Needs CGO + libladybug (CGO_CFLAGS/CGO_LDFLAGS). Prefer the wrapper
|
||||||
|
// bin/kb/search which sets those and builds a binary for the embed daemon.
|
||||||
|
// NOTE: never run `gofmt -w` on this file — it breaks the shebang.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/eSlider/2dph/internal/brain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
os.Exit(brain.Main(os.Args[1:]))
|
||||||
|
}
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
module github.com/eSlider/2dph/bin/chats
|
|
||||||
|
|
||||||
go 1.25.0
|
|
||||||
+363
-50
@@ -8,6 +8,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@@ -25,21 +26,248 @@ type lnInboxItem struct {
|
|||||||
Unread bool `json:"unread"`
|
Unread bool `json:"unread"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type lnInboxEnvelope struct {
|
// mcp-server-linkedin v4.22 returns get_inbox / get_conversation as
|
||||||
Results []lnInboxItem `json:"results"`
|
// {url, sections:{inbox|conversation: textblob}, references:{...}}.
|
||||||
HasMore bool `json:"hasMore"`
|
// The conversation list lives in references (kind=conversation); messages live
|
||||||
|
// in the sections text blob, delimited by "<From> sent the following message
|
||||||
|
// at <time>" markers. See testdata/linkedin_*.json for the wire shape.
|
||||||
|
|
||||||
|
type lnEnvelope struct {
|
||||||
|
URL string `json:"url"`
|
||||||
|
Sections map[string]any `json:"sections"`
|
||||||
|
References map[string]any `json:"references"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type lnReference struct {
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
URL string `json:"url"`
|
||||||
|
Text string `json:"text"`
|
||||||
|
Context string `json:"context"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type lnMessage struct {
|
type lnMessage struct {
|
||||||
From string `json:"from"`
|
From string `json:"from"`
|
||||||
Date string `json:"date"`
|
Date string `json:"date"`
|
||||||
Text string `json:"text"`
|
Text string `json:"text"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type lnConvEnvelope struct {
|
var (
|
||||||
Results []lnMessage `json:"results"`
|
lnWeekdays = map[string]time.Weekday{
|
||||||
HasMore bool `json:"hasMore"`
|
"SUNDAY": time.Sunday, "MONDAY": time.Monday, "TUESDAY": time.Tuesday,
|
||||||
TotalCount int `json:"total_count"`
|
"WEDNESDAY": time.Wednesday, "THURSDAY": time.Thursday,
|
||||||
|
"FRIDAY": time.Friday, "SATURDAY": time.Saturday,
|
||||||
|
}
|
||||||
|
lnMsgStartRe = regexp.MustCompile(`^(.+?) sent the following messages? at (.+)$`)
|
||||||
|
lnTimeRe = regexp.MustCompile(`\d{1,2}:\d{2}\s*[AP]M`)
|
||||||
|
)
|
||||||
|
|
||||||
|
func isWeekdayLine(s string) bool {
|
||||||
|
if _, ok := lnWeekdays[s]; ok {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
switch s {
|
||||||
|
case "TODAY", "YESTERDAY", "THIS WEEK", "LAST WEEK":
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return lnMonthDayRe.MatchString(s)
|
||||||
|
}
|
||||||
|
|
||||||
|
var lnMonthDayRe = regexp.MustCompile(`^[A-Z]{3}\s+\d{1,2}$`)
|
||||||
|
|
||||||
|
// parseLinkedInInbox extracts conversations from a get_inbox response.
|
||||||
|
func parseLinkedInInbox(text string) []lnInboxItem {
|
||||||
|
var env lnEnvelope
|
||||||
|
if err := json.Unmarshal([]byte(text), &env); err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
refs, _ := env.References["inbox"].([]any)
|
||||||
|
var items []lnInboxItem
|
||||||
|
for _, r := range refs {
|
||||||
|
rr, ok := r.(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if rr["kind"] != "conversation" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
u, _ := rr["url"].(string)
|
||||||
|
tid := threadIDFromURL(u)
|
||||||
|
if !validThreadID(tid) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name, _ := rr["text"].(string)
|
||||||
|
items = append(items, lnInboxItem{
|
||||||
|
ThreadID: tid,
|
||||||
|
Participants: name,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return items
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseLinkedInConversation parses the sections.conversation text blob into
|
||||||
|
// messages. Messages are delimited by "<From> sent the following message(s) at
|
||||||
|
// <time>" lines; each message body runs until the next marker. Day headers
|
||||||
|
// (all-caps weekdays) provide date context; times are mapped to the most
|
||||||
|
// recent matching weekday.
|
||||||
|
func parseLinkedInConversation(text string) []lnMessage {
|
||||||
|
var env lnEnvelope
|
||||||
|
if err := json.Unmarshal([]byte(text), &env); err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
blob, _ := env.Sections["conversation"].(string)
|
||||||
|
if blob == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var msgs []lnMessage
|
||||||
|
var cur *lnMessage
|
||||||
|
var body []string
|
||||||
|
day := ""
|
||||||
|
|
||||||
|
flush := func() {
|
||||||
|
if cur == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
cur.Text = strings.TrimSpace(strings.Join(body, "\n"))
|
||||||
|
if ts := linkedInTimestamp(day, cur.Date); ts != "" {
|
||||||
|
cur.Date = ts
|
||||||
|
}
|
||||||
|
if cur.Text != "" {
|
||||||
|
msgs = append(msgs, *cur)
|
||||||
|
}
|
||||||
|
cur = nil
|
||||||
|
body = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, raw := range strings.Split(blob, "\n") {
|
||||||
|
line := strings.TrimSpace(raw)
|
||||||
|
if line == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if isWeekdayLine(line) {
|
||||||
|
if line != day {
|
||||||
|
// A new day header terminates the previous message,
|
||||||
|
// which must keep the earlier date context.
|
||||||
|
flush()
|
||||||
|
}
|
||||||
|
day = line
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if m := lnMsgStartRe.FindStringSubmatch(line); m != nil {
|
||||||
|
flush()
|
||||||
|
cur = &lnMessage{From: strings.TrimSpace(m[1]), Date: strings.TrimSpace(m[2])}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if cur == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Skip "View X's profile" and the "<From> (pronouns) <time>" header.
|
||||||
|
if strings.HasPrefix(line, "View ") && strings.HasSuffix(line, "'s profile") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(line, cur.From) && lnTimeRe.MatchString(line) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
body = append(body, line)
|
||||||
|
}
|
||||||
|
flush()
|
||||||
|
return msgs
|
||||||
|
}
|
||||||
|
|
||||||
|
// linkedInTimestamp maps a weekday, relative, or MON DD date header + clock
|
||||||
|
// string to a timestamp, or returns "" when the clock cannot be parsed.
|
||||||
|
func linkedInTimestamp(day, clock string) string {
|
||||||
|
t, err := time.Parse("3:04 PM", clock)
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
var d time.Time
|
||||||
|
if wd, ok := lnWeekdays[day]; ok {
|
||||||
|
diff := (int(now.Weekday()) - int(wd) + 7) % 7
|
||||||
|
d = now.AddDate(0, 0, -diff)
|
||||||
|
} else {
|
||||||
|
switch day {
|
||||||
|
case "TODAY":
|
||||||
|
d = now
|
||||||
|
case "YESTERDAY":
|
||||||
|
d = now.AddDate(0, 0, -1)
|
||||||
|
case "THIS WEEK":
|
||||||
|
diff := int(now.Weekday())
|
||||||
|
d = now.AddDate(0, 0, -diff)
|
||||||
|
case "LAST WEEK":
|
||||||
|
diff := int(now.Weekday()) + 7
|
||||||
|
d = now.AddDate(0, 0, -diff)
|
||||||
|
default:
|
||||||
|
if m := lnMonthDayRe.FindStringSubmatch(day); m != nil {
|
||||||
|
// MON DD without a year: resolve to the most recent
|
||||||
|
// occurrence that is not in the future.
|
||||||
|
d = monthDayDate(day, now)
|
||||||
|
if d.IsZero() {
|
||||||
|
return t.Format("15:04")
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// No date context; keep bare clock time.
|
||||||
|
return t.Format("15:04")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
res := time.Date(d.Year(), d.Month(), d.Day(), t.Hour(), t.Minute(), 0, 0, time.UTC)
|
||||||
|
return res.UTC().Format(time.RFC3339)
|
||||||
|
}
|
||||||
|
|
||||||
|
var lnMonths = map[string]time.Month{
|
||||||
|
"JAN": time.January, "FEB": time.February, "MAR": time.March,
|
||||||
|
"APR": time.April, "MAY": time.May, "JUN": time.June,
|
||||||
|
"JUL": time.July, "AUG": time.August, "SEP": time.September,
|
||||||
|
"OCT": time.October, "NOV": time.November, "DEC": time.December,
|
||||||
|
}
|
||||||
|
|
||||||
|
// monthDayDate resolves "MON DD" to the most recent occurrence of that date,
|
||||||
|
// preferring the current year and falling back to the previous year when the
|
||||||
|
// date is in the future. Returns zero time when unresolvable.
|
||||||
|
func monthDayDate(day string, now time.Time) time.Time {
|
||||||
|
parts := strings.Fields(day)
|
||||||
|
if len(parts) != 2 {
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
mo, ok := lnMonths[parts[0]]
|
||||||
|
if !ok {
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
var dd int
|
||||||
|
if _, err := fmt.Sscanf(parts[1], "%d", &dd); err != nil {
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
if dd < 1 || dd > 31 {
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
d := time.Date(now.Year(), mo, dd, 0, 0, 0, 0, time.UTC)
|
||||||
|
if d.After(now) {
|
||||||
|
d = d.AddDate(-1, 0, 0)
|
||||||
|
}
|
||||||
|
if d.After(now) {
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
|
func threadIDFromURL(u string) string {
|
||||||
|
u = strings.TrimSuffix(u, "/")
|
||||||
|
idx := strings.LastIndex(u, "/")
|
||||||
|
if idx < 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return u[idx+1:]
|
||||||
|
}
|
||||||
|
|
||||||
|
// validThreadID rejects path segments that are not real thread ids (e.g. the
|
||||||
|
// literal "thread" or an empty trailing segment).
|
||||||
|
func validThreadID(id string) bool {
|
||||||
|
if id == "" || id == "thread" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewLinkedInMCPSource(userDataDir string) *LinkedInMCPSource {
|
func NewLinkedInMCPSource(userDataDir string) *LinkedInMCPSource {
|
||||||
@@ -53,16 +281,39 @@ func (s *LinkedInMCPSource) Sync(ctx context.Context, outDir string, limit int)
|
|||||||
s.limit = limit
|
s.limit = limit
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// getConversation fetches one thread, recreating the MCP server when it
|
||||||
|
// wedges. A single 429 makes mcp-server-linkedin close its browser and
|
||||||
|
// refuse every later call ("still has a browser open"), so a broken server
|
||||||
|
// must be restarted rather than hammered.
|
||||||
|
getConversation := func(threadID string) ([]lnMessage, error) {
|
||||||
|
client, err := newLinkedInMCP(ctx, s.userDataDir)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("linkedin mcp: %w", err)
|
||||||
|
}
|
||||||
|
defer client.Close()
|
||||||
|
msgs, err := client.GetConversation(ctx, "", threadID, msgLimitFor(s.limit))
|
||||||
|
if err != nil && wedged(err) {
|
||||||
|
fmt.Fprintf(os.Stderr, "chats: %s: server wedged, restarting broker\n", threadID)
|
||||||
|
time.Sleep(5 * time.Second)
|
||||||
|
client2, cerr := newLinkedInMCP(ctx, s.userDataDir)
|
||||||
|
if cerr == nil {
|
||||||
|
defer client2.Close()
|
||||||
|
msgs, err = client2.GetConversation(ctx, "", threadID, msgLimitFor(s.limit))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return msgs, err
|
||||||
|
}
|
||||||
|
|
||||||
client, err := newLinkedInMCP(ctx, s.userDataDir)
|
client, err := newLinkedInMCP(ctx, s.userDataDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("linkedin mcp: %w", err)
|
return fmt.Errorf("linkedin mcp: %w", err)
|
||||||
}
|
}
|
||||||
defer client.Close()
|
|
||||||
|
|
||||||
inbox, err := client.GetInbox(ctx, 50)
|
inbox, err := client.GetInbox(ctx, 50)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
client.Close()
|
||||||
return fmt.Errorf("get_inbox: %w", err)
|
return fmt.Errorf("get_inbox: %w", err)
|
||||||
}
|
}
|
||||||
|
client.Close()
|
||||||
if len(inbox) == 0 {
|
if len(inbox) == 0 {
|
||||||
fmt.Println("chats: no LinkedIn conversations found")
|
fmt.Println("chats: no LinkedIn conversations found")
|
||||||
return nil
|
return nil
|
||||||
@@ -88,18 +339,21 @@ func (s *LinkedInMCPSource) Sync(ctx context.Context, outDir string, limit int)
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
msgLimit := 100
|
msgs, err := getConversation(conv.ThreadID)
|
||||||
if s.limit > 0 {
|
|
||||||
msgLimit = s.limit
|
|
||||||
}
|
|
||||||
|
|
||||||
msgs, err := client.GetConversation(ctx, "", conv.ThreadID, msgLimit)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "chats: get_conversation %s: %v\n", convID, err)
|
fmt.Fprintf(os.Stderr, "chats: get_conversation %s: %v\n", convID, err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
jsonlPath := filepath.Join(chatDir, "messages.jsonl")
|
jsonlPath := filepath.Join(chatDir, "messages.jsonl")
|
||||||
|
|
||||||
|
// A rate-limited response can parse to zero messages. Never clobber
|
||||||
|
// previously synced data with an empty file.
|
||||||
|
if len(msgs) == 0 {
|
||||||
|
fmt.Fprintf(os.Stderr, "chats: %s (%s): 0 messages parsed, keeping existing file\n", chatName, convID)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
f, err := os.Create(jsonlPath)
|
f, err := os.Create(jsonlPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "chats: create %s: %v\n", jsonlPath, err)
|
fmt.Fprintf(os.Stderr, "chats: create %s: %v\n", jsonlPath, err)
|
||||||
@@ -137,6 +391,13 @@ func (s *LinkedInMCPSource) Sync(ctx context.Context, outDir string, limit int)
|
|||||||
f.Close()
|
f.Close()
|
||||||
|
|
||||||
fmt.Printf("chats: synced %s (%s) — %d messages\n", chatName, convID, written)
|
fmt.Printf("chats: synced %s (%s) — %d messages\n", chatName, convID, written)
|
||||||
|
|
||||||
|
// Pause between conversations to reduce LinkedIn rate limiting.
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -154,6 +415,7 @@ func newLinkedInMCP(ctx context.Context, userDataDir string) (*linkedInMCPClient
|
|||||||
"mcp-server-linkedin@latest",
|
"mcp-server-linkedin@latest",
|
||||||
"--user-data-dir", userDataDir,
|
"--user-data-dir", userDataDir,
|
||||||
"--no-auto-import",
|
"--no-auto-import",
|
||||||
|
"--no-daemon",
|
||||||
"--transport", "stdio",
|
"--transport", "stdio",
|
||||||
"--login-timeout", "10",
|
"--login-timeout", "10",
|
||||||
"--browser-wait", "1",
|
"--browser-wait", "1",
|
||||||
@@ -261,14 +523,90 @@ func (c *linkedInMCPClient) send(ctx context.Context, method string, params inte
|
|||||||
return nil, fmt.Errorf("no response: %w", c.stdout.Err())
|
return nil, fmt.Errorf("no response: %w", c.stdout.Err())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// msgLimitFor returns the per-conversation message cap for a sync.
|
||||||
|
func msgLimitFor(limit int) int {
|
||||||
|
if limit > 0 {
|
||||||
|
return limit
|
||||||
|
}
|
||||||
|
return 100
|
||||||
|
}
|
||||||
|
|
||||||
|
// wedged reports whether a conversation fetch failure means the MCP server
|
||||||
|
// closed its browser and will refuse every later call.
|
||||||
|
func wedged(err error) bool {
|
||||||
|
return strings.Contains(err.Error(), "still has a browser open")
|
||||||
|
}
|
||||||
|
|
||||||
|
// callTool invokes an MCP tool, retrying transient (rate-limit) failures.
|
||||||
|
func (c *linkedInMCPClient) callTool(ctx context.Context, name string, params map[string]interface{}) (json.RawMessage, error) {
|
||||||
|
var lastErr error
|
||||||
|
for attempt := 0; attempt < 3; attempt++ {
|
||||||
|
if attempt > 0 {
|
||||||
|
delay := time.Duration(1<<uint(attempt)) * 5 * time.Second
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
case <-time.After(delay):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result, err := c.send(ctx, "tools/call", map[string]interface{}{
|
||||||
|
"name": name,
|
||||||
|
"arguments": params,
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
// Tool-level errors surface as a successful RPC with an
|
||||||
|
// isError=true content entry.
|
||||||
|
if hint := toolErrorHint(result); hint != "" {
|
||||||
|
lastErr = fmt.Errorf("%s error: %s", name, hint)
|
||||||
|
if !isTransientLinkedInError(lastErr.Error()) {
|
||||||
|
return nil, lastErr
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
lastErr = err
|
||||||
|
if !isTransientLinkedInError(err.Error()) {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("%s: %w", name, lastErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// toolErrorHint returns the tool's error text when the result has isError set.
|
||||||
|
func toolErrorHint(result json.RawMessage) string {
|
||||||
|
var toolRes struct {
|
||||||
|
Content []struct {
|
||||||
|
Type string `json:"type"`
|
||||||
|
Text string `json:"text"`
|
||||||
|
} `json:"content"`
|
||||||
|
IsError bool `json:"isError"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(result, &toolRes); err != nil || !toolRes.IsError {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if len(toolRes.Content) > 0 {
|
||||||
|
return toolRes.Content[0].Text
|
||||||
|
}
|
||||||
|
return "unknown tool error"
|
||||||
|
}
|
||||||
|
|
||||||
|
// isTransientLinkedInError reports whether a fetch failed due to rate limiting
|
||||||
|
// or a transient server error, which may succeed on retry.
|
||||||
|
func isTransientLinkedInError(msg string) bool {
|
||||||
|
return strings.Contains(msg, "503") || strings.Contains(msg, "429") ||
|
||||||
|
strings.Contains(msg, "ERR_HTTP_RESPONSE_CODE_FAILURE") ||
|
||||||
|
strings.Contains(msg, "ERR_ABORTED") ||
|
||||||
|
strings.Contains(msg, "Error calling tool") ||
|
||||||
|
strings.Contains(msg, "Unexpected error") ||
|
||||||
|
strings.Contains(msg, "still has a browser open")
|
||||||
|
}
|
||||||
|
|
||||||
func (c *linkedInMCPClient) GetInbox(ctx context.Context, limit int) ([]lnInboxItem, error) {
|
func (c *linkedInMCPClient) GetInbox(ctx context.Context, limit int) ([]lnInboxItem, error) {
|
||||||
params := map[string]interface{}{
|
params := map[string]interface{}{
|
||||||
"limit": limit,
|
"limit": limit,
|
||||||
}
|
}
|
||||||
result, err := c.send(ctx, "tools/call", map[string]interface{}{
|
result, err := c.callTool(ctx, "get_inbox", params)
|
||||||
"name": "get_inbox",
|
|
||||||
"arguments": params,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -283,23 +621,12 @@ func (c *linkedInMCPClient) GetInbox(ctx context.Context, limit int) ([]lnInboxI
|
|||||||
if err := json.Unmarshal(result, &toolRes); err != nil {
|
if err := json.Unmarshal(result, &toolRes); err != nil {
|
||||||
return nil, fmt.Errorf("unmarshal tool: %w", err)
|
return nil, fmt.Errorf("unmarshal tool: %w", err)
|
||||||
}
|
}
|
||||||
if toolRes.IsError {
|
|
||||||
return nil, fmt.Errorf("get_inbox error")
|
|
||||||
}
|
|
||||||
if len(toolRes.Content) == 0 {
|
if len(toolRes.Content) == 0 {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
text := toolRes.Content[0].Text
|
text := toolRes.Content[0].Text
|
||||||
var env lnInboxEnvelope
|
return parseLinkedInInbox(text), nil
|
||||||
if err := json.Unmarshal([]byte(text), &env); err != nil {
|
|
||||||
var arr []lnInboxItem
|
|
||||||
if err2 := json.Unmarshal([]byte(text), &arr); err2 == nil {
|
|
||||||
return arr, nil
|
|
||||||
}
|
|
||||||
return nil, fmt.Errorf("parse inbox: %w", err)
|
|
||||||
}
|
|
||||||
return env.Results, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *linkedInMCPClient) GetConversation(ctx context.Context, username, threadID string, limit int) ([]lnMessage, error) {
|
func (c *linkedInMCPClient) GetConversation(ctx context.Context, username, threadID string, limit int) ([]lnMessage, error) {
|
||||||
@@ -308,10 +635,7 @@ func (c *linkedInMCPClient) GetConversation(ctx context.Context, username, threa
|
|||||||
"thread_id": threadID,
|
"thread_id": threadID,
|
||||||
"index": limit,
|
"index": limit,
|
||||||
}
|
}
|
||||||
result, err := c.send(ctx, "tools/call", map[string]interface{}{
|
result, err := c.callTool(ctx, "get_conversation", params)
|
||||||
"name": "get_conversation",
|
|
||||||
"arguments": params,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -326,23 +650,12 @@ func (c *linkedInMCPClient) GetConversation(ctx context.Context, username, threa
|
|||||||
if err := json.Unmarshal(result, &toolRes); err != nil {
|
if err := json.Unmarshal(result, &toolRes); err != nil {
|
||||||
return nil, fmt.Errorf("unmarshal tool: %w", err)
|
return nil, fmt.Errorf("unmarshal tool: %w", err)
|
||||||
}
|
}
|
||||||
if toolRes.IsError {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
if len(toolRes.Content) == 0 {
|
if len(toolRes.Content) == 0 {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
text := toolRes.Content[0].Text
|
text := toolRes.Content[0].Text
|
||||||
var env lnConvEnvelope
|
return parseLinkedInConversation(text), nil
|
||||||
if err := json.Unmarshal([]byte(text), &env); err != nil {
|
|
||||||
var arr []lnMessage
|
|
||||||
if err2 := json.Unmarshal([]byte(text), &arr); err2 == nil {
|
|
||||||
return arr, nil
|
|
||||||
}
|
|
||||||
return nil, fmt.Errorf("parse conv: %w", err)
|
|
||||||
}
|
|
||||||
return env.Results, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *linkedInMCPClient) Close() error {
|
func (c *linkedInMCPClient) Close() error {
|
||||||
|
|||||||
@@ -0,0 +1,215 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func readFixture(t *testing.T, name string) string {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(filepath.Join("testdata", name))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return string(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestParseLinkedInInbox verifies get_inbox parsing against the v4.22 wire
|
||||||
|
// format (testdata/linkedin_inbox.json — synthetic Alice/Bob/Charlie).
|
||||||
|
func TestParseLinkedInInbox(t *testing.T) {
|
||||||
|
text := readFixture(t, "linkedin_inbox.json")
|
||||||
|
items := parseLinkedInInbox(text)
|
||||||
|
if len(items) != 2 {
|
||||||
|
t.Fatalf("expected 2 conversations (empty thread url skipped), got %d", len(items))
|
||||||
|
}
|
||||||
|
|
||||||
|
first := items[0]
|
||||||
|
if first.ThreadID == "" {
|
||||||
|
t.Error("expected thread id extracted from reference url")
|
||||||
|
}
|
||||||
|
if first.Participants != "Alice Example" {
|
||||||
|
t.Errorf("participants=%q, want Alice Example", first.Participants)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(first.ThreadID, "2-") {
|
||||||
|
t.Errorf("unexpected thread id format %q", first.ThreadID)
|
||||||
|
}
|
||||||
|
if items[1].Participants != "Charlie Example" {
|
||||||
|
t.Errorf("second participant=%q, want Charlie Example", items[1].Participants)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestParseLinkedInInboxBadJSON verifies a non-JSON response yields no items
|
||||||
|
// rather than a panic or error.
|
||||||
|
func TestParseLinkedInInboxBadJSON(t *testing.T) {
|
||||||
|
if got := parseLinkedInInbox("Session expired"); len(got) != 0 {
|
||||||
|
t.Fatalf("expected no items for non-JSON, got %d", len(got))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestParseLinkedInConversation verifies message extraction from the sections
|
||||||
|
// blob (testdata/linkedin_conversation.json — synthetic Alice/Bob).
|
||||||
|
func TestParseLinkedInConversation(t *testing.T) {
|
||||||
|
text := readFixture(t, "linkedin_conversation.json")
|
||||||
|
msgs := parseLinkedInConversation(text)
|
||||||
|
if len(msgs) != 2 {
|
||||||
|
t.Fatalf("expected 2 messages, got %d", len(msgs))
|
||||||
|
}
|
||||||
|
|
||||||
|
if msgs[0].From != "Alice Example" {
|
||||||
|
t.Errorf("from=%q, want Alice Example", msgs[0].From)
|
||||||
|
}
|
||||||
|
if !strings.Contains(msgs[0].Text, "Senior Software Engineer") {
|
||||||
|
t.Errorf("alice text missing role, got %q", msgs[0].Text)
|
||||||
|
}
|
||||||
|
if msgs[0].Date == "" {
|
||||||
|
t.Error("expected message date")
|
||||||
|
}
|
||||||
|
if msgs[1].From != "Bob Example" {
|
||||||
|
t.Errorf("from=%q, want Bob Example", msgs[1].From)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestParseLinkedInConversationEmpty verifies empty/non-JSON blobs parse to
|
||||||
|
// zero messages.
|
||||||
|
func TestParseLinkedInConversationEmpty(t *testing.T) {
|
||||||
|
if got := parseLinkedInConversation("no data here"); len(got) != 0 {
|
||||||
|
t.Fatalf("expected 0 messages, got %d", len(got))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLinkedInTimestamp verifies weekday+clock resolution to a recent UTC date.
|
||||||
|
func TestLinkedInTimestamp(t *testing.T) {
|
||||||
|
// The most recent Wednesday before/equal to "now".
|
||||||
|
ts := linkedInTimestamp("WEDNESDAY", "10:02 AM")
|
||||||
|
parsed, err := time.Parse(time.RFC3339, ts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unparseable timestamp %q: %v", ts, err)
|
||||||
|
}
|
||||||
|
if parsed.Weekday() != time.Wednesday {
|
||||||
|
t.Errorf("expected Wednesday, got %s", parsed.Weekday())
|
||||||
|
}
|
||||||
|
if parsed.Hour() != 10 || parsed.Minute() != 2 {
|
||||||
|
t.Errorf("expected 10:02, got %02d:%02d", parsed.Hour(), parsed.Minute())
|
||||||
|
}
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
diff := now.Sub(parsed)
|
||||||
|
if diff < 0 || diff > 7*24*time.Hour {
|
||||||
|
t.Errorf("timestamp %s is not within the last week of %s", parsed, now)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := linkedInTimestamp("MONDAY", "garbage"); got != "" {
|
||||||
|
t.Errorf("expected empty for bad clock, got %q", got)
|
||||||
|
}
|
||||||
|
if got := linkedInTimestamp("", "1:22 PM"); got != "13:22" {
|
||||||
|
t.Errorf("expected bare 13:22 for missing weekday, got %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Relative day headers must resolve to full dates, not bare clocks.
|
||||||
|
today := linkedInTimestamp("TODAY", "9:42 AM")
|
||||||
|
yp, err := time.Parse(time.RFC3339, today)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("TODAY unparseable %q: %v", today, err)
|
||||||
|
}
|
||||||
|
if yp.Year() != now.Year() || yp.Month() != now.Month() || yp.Day() != now.Day() {
|
||||||
|
t.Errorf("TODAY expected %v, got %v", now, yp)
|
||||||
|
}
|
||||||
|
yest := linkedInTimestamp("YESTERDAY", "3:00 PM")
|
||||||
|
yp, err = time.Parse(time.RFC3339, yest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("YESTERDAY unparseable %q: %v", yest, err)
|
||||||
|
}
|
||||||
|
if yp.Day() != now.AddDate(0, 0, -1).Day() {
|
||||||
|
t.Errorf("YESTERDAY expected day %d, got %d", now.AddDate(0, 0, -1).Day(), yp.Day())
|
||||||
|
}
|
||||||
|
|
||||||
|
// MON DD header (e.g. "JUN 25"): must resolve to a full date. The
|
||||||
|
// timestamp should fall within the current year (falling back to the
|
||||||
|
// prior year if the date would be in the future).
|
||||||
|
md := linkedInTimestamp("JUN 25", "10:48 AM")
|
||||||
|
mp, err := time.Parse(time.RFC3339, md)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("MON DD unparseable %q: %v", md, err)
|
||||||
|
}
|
||||||
|
if mp.Year() != now.Year() && mp.Year() != now.Year()-1 {
|
||||||
|
t.Errorf("JUN 25 expected year %d or %d, got %d", now.Year(), now.Year()-1, mp.Year())
|
||||||
|
}
|
||||||
|
if mp.Month() != time.June || mp.Day() != 25 {
|
||||||
|
t.Errorf("JUN 25 expected Jun 25, got %s %d", mp.Month(), mp.Day())
|
||||||
|
}
|
||||||
|
if mp.After(now) {
|
||||||
|
t.Errorf("JUN 25 resolved to the future: %s > %s", mp, now)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTransientLinkedInError verifies rate-limit errors are retryable but
|
||||||
|
// genuine failures are not.
|
||||||
|
func TestTransientLinkedInError(t *testing.T) {
|
||||||
|
retryable := []string{
|
||||||
|
"get_conversation error: Error calling tool 'get_conversation'",
|
||||||
|
"get_conversation error: Unexpected error in get_conversation: net::ERR_HTTP_RESPONSE_CODE_FAILURE",
|
||||||
|
"rpc error 503: rate limited",
|
||||||
|
"rpc error 429: too many requests",
|
||||||
|
"get_conversation: get_conversation error: This server still has a browser open on the profile.",
|
||||||
|
}
|
||||||
|
for _, msg := range retryable {
|
||||||
|
if !isTransientLinkedInError(msg) {
|
||||||
|
t.Errorf("expected %q to be transient", msg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
permanent := []string{
|
||||||
|
"get_inbox error: bad credentials",
|
||||||
|
"rpc error -32602: Invalid request parameters",
|
||||||
|
"unmarshal: unexpected end of JSON input",
|
||||||
|
}
|
||||||
|
for _, msg := range permanent {
|
||||||
|
if isTransientLinkedInError(msg) {
|
||||||
|
t.Errorf("expected %q to be permanent", msg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMsgLimitFor verifies the per-conversation message cap resolution.
|
||||||
|
func TestMsgLimitFor(t *testing.T) {
|
||||||
|
if got := msgLimitFor(0); got != 100 {
|
||||||
|
t.Errorf("expected default 100, got %d", got)
|
||||||
|
}
|
||||||
|
if got := msgLimitFor(5); got != 5 {
|
||||||
|
t.Errorf("expected 5, got %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestWedged verifies the browser-open failure is recognized as a wedge.
|
||||||
|
func TestWedged(t *testing.T) {
|
||||||
|
if !wedged(errors.New("get_conversation error: This server still has a browser open on the profile")) {
|
||||||
|
t.Error("expected wedged error to be recognized")
|
||||||
|
}
|
||||||
|
if wedged(errors.New("get_conversation error: bad thing")) {
|
||||||
|
t.Error("unexpected wedge detection")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestThreadIDFromURL verifies thread id extraction.
|
||||||
|
func TestThreadIDFromURL(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
url, want string
|
||||||
|
}{
|
||||||
|
{"/messaging/thread/2-abc123/", "2-abc123"},
|
||||||
|
{"/messaging/thread/2-abc123", "2-abc123"},
|
||||||
|
{"", ""},
|
||||||
|
{"/messaging/thread/", ""},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
got := threadIDFromURL(c.url)
|
||||||
|
if c.want == "" && validThreadID(got) {
|
||||||
|
t.Errorf("threadIDFromURL(%q) = %q, want empty", c.url, got)
|
||||||
|
}
|
||||||
|
if c.want != "" && got != c.want {
|
||||||
|
t.Errorf("threadIDFromURL(%q) = %q, want %q", c.url, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Executable
+155
@@ -0,0 +1,155 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""chats/refresh-linkedin-session - refresh LinkedIn MCP session from webtop CDP.
|
||||||
|
|
||||||
|
bin/chats/refresh-linkedin-session [--cdp URL] [--root DIR]
|
||||||
|
"""
|
||||||
|
|
||||||
|
Reads the current LinkedIn cookies out of the running Thorium browser in the
|
||||||
|
work-webtop container via CDP (Network.getAllCookies), copies the live browser
|
||||||
|
profile onto the source profile directory, and rewrites the portable
|
||||||
|
cookies.json + source-state.json that mcp-server-linkedin requires.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
refresh-linkedin-session [--cdp http://127.0.0.1:9222] [--root /var/tmp/liprofile]
|
||||||
|
[--container work-webtop] [--profile thorium-profile]
|
||||||
|
|
||||||
|
After the headless driver uses a copied profile, LinkedIn rotates the session
|
||||||
|
in that copy, so this must run before every sync.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
import websockets
|
||||||
|
|
||||||
|
|
||||||
|
def cdp_tab(ws_json):
|
||||||
|
for t in ws_json:
|
||||||
|
if t.get("webSocketDebuggerUrl"):
|
||||||
|
return t["webSocketDebuggerUrl"]
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
async def get_cookies(ws_url):
|
||||||
|
async with websockets.connect(ws_url, max_size=50_000_000) as ws:
|
||||||
|
await ws.send(json.dumps({"id": 1, "method": "Network.getAllCookies", "params": {}}))
|
||||||
|
resp = await ws.recv()
|
||||||
|
return json.loads(resp).get("result", {}).get("cookies", [])
|
||||||
|
|
||||||
|
|
||||||
|
def write_source_state(root, profile_dir):
|
||||||
|
# Reuse the linkedin-mcp-server session_state module to write a valid
|
||||||
|
# source-state.json (same schema the daemon reads).
|
||||||
|
try:
|
||||||
|
from linkedin_mcp_server.session_state import canonical, write_source_state
|
||||||
|
|
||||||
|
write_source_state(canonical(__import__("pathlib").Path(profile_dir)))
|
||||||
|
return
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# Fallback: minimal schema-compatible state.
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
state = {
|
||||||
|
"version": 1,
|
||||||
|
"source_runtime_id": "linux-amd64-host",
|
||||||
|
"login_generation": str(uuid.uuid4()),
|
||||||
|
"created_at": None,
|
||||||
|
"profile_path": profile_dir,
|
||||||
|
"cookies_path": os.path.join(root, "cookies.json"),
|
||||||
|
}
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
state["created_at"] = datetime.now(timezone.utc).isoformat()
|
||||||
|
with open(os.path.join(root, "source-state.json"), "w") as f:
|
||||||
|
json.dump(state, f, indent=2)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
args = sys.argv[1:]
|
||||||
|
cdp = "http://127.0.0.1:9222"
|
||||||
|
root = "/var/tmp/liprofile"
|
||||||
|
container = "work-webtop"
|
||||||
|
cprofile = "thorium-profile"
|
||||||
|
for i in range(0, len(args), 2):
|
||||||
|
k = args[i]
|
||||||
|
v = args[i + 1] if i + 1 < len(args) else ""
|
||||||
|
if k == "--cdp":
|
||||||
|
cdp = v
|
||||||
|
elif k == "--root":
|
||||||
|
root = v
|
||||||
|
elif k == "--container":
|
||||||
|
container = v
|
||||||
|
elif k == "--profile":
|
||||||
|
cprofile = v
|
||||||
|
|
||||||
|
profile_dir = os.path.join(root, "profile")
|
||||||
|
os.makedirs(profile_dir, exist_ok=True)
|
||||||
|
|
||||||
|
# 1. Clear stale daemon/browser locks so the server can claim the profile.
|
||||||
|
for lock in ("profile-claim.lock", "profile.lock", "daemon.lock", "lease.lock"):
|
||||||
|
p = os.path.join(root, lock)
|
||||||
|
if os.path.exists(p):
|
||||||
|
os.remove(p)
|
||||||
|
for name in os.listdir(profile_dir):
|
||||||
|
if name.startswith("Singleton"):
|
||||||
|
os.remove(os.path.join(profile_dir, name))
|
||||||
|
for name in os.listdir(root):
|
||||||
|
if name.startswith("invalid-state-"):
|
||||||
|
shutil.rmtree(os.path.join(root, name), ignore_errors=True)
|
||||||
|
|
||||||
|
# 1. Copy the live browser profile (cookies DB + Local State) so the
|
||||||
|
# session the driver launches carries the current login.
|
||||||
|
subprocess.run(
|
||||||
|
["docker", "cp", f"{container}:/config/{cprofile}/Default", os.path.join(profile_dir, "Default")],
|
||||||
|
check=True, capture_output=True,
|
||||||
|
)
|
||||||
|
subprocess.run(
|
||||||
|
["docker", "cp", f"{container}:/config/{cprofile}/Local State", os.path.join(profile_dir, "Local State")],
|
||||||
|
check=True, capture_output=True,
|
||||||
|
)
|
||||||
|
for lock in ("SingletonLock", "SingletonCookie", "SingletonSocket"):
|
||||||
|
p = os.path.join(profile_dir, lock)
|
||||||
|
if os.path.exists(p):
|
||||||
|
os.remove(p)
|
||||||
|
|
||||||
|
# 2. Pull the live cookies out of the running browser.
|
||||||
|
with urllib.request.urlopen(f"{cdp}/json", timeout=5) as r:
|
||||||
|
tabs = json.loads(r.read())
|
||||||
|
ws_url = cdp_tab(tabs)
|
||||||
|
if not ws_url:
|
||||||
|
sys.stderr.write("refresh-linkedin-session: no CDP tab\n")
|
||||||
|
sys.exit(1)
|
||||||
|
cookies = asyncio.run(get_cookies(ws_url))
|
||||||
|
|
||||||
|
li = [c for c in cookies if "linkedin" in c.get("domain", "")]
|
||||||
|
out = []
|
||||||
|
for c in li:
|
||||||
|
domain = c.get("domain", "")
|
||||||
|
if domain in (".www.linkedin.com", "www.linkedin.com"):
|
||||||
|
domain = ".linkedin.com"
|
||||||
|
out.append({
|
||||||
|
"name": c["name"],
|
||||||
|
"value": c["value"].strip('"'),
|
||||||
|
"domain": domain,
|
||||||
|
"path": c.get("path", "/"),
|
||||||
|
"expires": c.get("expires", -1),
|
||||||
|
"httpOnly": c.get("httpOnly", False),
|
||||||
|
"secure": c.get("secure", False),
|
||||||
|
"sameSite": c.get("sameSite", "None"),
|
||||||
|
})
|
||||||
|
with open(os.path.join(root, "cookies.json"), "w") as f:
|
||||||
|
json.dump(out, f, indent=2)
|
||||||
|
|
||||||
|
write_source_state(root, profile_dir)
|
||||||
|
sys.stderr.write(f"refresh-linkedin-session: {len(out)} cookies, profile refreshed\n")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -6,33 +6,39 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"strings"
|
"path/filepath"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
func checkLinkedInSession(userDataDir string) (bool, error) {
|
func checkLinkedInSession(userDataDir string) (bool, error) {
|
||||||
cmd := exec.Command("uvx", "mcp-server-linkedin@latest",
|
// Validate the source-session files without launching a browser. A full
|
||||||
"--user-data-dir", userDataDir,
|
// `--status` run spawns Chromium and loads /feed/, doubling the automation
|
||||||
"--no-auto-import",
|
// exposed to LinkedIn (429 rate limits) before the sync even starts.
|
||||||
"--status",
|
root := filepath.Dir(userDataDir)
|
||||||
)
|
sessionFiles := []string{
|
||||||
out, err := cmd.CombinedOutput()
|
filepath.Join(root, "source-state.json"),
|
||||||
if err != nil {
|
filepath.Join(root, "cookies.json"),
|
||||||
return true, fmt.Errorf("status check: %w\n%s", err, string(out))
|
filepath.Join(userDataDir, "Default", "Cookies"),
|
||||||
}
|
}
|
||||||
return !strings.Contains(string(out), "✅"), nil
|
for _, f := range sessionFiles {
|
||||||
|
if _, err := os.Stat(f); err != nil {
|
||||||
|
return true, fmt.Errorf("missing session file %s", f)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func runSyncLinkedIn(args []string) int {
|
func runSyncLinkedIn(args []string) int {
|
||||||
fs := flag.NewFlagSet("chats sync linkedin", flag.ContinueOnError)
|
fs := flag.NewFlagSet("chats sync linkedin", flag.ContinueOnError)
|
||||||
limit := fs.Int("limit", 0, "max messages per conversation (0 = all)")
|
limit := fs.Int("limit", 0, "max messages per conversation (0 = all)")
|
||||||
|
refresh := fs.Bool("refresh", false, "refresh session from live webtop browser before sync")
|
||||||
help := fs.Bool("help", false, "")
|
help := fs.Bool("help", false, "")
|
||||||
fs.SetOutput(os.Stderr)
|
fs.SetOutput(os.Stderr)
|
||||||
if err := fs.Parse(args); err != nil {
|
if err := fs.Parse(args); err != nil {
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
if *help {
|
if *help {
|
||||||
fmt.Fprintln(os.Stderr, "usage: chats sync linkedin [--limit N]")
|
fmt.Fprintln(os.Stderr, "usage: chats sync linkedin [--limit N] [--refresh]")
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -42,15 +48,21 @@ func runSyncLinkedIn(args []string) int {
|
|||||||
userDataDir = home + "/.linkedin-mcp/profile"
|
userDataDir = home + "/.linkedin-mcp/profile"
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check session first
|
if *refresh {
|
||||||
|
if code := refreshLinkedInSession(userDataDir); code != 0 {
|
||||||
|
return code
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check session files first (no browser launch).
|
||||||
loginNeeded, err := checkLinkedInSession(userDataDir)
|
loginNeeded, err := checkLinkedInSession(userDataDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "chats: linkedin status check: %v\n", err)
|
fmt.Fprintf(os.Stderr, "chats: linkedin status check: %v\n", err)
|
||||||
}
|
}
|
||||||
if loginNeeded {
|
if loginNeeded {
|
||||||
fmt.Fprintf(os.Stderr, "chats: LinkedIn session expired. Run:\n")
|
fmt.Fprintf(os.Stderr, "chats: LinkedIn session missing. Run:\n")
|
||||||
fmt.Fprintf(os.Stderr, " uvx mcp-server-linkedin@latest --user-data-dir %s --login\n", userDataDir)
|
fmt.Fprintf(os.Stderr, " chats sync linkedin --refresh\n")
|
||||||
fmt.Fprintf(os.Stderr, "Then retry 'chats sync linkedin'\n")
|
fmt.Fprintf(os.Stderr, "or point LINKEDIN_USER_DATA_DIR at a valid session\n")
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -67,3 +79,27 @@ func runSyncLinkedIn(args []string) int {
|
|||||||
fmt.Printf("chats sync linkedin: completed in %s\n", time.Since(start).Round(time.Millisecond))
|
fmt.Printf("chats sync linkedin: completed in %s\n", time.Since(start).Round(time.Millisecond))
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// refreshLinkedInSession re-syncs the LinkedIn source session from the live
|
||||||
|
// webtop browser via the vendored refresh-linkedin-session helper.
|
||||||
|
func refreshLinkedInSession(userDataDir string) int {
|
||||||
|
exe, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "chats: resolve executable: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
helper := filepath.Join(filepath.Dir(exe), "refresh-linkedin-session")
|
||||||
|
if _, err := os.Stat(helper); err != nil {
|
||||||
|
// Fall back to the source tree helper next to this command file.
|
||||||
|
helper = "bin/chats/refresh-linkedin-session"
|
||||||
|
}
|
||||||
|
root := filepath.Dir(userDataDir)
|
||||||
|
cmd := exec.Command(helper, "--root", root)
|
||||||
|
cmd.Stdout = os.Stderr
|
||||||
|
cmd.Stderr = os.Stderr
|
||||||
|
if err := cmd.Run(); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "chats: linkedin session refresh: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|||||||
+12
@@ -0,0 +1,12 @@
|
|||||||
|
{
|
||||||
|
"url": "https://www.linkedin.com/messaging/thread/2-YWxpY2UtYm9iLXRocmVhZC0xMjM=/",
|
||||||
|
"sections": {
|
||||||
|
"conversation": "WEDNESDAY\nAlice Example sent the following message at 10:02 AM\nView Alice Example's profile\nAlice Example (She/Her) 10:02 AM\nHi Bob, we have a Senior Software Engineer role that matches your Go and Python background. Happy to share more if you are open to a chat.\n\nBob Example sent the following messages at 1:22 PM\nView Bob Example's profile\nBob Example 1:22 PM\nHi Alice, thanks for reaching out — yes, I am open to exploring a Senior Software Engineer role. Happy to do a short video call.\n"
|
||||||
|
},
|
||||||
|
"references": {
|
||||||
|
"conversation": [
|
||||||
|
{"kind": "person", "url": "/in/alice-example/", "text": "Alice Example"},
|
||||||
|
{"kind": "person", "url": "/in/bob-example/", "text": "Bob Example"}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
+34
@@ -0,0 +1,34 @@
|
|||||||
|
{
|
||||||
|
"url": "https://www.linkedin.com/messaging/",
|
||||||
|
"sections": {
|
||||||
|
"inbox": "Messaging\nInbox\nConversation List\nAlice Example\nExciting opportunity for a senior software engineer\n"
|
||||||
|
},
|
||||||
|
"references": {
|
||||||
|
"inbox": [
|
||||||
|
{
|
||||||
|
"kind": "conversation",
|
||||||
|
"url": "/messaging/thread/2-YWxpY2UtYm9iLXRocmVhZC0xMjM=/",
|
||||||
|
"context": "inbox",
|
||||||
|
"text": "Alice Example"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"kind": "person",
|
||||||
|
"url": "/in/alice-example/",
|
||||||
|
"text": "Alice Example",
|
||||||
|
"context": "inbox"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"kind": "conversation",
|
||||||
|
"url": "/messaging/thread/2-Y2hhcmxpZS1ib2ItdGhyZWFkLTQ1Ng==/",
|
||||||
|
"context": "inbox",
|
||||||
|
"text": "Charlie Example"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"kind": "conversation",
|
||||||
|
"url": "/messaging/thread/",
|
||||||
|
"context": "inbox",
|
||||||
|
"text": "should-be-skipped"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
+23
-20
@@ -1,34 +1,37 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# bin/kb/search - Go deduction search over the brain (model served by daemon).
|
# bin/kb/search — deprecated wrapper. Use bin/brain/search.go.
|
||||||
# Builds the kbsearch binary on first run / when source changes, then execs it.
|
# Sets CGO for ladybug, builds a binary (embed daemon needs a real executable),
|
||||||
|
# then execs it. Prints one deprecation line.
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
KB="$(cd "$(dirname "$0")/../.." && pwd)"
|
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||||
BIN="$KB/var/bin/kbsearch"
|
BIN="$ROOT/var/bin/brain-search"
|
||||||
SRC="$KB/bin/kbsearch"
|
SRC="$ROOT/internal/brain"
|
||||||
|
CMD="$ROOT/bin/brain"
|
||||||
|
|
||||||
mkdir -p "$KB/var/bin"
|
mkdir -p "$ROOT/var/bin"
|
||||||
|
|
||||||
# Rebuild if binary missing or any .go source newer
|
|
||||||
need_build=0
|
need_build=0
|
||||||
if [ ! -x "$BIN" ]; then
|
if [ ! -x "$BIN" ]; then
|
||||||
need_build=1
|
need_build=1
|
||||||
else
|
else
|
||||||
# Check if any .go in kbsearch is newer than binary
|
while IFS= read -r -d '' f; do
|
||||||
while IFS= read -r -d '' f; do
|
if [ "$f" -nt "$BIN" ]; then
|
||||||
if [ "$f" -nt "$BIN" ]; then
|
need_build=1
|
||||||
need_build=1
|
break
|
||||||
break
|
fi
|
||||||
fi
|
done < <(find "$SRC" "$CMD" -name '*.go' -print0 2>/dev/null)
|
||||||
done < <(find "$SRC" -name '*.go' -print0 2>/dev/null)
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ "$need_build" -eq 1 ]; then
|
if [ "$need_build" -eq 1 ]; then
|
||||||
echo "Building kbsearch..." >&2
|
echo "Building brain/search..." >&2
|
||||||
(cd "$SRC" && \
|
(
|
||||||
CGO_CFLAGS="-I$KB/lib-ladybug" \
|
cd "$ROOT" &&
|
||||||
CGO_LDFLAGS="-L$KB/lib-ladybug -Wl,-rpath,$KB/lib-ladybug" \
|
CGO_CFLAGS="-I$ROOT/lib-ladybug" \
|
||||||
go build -tags system_ladybug -o "$BIN" .) || exit 1
|
CGO_LDFLAGS="-L$ROOT/lib-ladybug -Wl,-rpath,$ROOT/lib-ladybug" \
|
||||||
|
go build -tags system_ladybug -o "$BIN" ./bin/brain
|
||||||
|
) || exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
echo "bin/kb/search is deprecated; use bin/brain/search.go" >&2
|
||||||
exec "$BIN" "$@"
|
exec "$BIN" "$@"
|
||||||
@@ -1,23 +0,0 @@
|
|||||||
module github.com/eSlider/2dph/bin/kbsearch
|
|
||||||
|
|
||||||
go 1.26.0
|
|
||||||
|
|
||||||
require (
|
|
||||||
github.com/LadybugDB/go-ladybug v0.17.0
|
|
||||||
github.com/chewxy/math32 v1.11.2
|
|
||||||
github.com/daulet/tokenizers v1.27.0
|
|
||||||
)
|
|
||||||
|
|
||||||
require (
|
|
||||||
github.com/apache/arrow-go/v18 v18.6.0 // indirect
|
|
||||||
github.com/goccy/go-json v0.10.6 // indirect
|
|
||||||
github.com/google/flatbuffers v25.12.19+incompatible // indirect
|
|
||||||
github.com/google/uuid v1.6.0 // indirect
|
|
||||||
github.com/klauspost/compress v1.18.5 // indirect
|
|
||||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
|
||||||
github.com/pierrec/lz4/v4 v4.1.26 // indirect
|
|
||||||
github.com/shopspring/decimal v1.4.0 // indirect
|
|
||||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
|
||||||
golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect
|
|
||||||
golang.org/x/sys v0.43.0 // indirect
|
|
||||||
)
|
|
||||||
@@ -1,44 +0,0 @@
|
|||||||
github.com/LadybugDB/go-ladybug v0.17.0 h1:RXDbkBjrbRmLdEbhGl4CLOIEzSt09gbP0n9UbKDEfwI=
|
|
||||||
github.com/LadybugDB/go-ladybug v0.17.0/go.mod h1:GeIXmE8XyF5TFS94NAuTag7vgCC+no/HTBMRA6Rd5Cs=
|
|
||||||
github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro=
|
|
||||||
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
|
||||||
github.com/apache/arrow-go/v18 v18.6.0 h1:GX/Jyd3R7mCLiECAwY9FWbbaYblie2WXBSz4Sw8fNpM=
|
|
||||||
github.com/apache/arrow-go/v18 v18.6.0/go.mod h1:gm3MiPpY82fLYK5VKPB3WoJbsiLVDfT7flD5/vHReKw=
|
|
||||||
github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc=
|
|
||||||
github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g=
|
|
||||||
github.com/chewxy/math32 v1.11.2 h1:IufN08Zwr1NKuWfY+4Tz55BcwKmyKKNdOP7KtumehnM=
|
|
||||||
github.com/chewxy/math32 v1.11.2/go.mod h1:dOB2rcuFrCn6UHrze36WSLVPKtzPMRAQvBvUwkSsLqs=
|
|
||||||
github.com/daulet/tokenizers v1.27.0 h1:MmFYAEDFz69s/nNQfHg59DWqHz3v94m99kEZ/JbL+s4=
|
|
||||||
github.com/daulet/tokenizers v1.27.0/go.mod h1:YjFY1o1HGMyWkQgbXJDghhvke/yFDp2vGdIO2hYs4MQ=
|
|
||||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
|
||||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
|
||||||
github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU=
|
|
||||||
github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
|
||||||
github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs=
|
|
||||||
github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
|
|
||||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
|
||||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
|
||||||
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
|
|
||||||
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
|
||||||
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
|
||||||
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
|
||||||
github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY=
|
|
||||||
github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
|
|
||||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
|
||||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
|
||||||
github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
|
|
||||||
github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
|
|
||||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
|
||||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
|
||||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
|
||||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
|
||||||
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
|
||||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
|
||||||
golang.org/x/exp v0.0.0-20260112195511-716be5621a96 h1:Z/6YuSHTLOHfNFdb8zVZomZr7cqNgTJvA8+Qz75D8gU=
|
|
||||||
golang.org/x/exp v0.0.0-20260112195511-716be5621a96/go.mod h1:nzimsREAkjBCIEFtHiYkrJyT+2uy9YZJB7H1k68CXZU=
|
|
||||||
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
|
|
||||||
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
|
||||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
|
||||||
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
|
||||||
@@ -1,44 +0,0 @@
|
|||||||
// bin/kbsearch - the Go implementation of bin/kb/search (nested module so the
|
|
||||||
// root `go test ./...` and CI never compile it against native ladyships).
|
|
||||||
//
|
|
||||||
// Usage (built/run by ./bin/kb/search):
|
|
||||||
//
|
|
||||||
// kbsearch "query" [--root facts|info] [--repo P] [-n N] [--json]
|
|
||||||
// kbsearch serve [port] start the embedding daemon
|
|
||||||
// kbsearch --list-model print the resolved model dir
|
|
||||||
//
|
|
||||||
// The potion-multilingual model is loaded only in `serve`; a CLI reuses the
|
|
||||||
// daemon over localhost HTTP (falling back to in-process embedding).
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"log"
|
|
||||||
"os"
|
|
||||||
"strconv"
|
|
||||||
)
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
if len(os.Args) > 1 && os.Args[1] == "serve" {
|
|
||||||
port := 17830
|
|
||||||
if len(os.Args) > 2 {
|
|
||||||
if p, err := strconv.Atoi(os.Args[2]); err == nil {
|
|
||||||
port = p
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := serve(port); err != nil {
|
|
||||||
log.Fatalf("kbsearch serve: %v", err)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if len(os.Args) > 1 && os.Args[1] == "--list-model" {
|
|
||||||
dir, err := modelDir()
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintln(os.Stderr, err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
fmt.Println(dir)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
os.Exit(runSearch(os.Args[1:]))
|
|
||||||
}
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
// Common types and helpers for kbsearch.
|
|
||||||
package main
|
|
||||||
|
|
||||||
import "os"
|
|
||||||
|
|
||||||
func eps() string { return os.Getenv("KBTEST_EPS") }
|
|
||||||
|
|
||||||
// Hit is one search result, mirroring the python script's dict shape.
|
|
||||||
type Hit struct {
|
|
||||||
ID string `json:"id"`
|
|
||||||
Text string `json:"text"`
|
|
||||||
Root string `json:"root"`
|
|
||||||
Source string `json:"-"` // for repo filtering, not in output
|
|
||||||
Score float64 `json:"score"`
|
|
||||||
Snippet string `json:"snippet,omitempty"`
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
"""D14 layout: bin/{subject}/{method}.go, libs in internal/, one go.mod."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|
||||||
|
|
||||||
|
class BinLayoutTest(unittest.TestCase):
|
||||||
|
def test_brain_search_shebang_exists(self) -> None:
|
||||||
|
p = ROOT / "bin" / "brain" / "search.go"
|
||||||
|
self.assertTrue(p.is_file(), "missing bin/brain/search.go")
|
||||||
|
first = p.read_text().splitlines()[0]
|
||||||
|
self.assertTrue(
|
||||||
|
first.startswith("//usr/bin/env go run"),
|
||||||
|
f"shebang first line, got {first!r}",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_no_nested_go_mod_under_bin(self) -> None:
|
||||||
|
nested = list((ROOT / "bin").rglob("go.mod"))
|
||||||
|
self.assertEqual(nested, [], f"nested go.mod files: {nested}")
|
||||||
|
|
||||||
|
def test_rank_lives_in_internal_brain(self) -> None:
|
||||||
|
self.assertTrue(
|
||||||
|
(ROOT / "internal" / "brain" / "rank" / "rank.go").is_file(),
|
||||||
|
"ranking must live in internal/brain/rank (cgo-free)",
|
||||||
|
)
|
||||||
|
self.assertFalse(
|
||||||
|
(ROOT / "bin" / "kbsearch").exists(),
|
||||||
|
"bin/kbsearch nested module must be gone",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_no_main_go_under_bin_brain(self) -> None:
|
||||||
|
main = ROOT / "bin" / "brain" / "main.go"
|
||||||
|
self.assertFalse(main.exists(), "bin/brain/main.go is not a method")
|
||||||
@@ -6,5 +6,6 @@ Brain/ops/eSlider stack. Facts need proof or they are
|
|||||||
|
|
||||||
- [PLAN.md](../PLAN.md) — decisions, execution order, open questions (v2)
|
- [PLAN.md](../PLAN.md) — decisions, execution order, open questions (v2)
|
||||||
- [design](design.md) — schema, deduction model, sources
|
- [design](design.md) — schema, deduction model, sources
|
||||||
|
- [Gitea issues](https://git.produktor.io/eSlider/2dph/issues) — work board (origin)
|
||||||
|
|
||||||
Published docs live here and mirror the project state.
|
Published docs live here and mirror the project state.
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
# CRM association proof (oo CLI ↔ corpus)
|
||||||
|
|
||||||
|
Proven with `oo` (eslider/go-onlyoffice) against the OnlyOffice portal
|
||||||
|
(`office.produktor.io`). Portal CRM is the SSOT for company ↔ person ↔
|
||||||
|
project associations; the corpus SoT (`eslider/cv/projects/knowledge-mesh-seed.yaml`)
|
||||||
|
is the second, independent source. Facts that can be backed by both are
|
||||||
|
written to the brain under `root=facts` by `bin/facts/crm`.
|
||||||
|
|
||||||
|
## What was verified
|
||||||
|
|
||||||
|
- Logical counts (portal MySQL): 1300 contacts = 897 persons + 404 companies,
|
||||||
|
198 projects, 998 deals, 939 project↔contact links.
|
||||||
|
- Every client company linked to a project has ≥1 person underneath.
|
||||||
|
- Every person `company_id` resolves to an existing company.
|
||||||
|
- Corpus org list (9) maps 1:1 onto CRM companies:
|
||||||
|
ProProdukt SL / produktor.io, Dyvenia, Immowelt AG, WhereGroup,
|
||||||
|
Keynote SIGOS, D2S/SYSTEMS, GRID, Pack und Cup, Markets Platform.
|
||||||
|
- 78 person↔company association facts written to the brain
|
||||||
|
(`how=crm-crosscheck`, `type=association`). Recall@5 in `bin/kb/eval` = 1.0.
|
||||||
|
|
||||||
|
## Mistakes found
|
||||||
|
|
||||||
|
| # | Mistake | Fix |
|
||||||
|
|---|---------|-----|
|
||||||
|
| 1 | Duplicate legal entity `GoldenRatio.Exchange` (contact 759) vs `Golden Ratio Exchange` (763); 3 deals (211, 287, 559) were linked to 759 | `oo contacts merge 759 763` — 763 kept, 759 removed, deal links re-pointed to 763 |
|
||||||
|
| 2 | `env/`-wide: OnlyOffice creds file used wrong UX (user `eslider`, password with `$2` suffix) making `oo` auth fail | `.env` fixed to `eslider@gmail.com` + clean password; `.env` stays gitignored |
|
||||||
|
|
||||||
|
## Gates after fix
|
||||||
|
|
||||||
|
- `uv run python -m unittest discover -s bin/tools -t .` → 26 tests OK
|
||||||
|
- `bin/facts/audit self` + `bin/facts/audit db` → ok
|
||||||
|
- `bin/kb/eval` → recall@5 = 1.0
|
||||||
|
- `go test ./...` (bin/server + bin/watch) → ok
|
||||||
@@ -1,8 +1,25 @@
|
|||||||
module github.com/eSlider/2dph
|
module github.com/eSlider/2dph
|
||||||
|
|
||||||
go 1.25.0
|
go 1.26
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
github.com/LadybugDB/go-ladybug v0.17.0
|
||||||
github.com/arran4/golang-ical v0.3.5
|
github.com/arran4/golang-ical v0.3.5
|
||||||
|
github.com/chewxy/math32 v1.11.2
|
||||||
|
github.com/daulet/tokenizers v1.27.0
|
||||||
golang.org/x/text v0.40.0
|
golang.org/x/text v0.40.0
|
||||||
)
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/apache/arrow-go/v18 v18.6.0 // indirect
|
||||||
|
github.com/goccy/go-json v0.10.6 // indirect
|
||||||
|
github.com/google/flatbuffers v25.12.19+incompatible // indirect
|
||||||
|
github.com/google/uuid v1.6.0 // indirect
|
||||||
|
github.com/klauspost/compress v1.18.5 // indirect
|
||||||
|
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||||
|
github.com/pierrec/lz4/v4 v4.1.26 // indirect
|
||||||
|
github.com/shopspring/decimal v1.4.0 // indirect
|
||||||
|
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||||
|
golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect
|
||||||
|
golang.org/x/sys v0.43.0 // indirect
|
||||||
|
)
|
||||||
|
|||||||
@@ -1,14 +1,50 @@
|
|||||||
|
github.com/LadybugDB/go-ladybug v0.17.0 h1:RXDbkBjrbRmLdEbhGl4CLOIEzSt09gbP0n9UbKDEfwI=
|
||||||
|
github.com/LadybugDB/go-ladybug v0.17.0/go.mod h1:GeIXmE8XyF5TFS94NAuTag7vgCC+no/HTBMRA6Rd5Cs=
|
||||||
|
github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro=
|
||||||
|
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||||
|
github.com/apache/arrow-go/v18 v18.6.0 h1:GX/Jyd3R7mCLiECAwY9FWbbaYblie2WXBSz4Sw8fNpM=
|
||||||
|
github.com/apache/arrow-go/v18 v18.6.0/go.mod h1:gm3MiPpY82fLYK5VKPB3WoJbsiLVDfT7flD5/vHReKw=
|
||||||
|
github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc=
|
||||||
|
github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g=
|
||||||
github.com/arran4/golang-ical v0.3.5 h1:bbz6ld4dC+MmCKiFfOd6SkmIGnhNMBACZ485ULh7p9A=
|
github.com/arran4/golang-ical v0.3.5 h1:bbz6ld4dC+MmCKiFfOd6SkmIGnhNMBACZ485ULh7p9A=
|
||||||
github.com/arran4/golang-ical v0.3.5/go.mod h1:OnguFgjN0Hmx8jzpmWcC+AkHio94ujmLHKoaef7xQh8=
|
github.com/arran4/golang-ical v0.3.5/go.mod h1:OnguFgjN0Hmx8jzpmWcC+AkHio94ujmLHKoaef7xQh8=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/chewxy/math32 v1.11.2 h1:IufN08Zwr1NKuWfY+4Tz55BcwKmyKKNdOP7KtumehnM=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/chewxy/math32 v1.11.2/go.mod h1:dOB2rcuFrCn6UHrze36WSLVPKtzPMRAQvBvUwkSsLqs=
|
||||||
|
github.com/daulet/tokenizers v1.27.0 h1:MmFYAEDFz69s/nNQfHg59DWqHz3v94m99kEZ/JbL+s4=
|
||||||
|
github.com/daulet/tokenizers v1.27.0/go.mod h1:YjFY1o1HGMyWkQgbXJDghhvke/yFDp2vGdIO2hYs4MQ=
|
||||||
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||||
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU=
|
||||||
|
github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||||
|
github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs=
|
||||||
|
github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
|
||||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
|
||||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||||
|
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
||||||
|
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||||
|
github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY=
|
||||||
|
github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
|
||||||
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||||
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
|
||||||
|
github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
|
||||||
|
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||||
|
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||||
|
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||||
|
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||||
|
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||||
|
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||||
|
golang.org/x/exp v0.0.0-20260112195511-716be5621a96 h1:Z/6YuSHTLOHfNFdb8zVZomZr7cqNgTJvA8+Qz75D8gU=
|
||||||
|
golang.org/x/exp v0.0.0-20260112195511-716be5621a96/go.mod h1:nzimsREAkjBCIEFtHiYkrJyT+2uy9YZJB7H1k68CXZU=
|
||||||
|
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
|
||||||
|
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||||
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
|
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
|
||||||
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
|
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
|
||||||
|
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||||
|
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
|
|||||||
@@ -1,10 +1,12 @@
|
|||||||
// Brain connection management using go-ladybug.
|
//go:build cgo && system_ladybug
|
||||||
package main
|
|
||||||
|
package brain
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
lbug "github.com/LadybugDB/go-ladybug"
|
lbug "github.com/LadybugDB/go-ladybug"
|
||||||
)
|
)
|
||||||
@@ -44,10 +46,10 @@ func dbPath() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func openBrain() error {
|
func openBrain() error {
|
||||||
return openWithOpts(2, eps())
|
return openWithSandbox(eps())
|
||||||
}
|
}
|
||||||
|
|
||||||
func openWithOpts(allow int, epsv string) error {
|
func openWithSandbox(epsv string) error {
|
||||||
cfg := lbug.DefaultSystemConfig()
|
cfg := lbug.DefaultSystemConfig()
|
||||||
cfg.MaxNumThreads = 8
|
cfg.MaxNumThreads = 8
|
||||||
cfg.BufferPoolSize = 1 << 30 // 1GB
|
cfg.BufferPoolSize = 1 << 30 // 1GB
|
||||||
@@ -57,20 +59,30 @@ func openWithOpts(allow int, epsv string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("OpenDatabase: %w", err)
|
return fmt.Errorf("OpenDatabase: %w", err)
|
||||||
}
|
}
|
||||||
if epsv != "" {
|
|
||||||
if _, err := conn.Query("SET STREAM_SANDBOX = '" + epsv + "'"); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
conn, err = lbug.OpenConnection(db)
|
conn, err = lbug.OpenConnection(db)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
closeBrain()
|
||||||
return fmt.Errorf("OpenConnection: %w", err)
|
return fmt.Errorf("OpenConnection: %w", err)
|
||||||
}
|
}
|
||||||
|
// Session settings need a live connection; running this before
|
||||||
|
// OpenConnection dereferenced a nil *Connection.
|
||||||
|
if epsv != "" {
|
||||||
|
if strings.ContainsAny(epsv, "'\\") {
|
||||||
|
closeBrain()
|
||||||
|
return fmt.Errorf("SET STREAM_SANDBOX: invalid value")
|
||||||
|
}
|
||||||
|
if _, err := conn.Query("SET STREAM_SANDBOX = '" + epsv + "'"); err != nil {
|
||||||
|
closeBrain()
|
||||||
|
return fmt.Errorf("SET STREAM_SANDBOX: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
if _, err := conn.Query("LOAD EXTENSION FTS"); err != nil {
|
if _, err := conn.Query("LOAD EXTENSION FTS"); err != nil {
|
||||||
|
closeBrain()
|
||||||
return fmt.Errorf("LOAD EXTENSION FTS: %w", err)
|
return fmt.Errorf("LOAD EXTENSION FTS: %w", err)
|
||||||
}
|
}
|
||||||
if _, err := conn.Query("LOAD EXTENSION VECTOR"); err != nil {
|
if _, err := conn.Query("LOAD EXTENSION VECTOR"); err != nil {
|
||||||
|
closeBrain()
|
||||||
return fmt.Errorf("LOAD EXTENSION VECTOR: %w", err)
|
return fmt.Errorf("LOAD EXTENSION VECTOR: %w", err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
// Package brain is deduction search over Ladybug (FTS + HNSW).
|
||||||
|
// Query/embed code that needs cgo lives behind the system_ladybug tag.
|
||||||
|
package brain
|
||||||
@@ -1,9 +1,11 @@
|
|||||||
|
//go:build cgo && system_ladybug
|
||||||
|
|
||||||
// StaticModel wraps the potion-multilingual-128m embedding model.
|
// StaticModel wraps the potion-multilingual-128m embedding model.
|
||||||
//
|
//
|
||||||
// Mirrors model2vec.StaticModel: tokenizer (daulet Unigram) + safetensors matrix.
|
// Mirrors model2vec.StaticModel: tokenizer (daulet Unigram) + safetensors matrix.
|
||||||
// Embed(text) applies the same preprocessing: median_token_length pre-truncation,
|
// Embed(text) applies the same preprocessing: median_token_length pre-truncation,
|
||||||
// add_special_tokens=false, drop unk (id=1), truncate to 512, mean pool, L2 normalize +1e-32.
|
// add_special_tokens=false, drop unk (id=1), truncate to 512, mean pool, L2 normalize +1e-32.
|
||||||
package main
|
package brain
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
@@ -1,5 +1,4 @@
|
|||||||
// modelDir returns the resolved potion-multilingual-128m model directory.
|
package brain
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
package rank
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
const Usage = `usage: bin/brain/search.go "query" [--root facts|info] [--repo REPO] [-n N] [--json]
|
||||||
|
bin/brain/search.go serve [port]
|
||||||
|
bin/brain/search.go --list-model`
|
||||||
|
|
||||||
|
type Options struct {
|
||||||
|
Query string
|
||||||
|
Root string
|
||||||
|
Repo string
|
||||||
|
Limit int
|
||||||
|
JSONOut bool
|
||||||
|
ListModel bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseArgs reads flags. Unknown flags are an error: silently dropping them
|
||||||
|
// meant `--hop 1` vanished and its argument `1` was appended to the query.
|
||||||
|
// --hop is recognised so it cannot be swallowed; it is not implemented until
|
||||||
|
// File/FROM_FILE edges exist.
|
||||||
|
func ParseArgs(args []string) (Options, error) {
|
||||||
|
opt := Options{Limit: 20}
|
||||||
|
var queryArgs []string
|
||||||
|
|
||||||
|
for i := 0; i < len(args); i++ {
|
||||||
|
arg := args[i]
|
||||||
|
wantsValue := arg == "--root" || arg == "--repo" || arg == "-n" || arg == "--hop"
|
||||||
|
if wantsValue && i+1 >= len(args) {
|
||||||
|
return opt, fmt.Errorf("%s needs a value", arg)
|
||||||
|
}
|
||||||
|
switch arg {
|
||||||
|
case "--root":
|
||||||
|
i++
|
||||||
|
opt.Root = args[i]
|
||||||
|
if opt.Root != "facts" && opt.Root != "info" {
|
||||||
|
return opt, fmt.Errorf("--root must be facts or info, got %q", opt.Root)
|
||||||
|
}
|
||||||
|
case "--repo":
|
||||||
|
i++
|
||||||
|
opt.Repo = args[i]
|
||||||
|
case "-n":
|
||||||
|
i++
|
||||||
|
n, err := strconv.Atoi(args[i])
|
||||||
|
if err != nil || n < 1 {
|
||||||
|
return opt, fmt.Errorf("-n must be a positive integer, got %q", args[i])
|
||||||
|
}
|
||||||
|
opt.Limit = n
|
||||||
|
case "--hop":
|
||||||
|
return opt, fmt.Errorf("--hop is not implemented yet (needs File/FROM_FILE edges)")
|
||||||
|
case "--json":
|
||||||
|
opt.JSONOut = true
|
||||||
|
case "--list-model":
|
||||||
|
opt.ListModel = true
|
||||||
|
default:
|
||||||
|
if strings.HasPrefix(arg, "-") {
|
||||||
|
return opt, fmt.Errorf("unknown flag %q", arg)
|
||||||
|
}
|
||||||
|
queryArgs = append(queryArgs, arg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
opt.Query = strings.TrimSpace(strings.Join(queryArgs, " "))
|
||||||
|
if opt.Query == "" && !opt.ListModel {
|
||||||
|
return opt, fmt.Errorf("no query given")
|
||||||
|
}
|
||||||
|
return opt, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
package rank
|
||||||
|
|
||||||
|
// BM25 ranks best-first, so the top hits are the *highest* scores; cosine
|
||||||
|
// distance ranks best-first ascending. Both mirror kblib.py.
|
||||||
|
const FTSStmt = "CALL QUERY_FTS_INDEX('Leaf', 'id', $q) " +
|
||||||
|
"RETURN node.id, node.text, node.root, node.source, score ORDER BY score DESC LIMIT $n"
|
||||||
|
|
||||||
|
const VecStmt = "CALL QUERY_VECTOR_INDEX('Leaf', 'Leaf_vec', $q, $n) " +
|
||||||
|
"RETURN node.id, node.text, node.root, node.source, distance ORDER BY distance LIMIT $n"
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
// Package rank is the cgo-free ranking and CLI parsing for brain search.
|
||||||
|
// CI can `go test ./rank` without the native ladybug library.
|
||||||
|
package rank
|
||||||
|
|
||||||
|
import (
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Hit is one search result, mirroring the python script's dict shape.
|
||||||
|
type Hit struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Text string `json:"text"`
|
||||||
|
Root string `json:"root"`
|
||||||
|
Source string `json:"-"`
|
||||||
|
Score float64 `json:"score"`
|
||||||
|
Snippet string `json:"snippet,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// rrfK dampens the contribution of low ranks; same constant as kblib.py.
|
||||||
|
const rrfK = 60
|
||||||
|
|
||||||
|
// RankAndFilter fuses the two hit lists, applies --root/--repo, then cuts to
|
||||||
|
// limit. Cutting first dropped every matching leaf ranked below the cut, so
|
||||||
|
// `--root facts` came back empty whenever info leafs filled the top N.
|
||||||
|
// limit <= 0 keeps everything.
|
||||||
|
func RankAndFilter(fts, vec []Hit, root, repo string, limit int) []Hit {
|
||||||
|
out := Hybrid(fts, vec, 0)
|
||||||
|
if root != "" {
|
||||||
|
out = FilterRoot(out, root)
|
||||||
|
}
|
||||||
|
if repo != "" {
|
||||||
|
out = FilterRepo(out, repo)
|
||||||
|
}
|
||||||
|
if limit > 0 && len(out) > limit {
|
||||||
|
out = out[:limit]
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Hybrid merges FTS and vector hits by reciprocal rank fusion.
|
||||||
|
// limit <= 0 returns the full fused list.
|
||||||
|
func Hybrid(fts, vec []Hit, limit int) []Hit {
|
||||||
|
byID := make(map[string]Hit, len(fts)+len(vec))
|
||||||
|
rrf := make(map[string]float64, len(fts)+len(vec))
|
||||||
|
|
||||||
|
for i, h := range fts {
|
||||||
|
byID[h.ID] = h
|
||||||
|
rrf[h.ID] += 1.0 / (rrfK + float64(i+1))
|
||||||
|
}
|
||||||
|
for i, h := range vec {
|
||||||
|
if existing, ok := byID[h.ID]; !ok {
|
||||||
|
byID[h.ID] = h
|
||||||
|
} else if existing.Score == 0 {
|
||||||
|
existing.Score = h.Score
|
||||||
|
byID[h.ID] = existing
|
||||||
|
}
|
||||||
|
rrf[h.ID] += 1.0 / (rrfK + float64(i+1))
|
||||||
|
}
|
||||||
|
|
||||||
|
ids := make([]string, 0, len(rrf))
|
||||||
|
for id := range rrf {
|
||||||
|
ids = append(ids, id)
|
||||||
|
}
|
||||||
|
sort.Slice(ids, func(i, j int) bool {
|
||||||
|
if rrf[ids[i]] != rrf[ids[j]] {
|
||||||
|
return rrf[ids[i]] > rrf[ids[j]]
|
||||||
|
}
|
||||||
|
return ids[i] < ids[j]
|
||||||
|
})
|
||||||
|
if limit > 0 && len(ids) > limit {
|
||||||
|
ids = ids[:limit]
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make([]Hit, 0, len(ids))
|
||||||
|
for _, id := range ids {
|
||||||
|
out = append(out, byID[id])
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func FilterRoot(hits []Hit, root string) []Hit {
|
||||||
|
var out []Hit
|
||||||
|
for _, h := range hits {
|
||||||
|
if h.Root == root {
|
||||||
|
out = append(out, h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func FilterRepo(hits []Hit, repo string) []Hit {
|
||||||
|
var out []Hit
|
||||||
|
for _, h := range hits {
|
||||||
|
if strings.Contains(h.Source, repo) {
|
||||||
|
out = append(out, h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,149 @@
|
|||||||
|
// Unit tests for ranking/filtering and CLI parsing (no db, no model, offline).
|
||||||
|
package rank
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func h(id, root, source string) Hit {
|
||||||
|
return Hit{ID: id, Text: id, Root: root, Source: source}
|
||||||
|
}
|
||||||
|
|
||||||
|
func ids(hits []Hit) []string {
|
||||||
|
out := make([]string, len(hits))
|
||||||
|
for i, hit := range hits {
|
||||||
|
out[i] = hit.ID
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func eq(t *testing.T, got []Hit, want ...string) {
|
||||||
|
t.Helper()
|
||||||
|
g := ids(got)
|
||||||
|
if len(g) != len(want) {
|
||||||
|
t.Fatalf("got %v, want %v", g, want)
|
||||||
|
}
|
||||||
|
for i := range want {
|
||||||
|
if g[i] != want[i] {
|
||||||
|
t.Fatalf("got %v, want %v", g, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A facts leaf that ranks below the limit in the unfiltered list must still
|
||||||
|
// be returned for --root facts. Filtering after truncation loses it.
|
||||||
|
func TestRankAndFilterFiltersBeforeLimit(t *testing.T) {
|
||||||
|
fts := []Hit{
|
||||||
|
h("i1", "info", "docs/a.md"),
|
||||||
|
h("i2", "info", "docs/b.md"),
|
||||||
|
h("i3", "info", "docs/c.md"),
|
||||||
|
h("f1", "facts", "docker ps x compose"),
|
||||||
|
}
|
||||||
|
eq(t, RankAndFilter(fts, nil, "facts", "", 2), "f1")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRankAndFilterRepoFiltersBeforeLimit(t *testing.T) {
|
||||||
|
fts := []Hit{
|
||||||
|
h("a", "info", "eSlider/2dph:README.md"),
|
||||||
|
h("b", "info", "eSlider/2dph:PLAN.md"),
|
||||||
|
h("c", "info", "eSlider/ops:compose.yaml"),
|
||||||
|
}
|
||||||
|
eq(t, RankAndFilter(fts, nil, "", "ops", 2), "c")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRankAndFilterTruncatesToLimit(t *testing.T) {
|
||||||
|
fts := []Hit{h("a", "info", "x"), h("b", "info", "x"), h("c", "info", "x")}
|
||||||
|
eq(t, RankAndFilter(fts, nil, "", "", 2), "a", "b")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRankAndFilterLimitZeroKeepsAll(t *testing.T) {
|
||||||
|
fts := []Hit{h("a", "info", "x"), h("b", "info", "x")}
|
||||||
|
eq(t, RankAndFilter(fts, nil, "", "", 0), "a", "b")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHybridFusesBothRetrievers(t *testing.T) {
|
||||||
|
fts := []Hit{h("only-fts", "info", "x"), h("both", "info", "x")}
|
||||||
|
vec := []Hit{h("only-vec", "info", "x"), h("both", "info", "x")}
|
||||||
|
eq(t, Hybrid(fts, vec, 0), "both", "only-fts", "only-vec")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHybridTiesAreDeterministic(t *testing.T) {
|
||||||
|
fts := []Hit{h("b", "info", "x"), h("a", "info", "x")}
|
||||||
|
first := ids(Hybrid(fts, nil, 0))
|
||||||
|
for i := 0; i < 50; i++ {
|
||||||
|
got := ids(Hybrid(fts, nil, 0))
|
||||||
|
for j := range first {
|
||||||
|
if got[j] != first[j] {
|
||||||
|
t.Fatalf("unstable order: %v then %v", first, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHybridKeepsVectorScoreForSharedHit(t *testing.T) {
|
||||||
|
fts := []Hit{{ID: "x", Root: "info", Score: 0}}
|
||||||
|
vec := []Hit{{ID: "x", Root: "info", Score: 0.87}}
|
||||||
|
got := Hybrid(fts, vec, 0)
|
||||||
|
if len(got) != 1 || got[0].Score != 0.87 {
|
||||||
|
t.Fatalf("got %+v, want score 0.87", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The old parser dropped unknown flags and appended their arguments to the
|
||||||
|
// query, so `search "q" --hop 1` searched for "q 1". --hop is not implemented
|
||||||
|
// here (needs File edges); it must still fail closed instead of changing q.
|
||||||
|
func TestParseHopIsNotSwallowedIntoTheQuery(t *testing.T) {
|
||||||
|
_, err := ParseArgs([]string{"what runs on arc-2", "--hop", "1"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected --hop to error (not implemented), not be swallowed")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "--hop") {
|
||||||
|
t.Fatalf("error should name --hop, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseRejectsUnknownFlags(t *testing.T) {
|
||||||
|
if _, err := ParseArgs([]string{"query", "--nope"}); err == nil {
|
||||||
|
t.Fatal("unknown flag accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseRejectsBadValues(t *testing.T) {
|
||||||
|
for _, args := range [][]string{
|
||||||
|
{"q", "-n", "zero"},
|
||||||
|
{"q", "-n", "0"},
|
||||||
|
{"q", "--root", "nonsense"},
|
||||||
|
{"q", "--hop"},
|
||||||
|
{"--json"},
|
||||||
|
} {
|
||||||
|
if _, err := ParseArgs(args); err == nil {
|
||||||
|
t.Errorf("accepted %v", args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseDefaults(t *testing.T) {
|
||||||
|
opt, err := ParseArgs([]string{"two", "words", "--json"})
|
||||||
|
if err != nil || opt.Query != "two words" || opt.Limit != 20 || !opt.JSONOut {
|
||||||
|
t.Fatalf("got %+v err=%v", opt, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestListModelNeedsNoQuery(t *testing.T) {
|
||||||
|
if _, err := ParseArgs([]string{"--list-model"}); err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUsageNamesBrainSearch(t *testing.T) {
|
||||||
|
if !strings.Contains(Usage, "bin/brain/search.go") {
|
||||||
|
t.Fatalf("usage must name bin/brain/search.go, got:\n%s", Usage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFTSQueryOrdersByScoreDescending(t *testing.T) {
|
||||||
|
if !strings.Contains(FTSStmt, "ORDER BY score DESC") {
|
||||||
|
t.Fatalf("FTS query must order by score DESC, got:\n%s", FTSStmt)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
// Hybrid FTS + vector search implementation, plus daemon client/server.
|
//go:build cgo && system_ladybug
|
||||||
package main
|
|
||||||
|
package brain
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
@@ -13,12 +14,12 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
lbug "github.com/LadybugDB/go-ladybug"
|
lbug "github.com/LadybugDB/go-ladybug"
|
||||||
|
"github.com/eSlider/2dph/internal/brain/rank"
|
||||||
)
|
)
|
||||||
|
|
||||||
const defaultPort = 17830
|
const defaultPort = 17830
|
||||||
@@ -26,45 +27,15 @@ const daemonPath = "/embed"
|
|||||||
const healthPath = "/health"
|
const healthPath = "/health"
|
||||||
|
|
||||||
func runSearch(args []string) int {
|
func runSearch(args []string) int {
|
||||||
// Manual flag parsing to allow flags after query (like Python argparse)
|
opt, err := rank.ParseArgs(args)
|
||||||
root := ""
|
if err != nil {
|
||||||
repo := ""
|
fmt.Fprintf(os.Stderr, "brain/search: %v\n%s\n", err, rank.Usage)
|
||||||
limit := 20
|
return 2
|
||||||
jsonOut := false
|
|
||||||
listModel := false
|
|
||||||
|
|
||||||
var queryArgs []string
|
|
||||||
for i := 0; i < len(args); i++ {
|
|
||||||
switch args[i] {
|
|
||||||
case "--root":
|
|
||||||
if i+1 < len(args) {
|
|
||||||
root = args[i+1]
|
|
||||||
i++
|
|
||||||
}
|
|
||||||
case "--repo":
|
|
||||||
if i+1 < len(args) {
|
|
||||||
repo = args[i+1]
|
|
||||||
i++
|
|
||||||
}
|
|
||||||
case "-n":
|
|
||||||
if i+1 < len(args) {
|
|
||||||
if n, err := strconv.Atoi(args[i+1]); err == nil {
|
|
||||||
limit = n
|
|
||||||
}
|
|
||||||
i++
|
|
||||||
}
|
|
||||||
case "--json":
|
|
||||||
jsonOut = true
|
|
||||||
case "--list-model":
|
|
||||||
listModel = true
|
|
||||||
default:
|
|
||||||
if !strings.HasPrefix(args[i], "-") {
|
|
||||||
queryArgs = append(queryArgs, args[i])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
root, repo, limit, query := opt.Root, opt.Repo, opt.Limit, opt.Query
|
||||||
|
jsonOut := opt.JSONOut
|
||||||
|
|
||||||
if listModel {
|
if opt.ListModel {
|
||||||
dir, err := modelDir()
|
dir, err := modelDir()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintln(os.Stderr, err)
|
fmt.Fprintln(os.Stderr, err)
|
||||||
@@ -74,12 +45,6 @@ func runSearch(args []string) int {
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
query := strings.TrimSpace(strings.Join(queryArgs, " "))
|
|
||||||
if query == "" {
|
|
||||||
fmt.Fprintln(os.Stderr, "usage: kbsearch \"query\" [--root facts|info] [--repo REPO] [-n N] [--json]")
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := openBrain(); err != nil {
|
if err := openBrain(); err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "open brain: %v\n", err)
|
fmt.Fprintf(os.Stderr, "open brain: %v\n", err)
|
||||||
return 1
|
return 1
|
||||||
@@ -103,17 +68,7 @@ func runSearch(args []string) int {
|
|||||||
fmt.Fprintf(os.Stderr, "vec: %v\n", err)
|
fmt.Fprintf(os.Stderr, "vec: %v\n", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
results := hybrid(fts, vec, limit)
|
results := rank.RankAndFilter(fts, vec, root, repo, limit)
|
||||||
|
|
||||||
if root != "" {
|
|
||||||
results = filterRoot(results, root)
|
|
||||||
}
|
|
||||||
if repo != "" {
|
|
||||||
results = filterRepo(results, repo)
|
|
||||||
}
|
|
||||||
if len(results) > limit {
|
|
||||||
results = results[:limit]
|
|
||||||
}
|
|
||||||
|
|
||||||
for i := range results {
|
for i := range results {
|
||||||
if results[i].Text != "" {
|
if results[i].Text != "" {
|
||||||
@@ -150,10 +105,7 @@ func b2i(err error) int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func queryFTS(text string, limit int) ([]Hit, error) {
|
func queryFTS(text string, limit int) ([]Hit, error) {
|
||||||
stmt, err := conn.Prepare(
|
stmt, err := conn.Prepare(rank.FTSStmt)
|
||||||
"CALL QUERY_FTS_INDEX('Leaf', 'id', $q) " +
|
|
||||||
"RETURN node.id, node.text, node.root, node.source, score ORDER BY score LIMIT $n",
|
|
||||||
)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -170,10 +122,7 @@ func queryVector(emb []float64, limit int) ([]Hit, error) {
|
|||||||
for i, v := range emb {
|
for i, v := range emb {
|
||||||
embList[i] = v
|
embList[i] = v
|
||||||
}
|
}
|
||||||
stmt, err := conn.Prepare(
|
stmt, err := conn.Prepare(rank.VecStmt)
|
||||||
"CALL QUERY_VECTOR_INDEX('Leaf', 'Leaf_vec', $q, $n) " +
|
|
||||||
"RETURN node.id, node.text, node.root, node.source, distance ORDER BY distance LIMIT $n",
|
|
||||||
)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -215,10 +164,10 @@ func rowsToHits(res *lbug.QueryResult) ([]Hit, error) {
|
|||||||
|
|
||||||
// JSON output types
|
// JSON output types
|
||||||
type jsonOut struct {
|
type jsonOut struct {
|
||||||
Query string `json:"query"`
|
Query string `json:"query"`
|
||||||
RootFilter string `json:"root_filter"`
|
RootFilter string `json:"root_filter"`
|
||||||
Count int `json:"count"`
|
Count int `json:"count"`
|
||||||
Results []jsonHit `json:"results"`
|
Results []jsonHit `json:"results"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type jsonHit struct {
|
type jsonHit struct {
|
||||||
@@ -248,70 +197,6 @@ func toJSONOut(hits []Hit, query, rootFilter string) *jsonOut {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func hybrid(fts, vec []Hit, limit int) []Hit {
|
|
||||||
byID := make(map[string]Hit)
|
|
||||||
rrf := make(map[string]float64)
|
|
||||||
|
|
||||||
for rank, h := range fts {
|
|
||||||
byID[h.ID] = h
|
|
||||||
rrf[h.ID] += 1.0 / (60 + float64(rank+1))
|
|
||||||
}
|
|
||||||
for rank, h := range vec {
|
|
||||||
if _, ok := byID[h.ID]; !ok {
|
|
||||||
byID[h.ID] = h
|
|
||||||
} else {
|
|
||||||
existing := byID[h.ID]
|
|
||||||
if existing.Score == 0 {
|
|
||||||
existing.Score = h.Score
|
|
||||||
byID[h.ID] = existing
|
|
||||||
}
|
|
||||||
}
|
|
||||||
rrf[h.ID] += 1.0 / (60 + float64(rank+1))
|
|
||||||
}
|
|
||||||
|
|
||||||
type scored struct {
|
|
||||||
id string
|
|
||||||
rrf float64
|
|
||||||
}
|
|
||||||
var scoredList []scored
|
|
||||||
for id, v := range rrf {
|
|
||||||
scoredList = append(scoredList, scored{id, v})
|
|
||||||
}
|
|
||||||
sort.Slice(scoredList, func(i, j int) bool {
|
|
||||||
return scoredList[i].rrf > scoredList[j].rrf
|
|
||||||
})
|
|
||||||
|
|
||||||
var out []Hit
|
|
||||||
for i, s := range scoredList {
|
|
||||||
if i >= limit {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
h := byID[s.id]
|
|
||||||
out = append(out, h)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func filterRoot(hits []Hit, root string) []Hit {
|
|
||||||
var out []Hit
|
|
||||||
for _, h := range hits {
|
|
||||||
if h.Root == root {
|
|
||||||
out = append(out, h)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func filterRepo(hits []Hit, repo string) []Hit {
|
|
||||||
var out []Hit
|
|
||||||
for _, h := range hits {
|
|
||||||
if strings.Contains(h.Source, repo) {
|
|
||||||
out = append(out, h)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func resultsToDicts(hits []Hit) []any {
|
func resultsToDicts(hits []Hit) []any {
|
||||||
out := make([]any, len(hits))
|
out := make([]any, len(hits))
|
||||||
for i, h := range hits {
|
for i, h := range hits {
|
||||||
@@ -363,7 +248,7 @@ func serve(port int) error {
|
|||||||
})
|
})
|
||||||
|
|
||||||
addr := fmt.Sprintf("127.0.0.1:%d", port)
|
addr := fmt.Sprintf("127.0.0.1:%d", port)
|
||||||
log.Printf("kbsearch daemon listening on %s", addr)
|
log.Printf("brain search daemon listening on %s", addr)
|
||||||
return http.ListenAndServe(addr, mux)
|
return http.ListenAndServe(addr, mux)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -382,10 +267,16 @@ func embedQuery(text string) ([]float64, error) {
|
|||||||
port = p
|
port = p
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
emb, err := tryDaemon(text, port)
|
if emb, err := tryDaemon(text, port); err == nil {
|
||||||
if err == nil {
|
|
||||||
return emb, nil
|
return emb, nil
|
||||||
}
|
}
|
||||||
|
if os.Getenv("KBSEARCH_NO_DAEMON") == "" {
|
||||||
|
if err := ensureDaemon(port); err == nil {
|
||||||
|
if emb, err := tryDaemon(text, port); err == nil {
|
||||||
|
return emb, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
model, err := loadModel()
|
model, err := loadModel()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -447,9 +338,13 @@ func ensureDaemon(port int) error {
|
|||||||
cmd.Dir, _ = filepath.Split(self)
|
cmd.Dir, _ = filepath.Split(self)
|
||||||
cmd.Stdout = nil
|
cmd.Stdout = nil
|
||||||
cmd.Stderr = nil
|
cmd.Stderr = nil
|
||||||
|
cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
|
||||||
if err := cmd.Start(); err != nil {
|
if err := cmd.Start(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if cmd.Process != nil {
|
||||||
|
_ = cmd.Process.Release()
|
||||||
|
}
|
||||||
|
|
||||||
for i := 0; i < 40; i++ {
|
for i := 0; i < 40; i++ {
|
||||||
time.Sleep(250 * time.Millisecond)
|
time.Sleep(250 * time.Millisecond)
|
||||||
@@ -463,3 +358,21 @@ func ensureDaemon(port int) error {
|
|||||||
}
|
}
|
||||||
return fmt.Errorf("daemon failed to start on port %d", port)
|
return fmt.Errorf("daemon failed to start on port %d", port)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Main is the bin/brain/search.go entry: search, serve, or --list-model.
|
||||||
|
func Main(args []string) int {
|
||||||
|
if len(args) > 0 && args[0] == "serve" {
|
||||||
|
port := defaultPort
|
||||||
|
if len(args) > 1 {
|
||||||
|
if p, err := strconv.Atoi(args[1]); err == nil {
|
||||||
|
port = p
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := serve(port); err != nil {
|
||||||
|
log.Printf("brain/search serve: %v", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return runSearch(args)
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
package brain
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/eSlider/2dph/internal/brain/rank"
|
||||||
|
)
|
||||||
|
|
||||||
|
func eps() string { return os.Getenv("KBTEST_EPS") }
|
||||||
|
|
||||||
|
// Hit is the search hit type; ranking lives in package rank so CI can test
|
||||||
|
// it without the native ladybug library.
|
||||||
|
type Hit = rank.Hit
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
// YAML emitter ported from bin/kb/yamlout.py — preserves insertion order.
|
// YAML emitter ported from bin/kb/yamlout.py — preserves insertion order.
|
||||||
package main
|
package brain
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
Reference in New Issue
Block a user