From a3310424886acfded6e6f41b30003aceab54c189 Mon Sep 17 00:00:00 2001 From: Andrey Oblivantsev Date: Fri, 14 Aug 2026 11:32:23 +0100 Subject: [PATCH] feat: DuckDB quantiles in-process (gcc CGO), not Ladybug. (#34) OQ3: internal/duckstats + bin/qa/stats.go. Zig stays Ladybug-only. mikefarah/yq for small structured slices. Gitea #30. --- .github/workflows/ci.yml | 6 +- .gitignore | 2 + AGENTS.md | 8 +- PLAN.md | 7 +- bin/qa/stats.go | 73 +++++ bin/reasoner/bakeoff.go | 13 +- bin/tools/test_bin_layout.py | 26 ++ bin/tools/test_published_docs.py | 2 +- bin/tools/test_skills.py | 14 + bin/tools/test_system_perf.py | 36 +++ compose.yaml | 37 ++- deploy/picoclaw/config.json | 33 +++ docs/README.md | 2 +- docs/picoclaw.md | 28 +- docs/reasoner.md | 6 +- docs/roadmap.md | 12 +- go.mod | 8 + go.sum | 16 ++ internal/duckstats/duckstats.go | 47 ++++ internal/duckstats/duckstats_test.go | 51 ++++ internal/reasoner/client.go | 20 +- qa/system_perf.py | 257 ++++++++++++++++++ skills/duckdb/SKILL.md | 38 +++ skills/picoclaw/SKILL.md | 10 +- skills/postgres/SKILL.md | 2 +- skills/web-search/SKILL.md | 2 +- .../web-search/reference/instance-tuning.md | 2 +- skills/yq/SKILL.md | 33 +++ 28 files changed, 745 insertions(+), 46 deletions(-) create mode 100755 bin/qa/stats.go create mode 100644 bin/tools/test_system_perf.py create mode 100644 deploy/picoclaw/config.json create mode 100644 internal/duckstats/duckstats.go create mode 100644 internal/duckstats/duckstats_test.go create mode 100755 qa/system_perf.py create mode 100644 skills/duckdb/SKILL.md create mode 100644 skills/yq/SKILL.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 95ea8b9..dc4b618 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,10 +45,10 @@ jobs: run: | uv run python -m unittest discover -s bin/tools -t . - - name: Go tests (root module, no ladybug cgo) + - name: Go tests (root module; duckdb-go CGO via gcc, no ladybug) run: | - go vet ./... - go test ./... -count=1 + CC=gcc CXX=g++ CGO_CFLAGS= CGO_LDFLAGS= go vet ./... + CC=gcc CXX=g++ CGO_CFLAGS= CGO_LDFLAGS= go test ./... -count=1 - name: brain ranking tests (no cgo / no ladybug) run: go test ./internal/brain/rank -count=1 diff --git a/.gitignore b/.gitignore index 70bfb12..001fb10 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ __pycache__/ lib-ladybug/ go.work.local models/ +# Purged from git history. Do not re-add. +docs/crm-associations-proof.md diff --git a/AGENTS.md b/AGENTS.md index 7c65691..d04c932 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,7 +48,8 @@ bin/postgres/ query.go (read-only YAML) bin/git/ import.go (go-git history; Python shim execs it) bin/web/ search.go (SearXNG; Python shim execs it) bin/reasoner/ bakeoff.go (D18 CPU OpenAI tool-call bake-off) -internal/ shared Go (brain/rank is cgo-free; chats parsers; gitlog; websearch; reasoner) +internal/ shared Go (brain/rank is cgo-free; chats parsers; gitlog; websearch; reasoner; duckstats) +bin/qa/ stats.go (DuckDB quantiles / JSONL count; gcc CGO, not Zig) bin/watch/ corpus watcher (used by bin/brain/watch.go) bin/tools/ vendored python libs behind bin/* (kblib, yamlout, websearch) bin/cgo/ zig zcc zc++ (CGO via zig cc, not gcc) @@ -101,13 +102,16 @@ bin/git/import.go [REPO] [--json] [--limit N] # go-git history → commit le bin/web/search.go "query" [--json] # SearXNG; throttled ≠ absence bin/reasoner/bakeoff.go [--model ID] [--json] # D18 CPU tool-call bake-off bin/postgres/query.go --profile onlyoffice -c 'SELECT 1' +bin/qa/stats.go # D22 DuckDB quantiles / JSONL (gcc CGO) bin/mail/ocr.go # tesseract eng+deu (scans) bin/md/tables # what the graph holds → YAML bin/brain/deduce "question" # thinking wrapper ``` Never start a shell command with `cd` — use the tool working-directory -parameter. Search before reading whole files. +parameter. Search before reading whole files. For YAML/JSON/XML/CSV/TOML/HCL +prefer mikefarah/yq (`skills/yq/SKILL.md`). For bulk rows and quantiles use +duckdb-go (`internal/duckstats`, `skills/duckdb/SKILL.md`), not Ladybug. ## GitHub safety rules (ABSOLUTE — never violate) diff --git a/PLAN.md b/PLAN.md index 5305cc3..ca62089 100644 --- a/PLAN.md +++ b/PLAN.md @@ -48,6 +48,7 @@ detective method: **a fact needs ≥2 independent sources or it is | D19 | git history | [go-git](https://github.com/go-git/go-git) via `bin/git/import.go`. No subprocess of the git binary. Conversion prints commit leafs; brain write is `bin/brain/index.go`. | | D20 | agent API | OpenAPI + MCP are generated from the same `internal/httpapi.Ops` table as `bin/brain/serve.go` handlers. `GET /openapi.json`, `POST /mcp` (JSON-RPC tools/list + tools/call). Tool names match OpenAPI paths (`search`/`get`/`stats`/`audit`/`ingest`). | | D21 | CGO | Ladybug/tokenizers CGO is compiled with **Zig** (`bin/cgo/zcc` → `zig cc -target …-linux-gnu`), not gcc. `bin/cgo/zig` pins Zig 0.14.1 + liblbug 0.19.1 + libtokenizers 1.27.0. Compose `target: api` has no CPython; write/rebuild is profile `index`. | +| D22 | analytics | **duckdb-go** in-process (`internal/duckstats`, `bin/qa/stats.go`) for quantiles/JSONL. Links with **gcc/g++**, not Zig. Ladybug stays the graph; web-search cache stays modernc sqlite. Slice small structured docs with **mikefarah/yq**, not kislyuk/jq. [#30](https://git.produktor.io/eSlider/2dph/issues/30). | ## Architecture @@ -110,7 +111,7 @@ Common props on every node/edge: `root`, `confidence`, `evidence[]`, `how`, - `bin/{subject}/{method}` — line 2 is a usage comment (mirrors `psql-yq`). - bash + python primary; golang via Go shebang when a compiled helper is right. -- YAML default output, `--json` for machines. Slice with `yq`. +- YAML default output, `--json` for machines. Slice with mikefarah/yq. - Everything that touches the network / DB is read-only, throttled, cached. - Tests (TDD) gate every commit; `gh` + CI/CD on every push. @@ -122,8 +123,8 @@ Common props on every node/edge: `root`, `confidence`, `evidence[]`, `how`, `eng+deu` (`bin/mail/ocr.go`, `internal/ocr`). No gocv, no gosseract CGO (D21 Zig owns Ladybug CGO). Optional `OCR_ENGINE=paddle` / compose profile `ocr-paddle`. Docling left the default path. [#6](https://git.produktor.io/eSlider/2dph/issues/6). -- OQ3: optional duckdb-md layer for `SELECT … FORMAT MARKDOWN` export/write-back. - [#30](https://git.produktor.io/eSlider/2dph/issues/30). +- OQ3: **in** — duckdb-go (`internal/duckstats`, `bin/qa/stats.go`) for + quantiles / JSONL count. Not a second graph. [#30](https://git.produktor.io/eSlider/2dph/issues/30). - OQ4: YAML-first storage for leafs — deferred: JSON is ~10x faster to serialize and unambiguous; YAML only where humans edit files. diff --git a/bin/qa/stats.go b/bin/qa/stats.go new file mode 100755 index 0000000..65d9365 --- /dev/null +++ b/bin/qa/stats.go @@ -0,0 +1,73 @@ +//usr/bin/env go run -tags=qa_stats "$0" "$@"; exit +//go:build qa_stats +// +// bin/qa/stats.go - DuckDB quantiles over a JSON number array or JSONL count. +// +// ./bin/qa/stats.go <<< '[1,2,3,4,5]' +// ./bin/qa/stats.go --jsonl rows.jsonl +// +// NOTE: never run `gofmt -w` on this file — it breaks the shebang. +// DuckDB CGO needs gcc/g++ (not Zig). After eval "$(bin/cgo/zig env)": +// CC=gcc CXX=g++ CGO_CFLAGS= CGO_LDFLAGS= ./bin/qa/stats.go +package main + +import ( + "encoding/json" + "fmt" + "io" + "os" + "strings" + + "github.com/eSlider/2dph/internal/duckstats" +) + +func main() { + os.Exit(run(os.Args[1:])) +} + +func run(args []string) int { + jsonl := "" + for i := 0; i < len(args); i++ { + a := args[i] + switch { + case a == "--jsonl" && i+1 < len(args): + i++ + jsonl = args[i] + case strings.HasPrefix(a, "--jsonl="): + jsonl = strings.TrimPrefix(a, "--jsonl=") + case a == "-h" || a == "--help": + fmt.Fprintln(os.Stderr, "bin/qa/stats.go [--jsonl FILE] # stdin = JSON [float,…]") + return 0 + default: + fmt.Fprintln(os.Stderr, "unknown arg:", a) + return 2 + } + } + if jsonl != "" { + n, err := duckstats.CountJSONL(jsonl) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + fmt.Printf("n: %d\n", n) + return 0 + } + raw, err := io.ReadAll(os.Stdin) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + var samples []float64 + if err := json.Unmarshal(raw, &samples); err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + s, err := duckstats.Quantiles(samples) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + fmt.Printf("n: %d\nmin: %g\np50: %g\np95: %g\nmax: %g\navg: %g\n", + s.N, s.Min, s.P50, s.P95, s.Max, s.Avg) + return 0 +} diff --git a/bin/reasoner/bakeoff.go b/bin/reasoner/bakeoff.go index 5e345a5..6fa44b7 100755 --- a/bin/reasoner/bakeoff.go +++ b/bin/reasoner/bakeoff.go @@ -7,7 +7,7 @@ // ./bin/reasoner/bakeoff.go --model MichelRosselli/bonsai-27b:Q1_0 --json // // Measures OpenAI tool_calls (search/get/audit) and RSS from Ollama /api/ps, not VRAM. -// PicoClaw is not in this repo; the tool names match internal/httpapi MCP ops. +// PicoClaw is compose profile picoclaw; tool names match internal/httpapi MCP ops. // NOTE: never run `gofmt -w` on this file — it breaks the shebang. package main @@ -17,6 +17,7 @@ import ( "os" "strings" + "github.com/eSlider/2dph/internal/duckstats" "github.com/eSlider/2dph/internal/reasoner" ) @@ -61,6 +62,14 @@ func run(args []string) int { } c := reasoner.Client{BaseURL: base, Model: model, Device: device} rep := reasoner.Run(c) + lat := make([]float64, 0, len(rep.Prompts)) + for _, p := range rep.Prompts { + lat = append(lat, float64(p.LatencyMS)) + } + if st, err := duckstats.Quantiles(lat); err == nil { + rep.LatencyP50MS = st.P50 + rep.LatencyP95MS = st.P95 + } raw, err := json.MarshalIndent(rep, "", " ") if err != nil { fmt.Fprintln(os.Stderr, err) @@ -76,6 +85,8 @@ func run(args []string) int { fmt.Printf("xml_leak: %d\n", rep.XMLLeak) fmt.Printf("rss_mb: %d\n", rep.RSSMB) fmt.Printf("vram_mb: %d\n", rep.VRAMMB) + fmt.Printf("latency_p50_ms: %g\n", rep.LatencyP50MS) + fmt.Printf("latency_p95_ms: %g\n", rep.LatencyP95MS) for _, p := range rep.Prompts { status := "fail" if p.OK { diff --git a/bin/tools/test_bin_layout.py b/bin/tools/test_bin_layout.py index 04010d7..07b7a51 100644 --- a/bin/tools/test_bin_layout.py +++ b/bin/tools/test_bin_layout.py @@ -217,6 +217,32 @@ class BinLayoutTest(unittest.TestCase): if "go-git/go-git" in line: self.assertNotIn("indirect", line) + def test_duckdb_go_is_direct_require(self) -> None: + text = (ROOT / "go.mod").read_text() + first = text.split("require (")[1].split(")")[0] + self.assertRegex(first, r"github.com/duckdb/duckdb-go/v2\s+v") + for line in first.splitlines(): + if "duckdb/duckdb-go" in line: + self.assertNotIn("indirect", line) + skill = (ROOT / "skills" / "duckdb" / "SKILL.md").read_text() + self.assertIn("github.com/duckdb/duckdb-go", skill) + self.assertIn("Ladybug", skill) + self.assertIn("sqlite", skill.lower()) + self.assertIn("gcc", skill.lower()) + self.assertIn("Zig", skill) + plan = (ROOT / "PLAN.md").read_text() + self.assertIn("D22", plan) + self.assertIn("duckdb-go", plan) + self._assert_shebang("bin/qa/stats.go") + reasoner = (ROOT / "internal" / "reasoner" / "client.go").read_text() + self.assertNotIn("duckdb", reasoner) + self.assertNotIn("duckstats", reasoner) + bakeoff = (ROOT / "bin" / "reasoner" / "bakeoff.go").read_text() + self.assertIn("internal/duckstats", bakeoff) + webcache = (ROOT / "internal" / "websearch" / "cache.go").read_text() + self.assertNotIn("duckdb", webcache) + self.assertIn("modernc.org/sqlite", webcache) + def test_cgo_uses_zig_not_gcc(self) -> None: for rel in ("bin/cgo/zig", "bin/cgo/zcc", "bin/cgo/zc++"): p = ROOT / rel diff --git a/bin/tools/test_published_docs.py b/bin/tools/test_published_docs.py index 458fe90..536a3ae 100644 --- a/bin/tools/test_published_docs.py +++ b/bin/tools/test_published_docs.py @@ -121,7 +121,7 @@ class PublishedDocsTest(unittest.TestCase): self.assertIn("D18", plan) self.assertIn("Qwen/Qwen3.5-9B", plan) compose = (ROOT / "compose.yaml").read_text() - self.assertIn('profiles: ["reasoner"]', compose) + self.assertIn('"reasoner"', compose) self.assertIn("OLLAMA_NUM_GPU", compose) self.assertIn("127.0.0.1:11435", compose) dockerfile = (ROOT / "Dockerfile").read_text() diff --git a/bin/tools/test_skills.py b/bin/tools/test_skills.py index 9539816..5245f49 100644 --- a/bin/tools/test_skills.py +++ b/bin/tools/test_skills.py @@ -47,3 +47,17 @@ class SkillsTest(unittest.TestCase): self.assertIn("throttled", skill.lower()) self.assertIn("not a negative finding", agents) self.assertIn("Fact-check every", agents) + + def test_yq_is_mikefarah_for_structured_data(self) -> None: + skill = (ROOT / "skills" / "yq" / "SKILL.md").read_text() + self.assertIn("https://github.com/mikefarah/yq", skill) + for fmt in ("YAML", "JSON", "XML", "CSV", "TOML", "HCL"): + self.assertIn(fmt, skill) + self.assertIn("not kislyuk", skill.lower()) + plan = (ROOT / "PLAN.md").read_text() + self.assertIn("mikefarah/yq", plan) + agents = (ROOT / "AGENTS.md").read_text() + self.assertIn("mikefarah/yq", agents) + web = (ROOT / "skills" / "web-search" / "SKILL.md").read_text() + self.assertIn("| yq ", web) + self.assertNotIn("| jq ", web) diff --git a/bin/tools/test_system_perf.py b/bin/tools/test_system_perf.py new file mode 100644 index 0000000..65c21d8 --- /dev/null +++ b/bin/tools/test_system_perf.py @@ -0,0 +1,36 @@ +"""qa/system_perf.py is an offline-gated system test (no live brain in CI).""" +from __future__ import annotations + +import ast +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + + +class SystemPerfScriptTest(unittest.TestCase): + def test_script_compiles_and_is_read_only(self) -> None: + path = ROOT / "qa" / "system_perf.py" + src = path.read_text() + compile(src, str(path), "exec") + self.assertIn("--json", src) + self.assertIn("qwen3.5:9b", src) + self.assertIn("--picoclaw", src) + self.assertIn("BRAIN_URL", src) + self.assertIn("tools/list", src) + self.assertIn("tools/call", src) + self.assertIn("GATE_HEALTH_MS", src) + self.assertIn("GATE_GET_P50_MS", src) + self.assertNotIn("kb.lbug", src) + self.assertNotIn("password", src.lower()) + self.assertNotIn("token", src.lower()) + + def test_script_does_not_write_ladybug(self) -> None: + tree = ast.parse((ROOT / "qa" / "system_perf.py").read_text()) + writes = [ + n.func.attr + for n in ast.walk(tree) + if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute) + and n.func.attr in {"write_text", "write_bytes", "dump"} + ] + self.assertEqual(writes, [], f"system_perf must not write files: {writes}") diff --git a/compose.yaml b/compose.yaml index 707d73f..4d1b3d0 100644 --- a/compose.yaml +++ b/compose.yaml @@ -2,7 +2,7 @@ # # docker compose up -d brain # API (Zig CGO serve) # docker compose --profile index run --rm index # Python rebuild -# docker compose --profile picoclaw up brain-mcp +# docker compose --profile picoclaw up -d # brain-mcp + CPU reasoner + PicoClaw gateway # docker compose --profile reasoner up -d reasoner # CPU Ollama :11435 # docker compose --profile searxng up -d # OCR_ENGINE=paddle docker compose --profile ocr-paddle run --rm ocr-paddle @@ -11,6 +11,14 @@ name: 2dph +networks: + default: + name: 2dph_sys + driver: bridge + ipam: + config: + - subnet: 10.23.42.0/24 + services: brain: image: ghcr.io/eslider/2dph:api @@ -98,8 +106,8 @@ services: - ./deploy/searxng/limiter.toml:/etc/searxng/limiter.toml:ro restart: unless-stopped - # MCP endpoint for an external agent (PicoClaw is not shipped here). - # docker compose --profile picoclaw up brain-mcp + # MCP endpoint for PicoClaw (and any MCP client). + # docker compose --profile picoclaw up -d brain-mcp: profiles: ["picoclaw"] image: ghcr.io/eslider/2dph:api @@ -121,7 +129,7 @@ services: # docker compose --profile reasoner up -d reasoner # docker compose --profile reasoner exec reasoner ollama pull qwen3.5:9b reasoner: - profiles: ["reasoner"] + profiles: ["reasoner", "picoclaw"] image: docker.io/ollama/ollama:latest environment: OLLAMA_NUM_GPU: "0" @@ -132,6 +140,26 @@ services: - reasoner-ollama:/root/.ollama restart: unless-stopped + # Official PicoClaw gateway. Config has no secrets (Ollama + HTTP MCP). + # Host network: brain/reasoner bind 127.0.0.1 only, so host.docker.internal + # (docker0) cannot reach them. Gateway 127.0.0.1:18790 (not the 18800 launcher). + # If :8630/:11435 are already bound, do not start brain-mcp/reasoner: + # docker compose --profile picoclaw up -d --no-deps picoclaw + picoclaw: + profiles: ["picoclaw"] + image: docker.io/sipeed/picoclaw:v0.3.1 + network_mode: host + depends_on: + - brain-mcp + - reasoner + environment: + PICOCLAW_GATEWAY_HOST: "127.0.0.1" + entrypoint: ["picoclaw", "gateway"] + volumes: + - picoclaw-home:/root/.picoclaw + - ./deploy/picoclaw/config.json:/root/.picoclaw/config.json:ro + restart: unless-stopped + # Optional PP-OCRv5 (not default). Default OCR is tesseract eng+deu. # OCR_ENGINE=paddle docker compose --profile ocr-paddle run --rm ocr-paddle ocr-paddle: @@ -145,3 +173,4 @@ volumes: kb-model: kb-var: reasoner-ollama: + picoclaw-home: diff --git a/deploy/picoclaw/config.json b/deploy/picoclaw/config.json new file mode 100644 index 0000000..ada5d4b --- /dev/null +++ b/deploy/picoclaw/config.json @@ -0,0 +1,33 @@ +{ + "agents": { + "defaults": { + "model_name": "qwen3.5-9b", + "max_tool_iterations": 8, + "max_tokens": 512, + "context_window": 8192 + } + }, + "model_list": [ + { + "model_name": "qwen3.5-9b", + "model": "ollama/qwen3.5:9b", + "api_base": "http://127.0.0.1:11435/v1", + "request_timeout": 600 + } + ], + "tools": { + "web": { + "enabled": false + }, + "mcp": { + "enabled": true, + "servers": { + "2dph": { + "enabled": true, + "type": "http", + "url": "http://127.0.0.1:8630/mcp" + } + } + } + } +} diff --git a/docs/README.md b/docs/README.md index d3f3762..6a8e18f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -20,7 +20,7 @@ Evidence-first knowledge graph. Facts need proof or they are | explanation | [roadmap](roadmap.md) — gap to v1 (epic #16) | | howto | [picoclaw](picoclaw.md) — MCP agent profile | | howto | [reasoner](reasoner.md) — CPU bake-off (D18) | -| reference | [PLAN.md](../PLAN.md) — decisions D1–D21 | +| reference | [PLAN.md](../PLAN.md) — decisions D1–D22 | Decisions the public face must name: **D3** SearXNG compose, **D6** Go service / Python write sidecar, **D14** `bin/{subject}/{method}.go`, **D15** Gitea origin, diff --git a/docs/picoclaw.md b/docs/picoclaw.md index 06ad39b..91f77d0 100644 --- a/docs/picoclaw.md +++ b/docs/picoclaw.md @@ -1,18 +1,34 @@ # PicoClaw profile (reference agent) -2dph is the memory/fact gate. PicoClaw (or any MCP client) is the agent loop -and is **not** shipped in this repo. +2dph is the memory/fact gate. Compose profile `picoclaw` runs the official +PicoClaw gateway (`docker.io/sipeed/picoclaw:v0.3.1`) plus `brain-mcp` and the +CPU reasoner. Default agent model is `qwen3.5:9b` (RAM path, D18). Weights stay +in the reasoner volume, not in the 2dph image. +No secrets in git: Ollama needs no key; MCP is local HTTP. ```bash -docker compose --profile picoclaw up brain-mcp +docker compose --profile picoclaw up -d +# already serving :8630 / :11435: +docker compose --profile picoclaw up -d --no-deps picoclaw ``` -The API listens on `127.0.0.1:8630`. Point the agent at -`http://127.0.0.1:8630/mcp` using [deploy/picoclaw/mcp.json.example](../deploy/picoclaw/mcp.json.example). +Gateway: `127.0.0.1:18790`. Brain MCP: `http://127.0.0.1:8630/mcp`. +Cursor-style clients can use [deploy/picoclaw/mcp.json.example](../deploy/picoclaw/mcp.json.example). +PicoClaw itself uses [deploy/picoclaw/config.json](../deploy/picoclaw/config.json) +(`127.0.0.1` + host network — loopback publishes are not reachable via docker0). OpenAPI: `GET http://127.0.0.1:8630/openapi.json`. Before a factual reply: `search` → `get` → `audit`. `throttled` is not a negative finding. See `skills/picoclaw/SKILL.md`. -No Cursor required. A live PicoClaw binary/image is an operator choice. +System performance (MCP gates + qwen3.5:9b tool_call + PicoClaw gateway): + +```bash +./qa/system_perf.py --json | yq '.gates' +REASONER_MODEL=qwen3.5:9b ./qa/system_perf.py --reasoner --picoclaw --json | yq '.reasoner' +``` + +The default agent model is `qwen3.5:9b`. PicoClaw `context_window` is 8192 +(heuristic `max_tokens*4` at 512 is 2048, too small for MCP tool schemas). +`request_timeout` is 600s for a CPU turn (tool_call + MCP search + answer). diff --git a/docs/reasoner.md b/docs/reasoner.md index ba4903c..dab945f 100644 --- a/docs/reasoner.md +++ b/docs/reasoner.md @@ -1,8 +1,8 @@ # Reasoner bake-off (D18) Pluggable OpenAI-compatible URL. 2dph does not ship weights. PicoClaw is -not in this repo; the bake-off hits the same tool names PicoClaw would -(`search` → `get` → `audit` from `internal/httpapi.Ops`). +compose profile `picoclaw` (`sipeed/picoclaw`); the bake-off hits the same +tool names (`search` → `get` → `audit` from `internal/httpapi.Ops`). ```bash docker compose --profile reasoner up -d reasoner @@ -11,6 +11,8 @@ REASONER_BASE_URL=http://127.0.0.1:11435/v1 REASONER_MODEL=qwen3.5:9b \ ./bin/reasoner/bakeoff.go --json ``` +JSON includes `latency_p50_ms` / `latency_p95_ms` from DuckDB (`internal/duckstats`, D22). + Host Ollama on `:11434` is left alone. This sidecar binds `127.0.0.1:11435` with `OLLAMA_NUM_GPU=0` (CPU). Measure RSS (`/api/ps` `size`), not VRAM. diff --git a/docs/roadmap.md b/docs/roadmap.md index d0ccb3a..b899705 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -36,14 +36,14 @@ Epic [#16](https://git.produktor.io/eSlider/2dph/issues/16) closed. ## v2 -[#6](https://git.produktor.io/eSlider/2dph/issues/6) OCR — `pdftotext` then -`pdftoppm` + tesseract `eng+deu`. Optional `ocr-paddle`. +[#6](https://git.produktor.io/eSlider/2dph/issues/6) OCR — **in**. +[#30](https://git.produktor.io/eSlider/2dph/issues/30) OQ3 duckdb-go — **in**. [#29](https://git.produktor.io/eSlider/2dph/issues/29) OQ1 contradiction -resolution. [#30](https://git.produktor.io/eSlider/2dph/issues/30) OQ3 duckdb-md. +resolution. ## Blockers -None for epic #16 (closed). Remaining v2: OQ1, OQ3, OQ4. +None for epic #16 (closed). Remaining v2: OQ1, OQ4. ``` question @@ -58,8 +58,8 @@ question ## Not v1 -OQ1 contradiction resolution, OQ3 duckdb-md export, OQ4 YAML-first leafs. -OCR (OQ2) is in: tesseract, not docling. +OQ1 contradiction resolution, OQ4 YAML-first leafs. +OCR (OQ2) and duckdb-go (OQ3/D22) are in. ## Close epic #16 when diff --git a/go.mod b/go.mod index 53081f4..3f1a59b 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/arran4/golang-ical v0.3.5 github.com/chewxy/math32 v1.11.2 github.com/daulet/tokenizers v1.27.0 + github.com/duckdb/duckdb-go/v2 v2.10505.0 github.com/go-git/go-git/v5 v5.19.2 golang.org/x/sys v0.47.0 golang.org/x/text v0.40.0 @@ -20,10 +21,17 @@ require ( github.com/apache/arrow-go/v18 v18.6.0 // indirect github.com/cloudflare/circl v1.6.3 // indirect github.com/cyphar/filepath-securejoin v0.6.1 // indirect + github.com/duckdb/duckdb-go-bindings v0.10505.0 // indirect + github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10505.0 // indirect + github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10505.0 // indirect + github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10505.0 // indirect + github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10505.0 // indirect + github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.10505.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.9.0 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/goccy/go-json v0.10.6 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/google/flatbuffers v25.12.19+incompatible // indirect diff --git a/go.sum b/go.sum index 7c26588..12da8fd 100644 --- a/go.sum +++ b/go.sum @@ -31,6 +31,20 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/duckdb/duckdb-go-bindings v0.10505.0 h1:/0pPsTLrcCsTGxT0VrHgJWnOcPe1tQL1vrki1v3jbAI= +github.com/duckdb/duckdb-go-bindings v0.10505.0/go.mod h1:HoD5xePkDj3VZbBnVVfxVVYIljZ9khCprWA7FgwIiC4= +github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10505.0 h1:FrMqquFBQlMsi34h2KZgCku54rqA8xEbXZ0NLVDKwYs= +github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10505.0/go.mod h1:EnAvZh1kNJHp5yF+M1ZHNEvapnmt6anq1xXHVrAGqMo= +github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10505.0 h1:lbRbpQwT1MmUhh/VTwukV9K8bxKByV3UghAP3MvsbBo= +github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10505.0/go.mod h1:IGLSeEcFhNeZF16aVjQCULD7TsFZKG5G7SyKJAXKp5c= +github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10505.0 h1:nrsaVYj3XYCRbS2FpdOMD/KHE7egRMr+/NR1IHmjT84= +github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10505.0/go.mod h1:KAIynZ0GHCS7X5fRyuFnQMg/SZBPK/bS9OCOVojClxw= +github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10505.0 h1:qM6oGDgwXBILJGbTY4fCy6QOczLpucUA6yn6g3ORjh4= +github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10505.0/go.mod h1:81SGOYoEUs8qaAfSk1wRfM5oobrIJ5KI7AzYhK6/bvQ= +github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.10505.0 h1:DjqZl9rYreHkSOqnqLmkrqH5T8UdQNcxZLJVZzGmXXA= +github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.10505.0/go.mod h1:K25pJL26ARblGDeuAkrdblFvUen92+CwksLtPEHRqqQ= +github.com/duckdb/duckdb-go/v2 v2.10505.0 h1:SWwvLn2Qx/RQSnQNupwgIF8VbnJ5A6OQU9lYb/mDETI= +github.com/duckdb/duckdb-go/v2 v2.10505.0/go.mod h1:m0PW4J4FG9hlFlVdXi6Ds9owpyIDaBdE2jyce00fGcE= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= @@ -47,6 +61,8 @@ github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMj github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= github.com/go-git/go-git/v5 v5.19.2 h1:wkfn7vOlUBu8ivAWKBWisTiwJK4jYHzTF8Ndv1LyGqY= github.com/go-git/go-git/v5 v5.19.2/go.mod h1:QqCBE1EFN5ddFmrliLQ3/ntRCUjZU3EJuwuB/jWEHjk= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= diff --git a/internal/duckstats/duckstats.go b/internal/duckstats/duckstats.go new file mode 100644 index 0000000..6d6bd8c --- /dev/null +++ b/internal/duckstats/duckstats.go @@ -0,0 +1,47 @@ +// Package duckstats runs in-process DuckDB for columnar aggregates. +// Graph facts stay in Ladybug. Web-search KV cache stays modernc sqlite. +package duckstats + +import ( + "database/sql" + "fmt" + + _ "github.com/duckdb/duckdb-go/v2" +) + +type Stats struct { + N int `json:"n"` + Min float64 `json:"min"` + P50 float64 `json:"p50"` + P95 float64 `json:"p95"` + Max float64 `json:"max"` + Avg float64 `json:"avg"` +} + +func Quantiles(samples []float64) (Stats, error) { + if len(samples) == 0 { + return Stats{}, fmt.Errorf("duckstats: empty samples") + } + db, err := sql.Open("duckdb", "") + if err != nil { + return Stats{}, err + } + defer db.Close() + var s Stats + err = db.QueryRow(` +SELECT count(v), min(v), quantile_cont(v, 0.5), quantile_cont(v, 0.95), max(v), avg(v) +FROM (SELECT unnest(?) AS v)`, samples).Scan( + &s.N, &s.Min, &s.P50, &s.P95, &s.Max, &s.Avg) + return s, err +} + +func CountJSONL(path string) (int64, error) { + db, err := sql.Open("duckdb", "") + if err != nil { + return 0, err + } + defer db.Close() + var n int64 + err = db.QueryRow(`SELECT count(*) FROM read_json_auto(?)`, path).Scan(&n) + return n, err +} diff --git a/internal/duckstats/duckstats_test.go b/internal/duckstats/duckstats_test.go new file mode 100644 index 0000000..c295499 --- /dev/null +++ b/internal/duckstats/duckstats_test.go @@ -0,0 +1,51 @@ +package duckstats + +import ( + "os" + "testing" +) + +func TestQuantilesEmpty(t *testing.T) { + _, err := Quantiles(nil) + if err == nil { + t.Fatal("empty slice must error") + } +} + +func TestQuantilesOdd(t *testing.T) { + s, err := Quantiles([]float64{1, 2, 3, 4, 5}) + if err != nil { + t.Fatal(err) + } + if s.N != 5 { + t.Fatalf("n=%d", s.N) + } + if s.Min != 1 || s.Max != 5 { + t.Fatalf("min=%v max=%v", s.Min, s.Max) + } + if s.P50 != 3 { + t.Fatalf("p50=%v want 3", s.P50) + } + if s.Avg != 3 { + t.Fatalf("avg=%v want 3", s.Avg) + } + if s.P95 < 4.5 || s.P95 > 5 { + t.Fatalf("p95=%v want in [4.5,5]", s.P95) + } +} + +func TestCountJSONL(t *testing.T) { + dir := t.TempDir() + p := dir + "/rows.jsonl" + body := "{\"ms\":1}\n{\"ms\":2}\n{\"ms\":3}\n" + if err := os.WriteFile(p, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + n, err := CountJSONL(p) + if err != nil { + t.Fatal(err) + } + if n != 3 { + t.Fatalf("count=%d want 3", n) + } +} diff --git a/internal/reasoner/client.go b/internal/reasoner/client.go index 802b418..794f837 100644 --- a/internal/reasoner/client.go +++ b/internal/reasoner/client.go @@ -105,15 +105,17 @@ type Client struct { } type Report struct { - Model string `json:"model"` - HF string `json:"hf_id"` - Device string `json:"device"` - ToolCallOK int `json:"tool_call_ok"` - ToolCallN int `json:"tool_call_n"` - XMLLeak int `json:"xml_leak"` - RSSMB int `json:"rss_mb"` - VRAMMB int `json:"vram_mb"` - Prompts []Result `json:"prompts"` + Model string `json:"model"` + HF string `json:"hf_id"` + Device string `json:"device"` + ToolCallOK int `json:"tool_call_ok"` + ToolCallN int `json:"tool_call_n"` + XMLLeak int `json:"xml_leak"` + RSSMB int `json:"rss_mb"` + VRAMMB int `json:"vram_mb"` + LatencyP50MS float64 `json:"latency_p50_ms,omitempty"` + LatencyP95MS float64 `json:"latency_p95_ms,omitempty"` + Prompts []Result `json:"prompts"` } func HFFor(model string) string { diff --git a/qa/system_perf.py b/qa/system_perf.py new file mode 100755 index 0000000..07e0b66 --- /dev/null +++ b/qa/system_perf.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +"""System performance test: PicoClaw surface (brain MCP) + optional reasoner. + + BRAIN_URL=http://127.0.0.1:8630 ./qa/system_perf.py --json + REASONER_BASE_URL=http://127.0.0.1:11435/v1 REASONER_MODEL=qwen3.5:9b \\ + ./qa/system_perf.py --reasoner --picoclaw --json + +Does not write Ladybug. Search includes web (D17); expect ~10s+ per search. +Exit 1 if health/get/audit gates fail. Reasoner is measured, not gated. +""" +from __future__ import annotations + +import argparse +import json +import os +import statistics +import sys +import time +import urllib.error +import urllib.request +from concurrent.futures import ThreadPoolExecutor + +DEFAULT_BRAIN = "http://127.0.0.1:8630" +DEFAULT_REASONER = "http://127.0.0.1:11435/v1" +DEFAULT_MODEL = "qwen3.5:9b" +DEFAULT_PICOCLAW = "http://127.0.0.1:18790" + +GATE_HEALTH_MS = 500 +GATE_GET_P50_MS = 50 +GATE_AUDIT_P50_MS = 50 + + +def _req(url: str, data: bytes | None = None, timeout: float = 90) -> bytes: + headers = {"Content-Type": "application/json"} if data is not None else {} + req = urllib.request.Request(url, data=data, headers=headers) + with urllib.request.urlopen(req, timeout=timeout) as res: + return res.read() + + +def timed(fn): + t0 = time.perf_counter() + out = fn() + return (time.perf_counter() - t0) * 1000.0, out + + +def stats(samples: list[float]) -> dict: + s = sorted(samples) + n = len(s) + return { + "n": n, + "min_ms": round(s[0], 1), + "p50_ms": round(s[n // 2], 1), + "p95_ms": round(s[min(n - 1, int(n * 0.95))], 1), + "max_ms": round(s[-1], 1), + "avg_ms": round(statistics.mean(s), 1), + } + + +def mcp(brain: str, method: str, params=None, timeout: float = 90) -> dict: + payload: dict = {"jsonrpc": "2.0", "id": 1, "method": method} + if params is not None: + payload["params"] = params + raw = _req(brain.rstrip("/") + "/mcp", json.dumps(payload).encode(), timeout=timeout) + return json.loads(raw.decode()) + + +def mcp_call(brain: str, name: str, arguments: dict, timeout: float = 90) -> tuple[bool, str]: + d = mcp(brain, "tools/call", {"name": name, "arguments": arguments}, timeout=timeout) + res = d.get("result") or {} + text = ((res.get("content") or [{}])[0].get("text") or "") + return (not res.get("isError")), text + + +def reasoner_tool_call(base: str, model: str, user: str) -> str: + payload = { + "model": model, + "messages": [ + {"role": "system", "content": "You are PicoClaw. Always call search before answering."}, + {"role": "user", "content": user}, + ], + "tools": [ + { + "type": "function", + "function": { + "name": "search", + "description": "deduction search", + "parameters": { + "type": "object", + "properties": {"q": {"type": "string"}}, + "required": ["q"], + }, + }, + } + ], + "tool_choice": "required", + } + raw = _req( + base.rstrip("/") + "/chat/completions", + json.dumps(payload).encode(), + timeout=600, + ) + chat = json.loads(raw.decode()) + tcs = chat["choices"][0]["message"].get("tool_calls") or [] + if not tcs: + return "" + return tcs[0]["function"]["name"] + + +def run(args: argparse.Namespace) -> dict: + brain = args.brain.rstrip("/") + report: dict = { + "brain": brain, + "device": "cpu", + "ok": True, + "gates": {}, + "mcp": {}, + } + ms, _ = timed(lambda: _req(brain + "/health", timeout=5)) + report["mcp"]["health"] = {"n": 1, "avg_ms": round(ms, 1)} + report["gates"]["health"] = ms <= GATE_HEALTH_MS + if ms > GATE_HEALTH_MS: + report["ok"] = False + + list_ms = [] + for _ in range(args.n): + ms, d = timed(lambda: mcp(brain, "tools/list", timeout=10)) + names = [t["name"] for t in ((d.get("result") or {}).get("tools") or [])] + if "search" not in names: + report["ok"] = False + list_ms.append(ms) + report["mcp"]["tools_list"] = stats(list_ms) + + audit_ms = [] + for _ in range(args.n): + ms, (ok, _) = timed(lambda: mcp_call(brain, "audit", {})) + if not ok: + report["ok"] = False + audit_ms.append(ms) + report["mcp"]["audit"] = stats(audit_ms) + report["gates"]["audit_p50"] = report["mcp"]["audit"]["p50_ms"] <= GATE_AUDIT_P50_MS + if not report["gates"]["audit_p50"]: + report["ok"] = False + + ok, text = mcp_call(brain, "search", {"q": "LadybugDB", "n": 2}, timeout=90) + inner = json.loads(text) if ok else {} + hits = inner.get("results") or [] + leaf_id = hits[0]["id"] if hits else "" + report["mcp"]["search_seed"] = { + "ok": ok, + "count": inner.get("count"), + "web": (inner.get("web") or {}).get("status"), + } + + get_ms = [] + if leaf_id: + for _ in range(args.n): + ms, (ok, _) = timed(lambda: mcp_call(brain, "get", {"id": leaf_id, "body": True})) + if not ok: + report["ok"] = False + get_ms.append(ms) + report["mcp"]["get"] = stats(get_ms) + report["gates"]["get_p50"] = report["mcp"]["get"]["p50_ms"] <= GATE_GET_P50_MS + if not report["gates"]["get_p50"]: + report["ok"] = False + + def one_get() -> float: + t0 = time.perf_counter() + mcp_call(brain, "get", {"id": leaf_id, "body": True}) + return (time.perf_counter() - t0) * 1000.0 + + t0 = time.perf_counter() + with ThreadPoolExecutor(max_workers=8) as ex: + conc = list(ex.map(lambda _: one_get(), range(8))) + wall = (time.perf_counter() - t0) * 1000.0 + report["mcp"]["get_concurrent_8"] = {**stats(conc), "wall_ms": round(wall, 1)} + + search_ms = [] + for q in ("LadybugDB", "model2vec"): + ms, (ok, text) = timed(lambda q=q: mcp_call(brain, "search", {"q": q, "n": 3}, timeout=90)) + inner = json.loads(text) if ok else {} + search_ms.append(ms) + report.setdefault("mcp", {}).setdefault("search_samples", []).append( + { + "q": q, + "ms": round(ms, 1), + "ok": ok, + "count": inner.get("count"), + "web": (inner.get("web") or {}).get("status"), + } + ) + if search_ms: + report["mcp"]["search"] = stats(search_ms) + + if args.reasoner: + base = args.reasoner_url + model = args.model + report["reasoner"] = {"base_url": base, "model": model, "calls": []} + for user in ( + "Use tools. Search the 2dph brain for LadybugDB. Call search.", + "Use tools. Search the 2dph brain for model2vec. Call search.", + ): + ms, name = timed(lambda user=user: reasoner_tool_call(base, model, user)) + report["reasoner"]["calls"].append({"ms": round(ms, 1), "tool": name}) + tools = [c["tool"] for c in report["reasoner"]["calls"]] + report["gates"]["reasoner_tool_call"] = bool(tools) and all(t == "search" for t in tools) + if not report["gates"]["reasoner_tool_call"]: + report["ok"] = False + + if args.picoclaw: + gw = args.picoclaw_url.rstrip("/") + ms, raw = timed(lambda: _req(gw + "/health", timeout=5)) + body = json.loads(raw.decode()) + report["picoclaw"] = { + "url": gw, + "health_ms": round(ms, 1), + "status": body.get("status"), + } + report["gates"]["picoclaw_health"] = body.get("status") == "ok" and ms <= GATE_HEALTH_MS + if not report["gates"]["picoclaw_health"]: + report["ok"] = False + return report + + +def main(argv: list[str]) -> int: + p = argparse.ArgumentParser(description="2dph system performance (MCP + optional reasoner)") + p.add_argument("--brain", default=os.environ.get("BRAIN_URL", DEFAULT_BRAIN)) + p.add_argument("--n", type=int, default=20) + p.add_argument("--json", action="store_true") + p.add_argument("--reasoner", action="store_true") + p.add_argument("--picoclaw", action="store_true") + p.add_argument("--picoclaw-url", default=os.environ.get("PICOCLAW_URL", DEFAULT_PICOCLAW)) + p.add_argument("--reasoner-url", default=os.environ.get("REASONER_BASE_URL", DEFAULT_REASONER)) + p.add_argument("--model", default=os.environ.get("REASONER_MODEL", DEFAULT_MODEL)) + args = p.parse_args(argv) + try: + report = run(args) + except (urllib.error.URLError, TimeoutError, OSError) as e: + print(f"system_perf: {e}", file=sys.stderr) + return 1 + if args.json: + print(json.dumps(report, indent=2)) + else: + print(f"ok={report['ok']} brain={report['brain']}") + for name, block in report.get("mcp", {}).items(): + if isinstance(block, dict) and "p50_ms" in block: + print(f" {name}: p50={block['p50_ms']} p95={block['p95_ms']} n={block['n']}") + elif name == "health": + print(f" health: {block.get('avg_ms')} ms") + for k, v in report.get("gates", {}).items(): + print(f" gate {k}: {v}") + for c in (report.get("reasoner") or {}).get("calls") or []: + print(f" reasoner {c['tool']}: {c['ms']} ms") + return 0 if report["ok"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/skills/duckdb/SKILL.md b/skills/duckdb/SKILL.md new file mode 100644 index 0000000..0598b89 --- /dev/null +++ b/skills/duckdb/SKILL.md @@ -0,0 +1,38 @@ +--- +name: duckdb +description: >- + Use https://github.com/duckdb/duckdb-go in-process for columnar analytics + (quantiles, GROUP BY, JSON/CSV/Parquet/JSONL scans) when that is faster than + nested Go loops. Not Ladybug. Not the web-search sqlite cache. Use when + aggregating samples, counting JSONL, or SQL over tabular files. +--- + +# duckdb-go + +Use https://github.com/duckdb/duckdb-go where it makes sense to get better performance in code. + +In-process DuckDB (`internal/duckstats`, `database/sql` driver `duckdb`). +Vectorized SQL over tables, JSONL, CSV, Parquet. CGO with bundled libs +(linux/darwin amd64/arm64). Links with **gcc/g++** (libstdc++), not Zig. +D21 Zig (`bin/cgo/zcc`) is Ladybug/tokenizers only. After +`eval "$(bin/cgo/zig env)"`: + +```bash +CC=gcc CXX=g++ CGO_CFLAGS= CGO_LDFLAGS= ./bin/qa/stats.go <<< '[1,2,3,4,5]' +CC=gcc CXX=g++ CGO_CFLAGS= CGO_LDFLAGS= go test ./internal/duckstats +``` + +| Store | Job | +|-------|-----| +| Ladybug | graph + FTS + HNSW (facts/info) | +| modernc sqlite | web-search KV cache + throttle | +| duckdb-go | OLAP: quantiles, counts, scans of many rows/files | +| mikefarah/yq | small YAML/JSON/XML/CSV/TOML/HCL slice, not bulk | + +```bash +./bin/qa/stats.go <<< '[1,2,3,4,5]' +./bin/qa/stats.go --jsonl path/to/rows.jsonl +``` + +Do not open Ladybug through DuckDB. Do not put secrets or client PII into +DuckDB files under the repo. diff --git a/skills/picoclaw/SKILL.md b/skills/picoclaw/SKILL.md index 86ba854..d51c4f4 100644 --- a/skills/picoclaw/SKILL.md +++ b/skills/picoclaw/SKILL.md @@ -1,15 +1,15 @@ --- name: picoclaw description: >- - 2dph is the memory/fact gate, not the agent loop. Use when wiring PicoClaw - or any MCP client: call brain search/get/audit before a factual reply. - throttled is not a negative finding. + 2dph is the memory/fact gate. Compose runs the official PicoClaw gateway. + Use when wiring PicoClaw or any MCP client: call brain search/get/audit + before a factual reply. throttled is not a negative finding. --- # PicoClaw — fact-check before assert -PicoClaw (or any agent) speaks MCP at `POST /mcp` on `bin/brain/serve.go`. -2dph does not run the agent loop. Compose: `docker compose --profile picoclaw up brain-mcp` +PicoClaw speaks MCP at `POST /mcp` on `bin/brain/serve.go`. Compose profile +`picoclaw` runs the official `sipeed/picoclaw` gateway plus `brain-mcp` (see [docs/picoclaw.md](../../docs/picoclaw.md)). ## Tool order (before a factual reply) diff --git a/skills/postgres/SKILL.md b/skills/postgres/SKILL.md index 34e08d4..86564f7 100644 --- a/skills/postgres/SKILL.md +++ b/skills/postgres/SKILL.md @@ -9,7 +9,7 @@ description: >- # postgres `bin/postgres/query.go` wraps vendored `bin/db/psql-yq`. Output is YAML -(cheaper than psql ASCII, easy to slice with `yq`). +(cheaper than psql ASCII, easy to slice with mikefarah/yq). ```bash bin/postgres/query.go --profile onlyoffice -s document_asset # column list diff --git a/skills/web-search/SKILL.md b/skills/web-search/SKILL.md index 276ef72..f62c067 100644 --- a/skills/web-search/SKILL.md +++ b/skills/web-search/SKILL.md @@ -11,7 +11,7 @@ description: >- ```bash bin/web/search.go "LadybugDB vector index" bin/web/search.go "model2vec multilingual" --category it -bin/web/search.go "hypervisor" --site example.com --json | jq -r '.results[].url' +bin/web/search.go "hypervisor" --site example.com --json | yq -r '.results[].url' bin/web/search.go "postgres partial index" --lang en --fresh year ``` diff --git a/skills/web-search/reference/instance-tuning.md b/skills/web-search/reference/instance-tuning.md index 5e73030..3b195ff 100644 --- a/skills/web-search/reference/instance-tuning.md +++ b/skills/web-search/reference/instance-tuning.md @@ -43,7 +43,7 @@ found" unless the client refuses to call it absence. ```bash for i in $(seq 10); do - bin/web/search.go "test $i" -n 1 --refresh --json | jq -r .status + bin/web/search.go "test $i" -n 1 --refresh --json | yq -r '.status' done ``` diff --git a/skills/yq/SKILL.md b/skills/yq/SKILL.md new file mode 100644 index 0000000..13e90e7 --- /dev/null +++ b/skills/yq/SKILL.md @@ -0,0 +1,33 @@ +--- +name: yq +description: >- + Use https://github.com/mikefarah/yq to work with YAML, JSON, XML, CSV, + TOML, HCL where it's efficient and less code. Use when slicing compose, + config, --json tool output, CSV/TOML/HCL/XML, or converting between those + formats. Not kislyuk Python yq. Not jq when yq already does the job. +--- + +# yq (mikefarah) + +Use https://github.com/mikefarah/yq to work with YAML, JSON, XML, CSV, TOML, HCL where it's efficient and less code. + +This is the Go `yq` (`yq --version` contains `mikefarah`). It is not +kislyuk/yq (Python, jq-syntax, YAML-only wrapper). `bin/db/psql-yq` already +calls this binary. + +Prefer `yq` over `python3 -c`, `jq`, or ad-hoc parsers when one expression +reads or converts the file. Keep Python/Go for HTTP, binary protocols, and +in-process tests. + +```bash +yq '.services.picoclaw.image' compose.yaml +yq -P . deploy/picoclaw/config.json # JSON → YAML +yq -o=json '.gates' # JSON stdin (qa/system_perf.py --json) +yq -p=csv -o=json . +yq -p=xml -o=json . +yq -p=toml '.package.name' file.toml +bin/brain/search.go "LadybugDB" --json | yq '.[].ref' +bin/web/search.go "hypervisor" --json | yq -r '.results[].url' +``` + +Do not print secrets, PII, or `$HOME/.config/brain/` through `yq`.