From e393cc6a99635af75ea83f95bb44483cda3bdd49 Mon Sep 17 00:00:00 2001 From: Andrey Oblivantsev Date: Thu, 13 Aug 2026 21:54:31 +0100 Subject: [PATCH] feat: compile ladybug CGO with Zig, not gcc (#24) API image is Go-only (compose target api). Python rebuild is profile index. bin/cgo/zig pins Zig 0.14.1, liblbug, and tokenizers. --- .dockerignore | 1 + .github/workflows/ci.yml | 8 +- AGENTS.md | 6 +- Dockerfile | 72 +++++++++++++---- PLAN.md | 12 +-- README.md | 7 +- bin/brain/get.go | 1 + bin/brain/search.go | 2 +- bin/cgo/zc++ | 23 ++++++ bin/cgo/zcc | 24 ++++++ bin/cgo/zig | 130 +++++++++++++++++++++++++++++++ bin/docker-entrypoint | 41 ++++++---- bin/kb/search | 10 +-- bin/tools/test_bin_layout.py | 19 +++++ bin/tools/test_published_docs.py | 14 ++++ compose.yaml | 61 ++++++++++----- docs/design.md | 8 +- 17 files changed, 363 insertions(+), 76 deletions(-) create mode 100755 bin/cgo/zc++ create mode 100755 bin/cgo/zcc create mode 100755 bin/cgo/zig diff --git a/.dockerignore b/.dockerignore index 8a687c1..6995e13 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,6 +3,7 @@ var .git .github +lib-ladybug __pycache__ *.pyc *.lbug diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41f9f99..040fdaa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,9 @@ jobs: bash -n bin/db/ssh-tunnel bash -n bin/docker-entrypoint bash -n bin/kb/search + bash -n bin/cgo/zig + sh -n bin/cgo/zcc + sh -n bin/cgo/zc++ - name: Python unit tests (offline, vendored tools) run: | @@ -54,9 +57,10 @@ jobs: run: | ./bin/facts/audit self 2>/dev/null || echo "audit: not yet implemented; gate skipped" - - name: kb/eval recall gate + - name: CGO via Zig (compile brain/search) run: | - ./bin/kb/eval 2>/dev/null || echo "eval: not yet implemented; gate skipped" + chmod +x bin/cgo/zig bin/cgo/zcc bin/cgo/zc++ + bin/cgo/zig go build -tags system_ladybug -o /tmp/brain-search ./bin/brain/search.go release: name: Release (semver) diff --git a/AGENTS.md b/AGENTS.md index f655523..02b5974 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -49,9 +49,10 @@ bin/web/ search.go (SearXNG; Python shim execs it) internal/ shared Go (brain/rank is cgo-free; chats parsers; gitlog; websearch) bin/watch/ corpus watcher (used by bin/brain/watch.go) bin/tools/ vendored python libs behind bin/* (kblib, yamlout, websearch) -bin/docker-entrypoint container entrypoint (brain index|search|serve|watch) +bin/cgo/ zig zcc zc++ (CGO via zig cc, not gcc) +bin/docker-entrypoint container entrypoint (api: serve|search|watch; index: python) compose.yaml docker composition (root level, not docker/) -Dockerfile multi-stage: python deps + static Go binaries +Dockerfile api (Zig CGO, no Python) + index (Python write) var/ kb.lbug, var/mail/*, caches (gitignored) .venv/ ladybug + model2vec + mistune ``` @@ -85,6 +86,7 @@ bin/facts/crm.go [--dry-run] # proof person↔company/comp bin/kb/search "query" [--repo X] # deprecated wrapper → bin/brain/search.go bin/brain/search.go "query" [--root facts|info] # deduction search → YAML bin/brain/search.go "query" --no-web # local graph only +eval "$(bin/cgo/zig env)" # Zig cc + liblbug (not gcc) bin/brain/get.go [--body] [--json] # Go read; Python bin/kb/get CI fallback bin/brain/stats.go [--json] bin/brain/eval.go [--json] # recall@5; questions in internal/brain/rank diff --git a/Dockerfile b/Dockerfile index 1b46f38..7064fe0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,13 @@ # syntax=docker/dockerfile:1 -FROM python:3.12-slim AS base +# +# docker build --target api -t 2dph:api . +# docker build --target index -t 2dph:index . +# +# API: Go + ladybug via Zig CGO (no CPython). +# Index: Python write path (profile `index` until brain/add is v2). + +# --- Python sidecar (Ladybug write / rebuild) --- +FROM python:3.12-slim AS index ENV PYTHONUNBUFFERED=1 \ PYTHONDONTWRITEBYTECODE=1 \ @@ -9,25 +17,11 @@ ENV PYTHONUNBUFFERED=1 \ WORKDIR /app RUN id -u 2dph 2>/dev/null || useradd --create-home --uid 1001 2dph -# deps layer-first: rebuild only on dependency change COPY requirements.lock.txt /tmp/requirements.lock.txt RUN python -m pip install --no-cache-dir -r /tmp/requirements.lock.txt \ && rm /tmp/requirements.lock.txt -# Go services: static binaries, no interpreter at runtime -FROM golang:1.25 AS go-build -WORKDIR /src -COPY go.mod ./ -COPY bin/server ./bin/server -COPY bin/watch ./bin/watch -RUN CGO_ENABLED=0 go build -o /serve ./bin/server \ - && CGO_ENABLED=0 go build -o /watch ./bin/watch - -# runtime: python toolchain + Go services -FROM base COPY . . -COPY --from=go-build /serve /app/bin/serve -COPY --from=go-build /watch /app/bin/watch RUN chmod +x /app/bin/docker-entrypoint \ && chown -R 2dph:2dph /app USER 2dph @@ -37,5 +31,51 @@ ENV PATH="/app/bin:${PATH}" \ KB_ROOT=/app HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ CMD python -c "import model2vec, ladybug, mistune; print('ok')" || exit 1 +ENTRYPOINT ["/app/bin/docker-entrypoint"] -ENTRYPOINT ["/app/bin/docker-entrypoint"] \ No newline at end of file +# --- Go API: CGO with Zig, not gcc --- +FROM golang:1.26-bookworm AS api-build +WORKDIR /src +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl xz-utils ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +COPY bin/cgo ./bin/cgo +RUN chmod +x bin/cgo/zig bin/cgo/zcc bin/cgo/zc++ \ + && ./bin/cgo/zig env >/dev/null + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . +ENV CGO_RPATH=/usr/local/lib +RUN eval "$(./bin/cgo/zig env)" \ + && go build -tags brain_serve,system_ladybug -o /out/brain-serve ./bin/brain/serve.go \ + && go build -tags system_ladybug -o /out/brain-search ./bin/brain/search.go \ + && CGO_ENABLED=0 go build -tags brain_watch -o /out/brain-watch ./bin/brain/watch.go + +FROM debian:bookworm-slim AS api +RUN apt-get update \ + && apt-get install -y --no-install-recommends libssl3 ca-certificates wget \ + && rm -rf /var/lib/apt/lists/* \ + && useradd --create-home --uid 1001 2dph +COPY --from=api-build /out/brain-serve /usr/local/bin/brain-serve +COPY --from=api-build /out/brain-search /usr/local/bin/brain-search +COPY --from=api-build /out/brain-watch /usr/local/bin/brain-watch +COPY --from=api-build /src/lib-ladybug/liblbug.so.0.19.1 /usr/local/lib/liblbug.so.0.19.1 +COPY bin/docker-entrypoint /usr/local/bin/docker-entrypoint +RUN chmod +x /usr/local/bin/docker-entrypoint \ + && ln -s liblbug.so.0.19.1 /usr/local/lib/liblbug.so.0 \ + && ln -s liblbug.so.0 /usr/local/lib/liblbug.so \ + && ldconfig +USER 2dph +ENV KB_ROOT=/data \ + KB_PORT=8630 \ + LD_LIBRARY_PATH=/usr/local/lib \ + HF_HOME=/data/hf +WORKDIR /data +EXPOSE 8630 +HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ + CMD wget -qO- http://127.0.0.1:8630/health || exit 1 +ENTRYPOINT ["/usr/local/bin/docker-entrypoint"] +CMD ["serve"] diff --git a/PLAN.md b/PLAN.md index 5213111..0898f0d 100644 --- a/PLAN.md +++ b/PLAN.md @@ -29,7 +29,7 @@ detective method: **a fact needs ≥2 independent sources or it is | D3 | web search | Go client `bin/web/search.go` (`internal/websearch`). SearXNG URL is config (`BRAIN_SEARCH_URL`). Optional Compose profile `searxng` (sanitized settings). Do not run a second copy on a host that already has one. Empty/`throttled` ≠ “nothing exists”. | | D4 | embeddings | **model2vec** `minishlab/potion-multilingual-128M` instead of embeddinggemma. | | D5 | parser | **mistune** for MD → leaf extraction (duckdb-md documented as future optional SQL/export layer, not v1). | -| D6 | graph engine | **LadybugDB**. Go is the service (`bin/brain/search.go`, `bin/brain/serve.go` in-process, `internal/brain`). Read path (`get.go` / `stats.go` / `eval.go`) is Go + cgo. Python `bin/kb/{get,stats,eval}` is the CI fallback (GitHub runners have no ladybug cgo). Index/write stays Python until the Go write path is safe. | +| D6 | graph engine | **LadybugDB**. Go is the service (`bin/brain/search.go`, `bin/brain/serve.go` in-process, `internal/brain`). Read path is Go + Zig CGO (D21). Python `bin/kb/{get,stats,eval}` is the CI fallback when Zig/libs are not fetched. Index/write stays Python (`compose --profile index`) until the Go write path is safe. | | D7 | db access | `db-yaml`/`psql-yq`-style, read-only, YAML out. OnlyOffice Postgres via SSH tunnel (`127.0.0.1:5433`). | | D8 | evidence | detective method: ≥2 independent sources or `(not confirmed)`. Auto-pair docker ps × compose × ssh-config × docs. | | D9 | facts/goal model | Who / What / How / Where / When + evidence + confidence on every edge. | @@ -44,6 +44,7 @@ detective method: **a fact needs ≥2 independent sources or it is | D18 | reasoner | Pluggable OpenAI-compatible URL. RAM: Qwen3.5-9B. Quality: Bonsai-27B or Qwen3.6-27B. No official Qwen3.6-9B. | | D19 | git history | [go-git](https://github.com/go-git/go-git) via `bin/git/import.go`. No subprocess of the git binary. Conversion prints commit leafs; brain write is `bin/brain/index.go`. | | D20 | agent API | OpenAPI + MCP are generated from the same `internal/httpapi.Ops` table as `bin/brain/serve.go` handlers. `GET /openapi.json`, `POST /mcp` (JSON-RPC tools/list + tools/call). Tool names match OpenAPI paths (`search`/`get`/`stats`/`audit`). | +| D21 | CGO | Ladybug/tokenizers CGO is compiled with **Zig** (`bin/cgo/zcc` → `zig cc -target …-linux-gnu`), not gcc. `bin/cgo/zig` pins Zig 0.14.1 + liblbug 0.19.1 + libtokenizers 1.27.0. Compose `target: api` has no CPython; write/rebuild is profile `index`. | ## Architecture @@ -59,7 +60,8 @@ detective method: **a fact needs ≥2 independent sources or it is brain/get.go stats.go eval.go # Go read (cgo); Python bin/kb/* CI fallback brain/watch.go brain/search.go deduction: facts → info → web-search - brain/serve.go HTTP API in-process + OpenAPI/MCP (D20); compose profile picoclaw + brain/serve.go HTTP API in-process + OpenAPI/MCP (D20); Zig CGO (D21) + cgo/zig zcc zc++ CGO toolchain (zig cc, not gcc) mail/import.go JSON → markdown (no brain write) markdown/import.go mistune leaves postgres/query.go read-only YAML (wraps bin/db/psql-yq) @@ -134,10 +136,8 @@ Common props on every node/edge: `root`, `confidence`, `evidence[]`, `how`, 2. `go test ./internal/brain/rank` (cgo-free ranking + flag parser) 3. python -m unittest discover -s bin/tools (includes published-docs SoT) 4. `bin/facts/audit self` (lexicon internal consistency; `bin/facts/audit.go` is the D14 wrapper) -5. `bin/kb/eval` (recall@5 ≥ 0.95). Local SoT is `bin/brain/eval.go`; CI uses - the Python twin until the runner has ladybug cgo. Questions live in - `internal/brain/rank`. -6. md-docs build/lint if docs tooling arrives. +5. `bin/kb/eval` (recall@5 ≥ 0.95). Local SoT is `bin/brain/eval.go` via Zig CGO. +6. `bin/cgo/zig go build -tags system_ladybug` (compile search with zig cc; fetches pinned zig+libs). Feedback loop: every commit → PR → CI → green/gate → merge. Same discipline as `db/tech-poc`: contract first where there is an OpenAPI/message shape. diff --git a/README.md b/README.md index 21c6de7..3a3713a 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ bin/brain/search.go "invoice from last week" # same s - **LadybugDB** — single `var/kb.lbug`, Cypher property graph, HNSW + BM25 in one engine, embedded (no server), ACID, read-only-safe for concurrent - readers. Read tools (`get` / `stats` / `eval`) are Go + cgo; Python + readers. Read tools (`get` / `stats` / `eval`) are Go + Zig CGO (`bin/cgo/zcc`); Python `bin/kb/{get,stats,eval}` is the CI fallback. **Never `DROP INDEX` FTS/VECTOR** on Ladybug 0.19: DROP leaves ghost catalog tables (`_0_Leaf_vec_UPPER`) so recreate fails while `SHOW_INDEXES` omits HNSW. Fresh indexes = delete `var/kb.lbug` + @@ -155,9 +155,8 @@ go test ./... && python -m unittest discover -s bin/tools -t . Docker (optional, cached model + var volumes): ```bash -docker compose run --rm brain index # (re)index corpus -docker compose run --rm brain search "query" # one-shot query -docker compose run --rm brain serve # bin/brain/serve.go +docker compose up -d brain # API (Zig CGO serve :8630) +docker compose --profile index run --rm index # Python Ladybug rebuild docker compose --profile picoclaw up brain-mcp # MCP on 127.0.0.1:8630 docker compose up brain-watch # auto re-index on change ``` diff --git a/bin/brain/get.go b/bin/brain/get.go index 1b432ad..a242851 100755 --- a/bin/brain/get.go +++ b/bin/brain/get.go @@ -8,6 +8,7 @@ // ./bin/brain/get.go --json // // Needs CGO + libladybug. Python bin/kb/get is the CI fallback (no cgo). +// CGO compiler is Zig (`eval "$(bin/cgo/zig env)"`), not gcc. // NOTE: never run `gofmt -w` on this file — it breaks the shebang. package main diff --git a/bin/brain/search.go b/bin/brain/search.go index 3206ebe..43eecbb 100755 --- a/bin/brain/search.go +++ b/bin/brain/search.go @@ -7,7 +7,7 @@ // ./bin/brain/search.go serve [port] // ./bin/brain/search.go --list-model // -// Needs CGO + libladybug (CGO_CFLAGS/CGO_LDFLAGS). Prefer the wrapper +// Needs CGO + libladybug via Zig (`eval "$(bin/cgo/zig env)"`), not gcc. // bin/kb/search which sets those and builds a binary for the embed daemon. // NOTE: never run `gofmt -w` on this file — it breaks the shebang. package main diff --git a/bin/cgo/zc++ b/bin/cgo/zc++ new file mode 100755 index 0000000..1ef8ba6 --- /dev/null +++ b/bin/cgo/zc++ @@ -0,0 +1,23 @@ +#!/bin/sh +# bin/cgo/zc++ — CGO CXX. Zig, not g++. +set -eu +ROOT="$(CDPATH= cd -- "$(dirname "$0")/../.." && pwd)" +case "$(uname -m)" in + x86_64|amd64) TARGET=x86_64-linux-gnu ;; + aarch64|arm64) TARGET=aarch64-linux-gnu ;; + *) + echo "zc++: unsupported arch $(uname -m)" >&2 + exit 2 + ;; +esac +if [ -n "${ZIG:-}" ] && [ -x "$ZIG" ]; then + : +elif [ -x "$ROOT/var/zig/zig" ]; then + ZIG="$ROOT/var/zig/zig" +elif command -v zig >/dev/null 2>&1; then + ZIG="$(command -v zig)" +else + echo "zc++: zig missing; run bin/cgo/zig first" >&2 + exit 127 +fi +exec "$ZIG" c++ -target "$TARGET" "$@" diff --git a/bin/cgo/zcc b/bin/cgo/zcc new file mode 100755 index 0000000..d7b8e98 --- /dev/null +++ b/bin/cgo/zcc @@ -0,0 +1,24 @@ +#!/bin/sh +# bin/cgo/zcc — CGO CC. Zig, not gcc. +# Go invokes CC with many args; a wrapper avoids spaces in $CC. +set -eu +ROOT="$(CDPATH= cd -- "$(dirname "$0")/../.." && pwd)" +case "$(uname -m)" in + x86_64|amd64) TARGET=x86_64-linux-gnu ;; + aarch64|arm64) TARGET=aarch64-linux-gnu ;; + *) + echo "zcc: unsupported arch $(uname -m)" >&2 + exit 2 + ;; +esac +if [ -n "${ZIG:-}" ] && [ -x "$ZIG" ]; then + : +elif [ -x "$ROOT/var/zig/zig" ]; then + ZIG="$ROOT/var/zig/zig" +elif command -v zig >/dev/null 2>&1; then + ZIG="$(command -v zig)" +else + echo "zcc: zig missing; run bin/cgo/zig first" >&2 + exit 127 +fi +exec "$ZIG" cc -target "$TARGET" "$@" diff --git a/bin/cgo/zig b/bin/cgo/zig new file mode 100755 index 0000000..18165ef --- /dev/null +++ b/bin/cgo/zig @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +# bin/cgo/zig — CGO toolchain: zig cc (not gcc) + pinned liblbug + libtokenizers. +# +# eval "$(bin/cgo/zig env)" # export CC/CXX/CGO_* +# bin/cgo/zig go build ... # ensure, then exec with env +# bin/cgo/zig ./bin/brain/search.go "query" +# +# Pins live in this file. Downloads land in var/ (gitignored). +set -euo pipefail + +ROOT="$(CDPATH= cd -- "$(dirname "$0")/../.." && pwd)" +ZIG_VERSION=0.14.1 +LBUG_VERSION=0.19.1 +TOKENIZERS_VERSION=1.27.0 + +arch="$(uname -m)" +case "$arch" in + x86_64|amd64) + ZIG_ARCH=x86_64 + LBUG_ARCH=x86_64 + TOK_ARCH=x86_64 + ZIG_SHA=24aeeec8af16c381934a6cd7d95c807a8cb2cf7df9fa40d359aa884195c4716c + LBUG_SHA=ed263ae913f68cb0ddba0b98548b58edaac49929766d03bdaaa83be46c68847d + TOK_SHA=72556cdca798dd4ea7cdaba308e5f0d68a8cb93b67c96edf485b7a0edd7b07f4 + ;; + aarch64|arm64) + ZIG_ARCH=aarch64 + LBUG_ARCH=aarch64 + TOK_ARCH=aarch64 + ZIG_SHA=f7a654acc967864f7a050ddacfaa778c7504a0eca8d2b678839c21eea47c992b + LBUG_SHA=b07df2cd533c3976a2a3025866d6420a5f35514d0a822ecc4b2902d55b4725b7 + TOK_SHA=e96545ad05930c26f51f63d932ee6d3bbd32bbed149e102c5290d587a2293067 + ;; + *) + echo "bin/cgo/zig: unsupported arch $arch" >&2 + exit 2 + ;; +esac + +CACHE="$ROOT/var/cache" +LIB="$ROOT/lib-ladybug" +ZIG_DIR="$ROOT/var/zig-dist" +ZIG_BIN="$ROOT/var/zig/zig" + +sha256of() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +fetch() { + local url="$1" dest="$2" expect="$3" + if [ -f "$dest" ] && [ "$(sha256of "$dest")" = "$expect" ]; then + return 0 + fi + mkdir -p "$(dirname "$dest")" + echo "fetch $url" >&2 + curl -fsSL "$url" -o "$dest" + local got + got="$(sha256of "$dest")" + if [ "$got" != "$expect" ]; then + echo "checksum mismatch $dest: got $got want $expect" >&2 + rm -f "$dest" + exit 1 + fi +} + +ensure_zig() { + if [ -n "${ZIG:-}" ] && [ -x "$ZIG" ]; then + return 0 + fi + if [ -x "$ZIG_BIN" ]; then + export ZIG="$ZIG_BIN" + return 0 + fi + if command -v zig >/dev/null 2>&1; then + export ZIG + ZIG="$(command -v zig)" + return 0 + fi + local tar="$CACHE/zig-${ZIG_ARCH}-linux-${ZIG_VERSION}.tar.xz" + fetch "https://ziglang.org/download/${ZIG_VERSION}/zig-${ZIG_ARCH}-linux-${ZIG_VERSION}.tar.xz" \ + "$tar" "$ZIG_SHA" + mkdir -p "$CACHE" + rm -rf "$ZIG_DIR" + tar -xJf "$tar" -C "$CACHE" + mv "$CACHE/zig-${ZIG_ARCH}-linux-${ZIG_VERSION}" "$ZIG_DIR" + mkdir -p "$ROOT/var/zig" + ln -sfn "$ZIG_DIR/zig" "$ZIG_BIN" + export ZIG="$ZIG_BIN" +} + +ensure_libs() { + mkdir -p "$LIB" + if [ ! -f "$LIB/liblbug.so" ]; then + local tar="$CACHE/liblbug-linux-${LBUG_ARCH}.tar.gz" + fetch "https://github.com/LadybugDB/ladybug/releases/download/v${LBUG_VERSION}/liblbug-linux-${LBUG_ARCH}.tar.gz" \ + "$tar" "$LBUG_SHA" + tar -xzf "$tar" -C "$LIB" + fi + if [ ! -f "$LIB/libtokenizers.a" ]; then + local tar="$CACHE/libtokenizers.linux-${TOK_ARCH}.tar.gz" + fetch "https://github.com/daulet/tokenizers/releases/download/v${TOKENIZERS_VERSION}/libtokenizers.linux-${TOK_ARCH}.tar.gz" \ + "$tar" "$TOK_SHA" + tar -xzf "$tar" -C "$LIB" + fi +} + +print_env() { + printf 'export ZIG=%q\n' "$ZIG" + printf 'export CC=%q\n' "$ROOT/bin/cgo/zcc" + printf 'export CXX=%q\n' "$ROOT/bin/cgo/zc++" + printf 'export CGO_ENABLED=1\n' + printf 'export CGO_CFLAGS=%q\n' "-I$LIB" + printf 'export CGO_LDFLAGS=%q\n' "-L$LIB -Wl,-rpath,${CGO_RPATH:-$LIB}" +} + +ensure_zig +ensure_libs + +cmd="${1:-env}" +if [ "$cmd" = "env" ]; then + print_env + exit 0 +fi + +eval "$(print_env)" +exec "$@" diff --git a/bin/docker-entrypoint b/bin/docker-entrypoint index adce335..a7e3256 100755 --- a/bin/docker-entrypoint +++ b/bin/docker-entrypoint @@ -1,13 +1,10 @@ #!/usr/bin/env bash # bin/docker-entrypoint - run 2dph tools inside the container. # -# brain shell (default) -# brain search bin/brain/search.go -# brain index bin/kb/index --with-mail -# brain watch compiled /app/bin/watch (bin/brain/watch.go) -# brain serve compiled /app/bin/serve (bin/brain/serve.go) -# brain extract bin/facts/extract (docker×compose pairing) -# brain audit bin/facts/audit +# API image (Zig CGO binaries): +# serve | search | watch +# Index image (Python write path, compose profile `index`): +# index | extract | audit | search (deprecated python wrapper) # # Usage comment starts at line 2 (self-describing convention). set -euo pipefail @@ -15,13 +12,27 @@ set -euo pipefail CMD="${1:-shell}" shift || true +if [ -x /usr/local/bin/brain-serve ]; then + case "$CMD" in + shell) exec bash ;; + serve) exec /usr/local/bin/brain-serve "$@" ;; + search) exec /usr/local/bin/brain-search "$@" ;; + watch) exec /usr/local/bin/brain-watch "$@" ;; + index) + echo "index is the Python sidecar: docker compose --profile index run --rm index" >&2 + exit 2 + ;; + *) echo "unknown command: $CMD (api: serve|search|watch)" >&2; exit 2 ;; + esac +fi + case "$CMD" in - shell) exec bash ;; - search) exec "$KB_PY" /app/bin/kb/search "$@" ;; - index) exec "$KB_PY" /app/bin/kb/index --with-mail "$@" ;; - watch) exec /app/bin/watch "$@" ;; - serve) exec /app/bin/serve "$@" ;; - extract) exec "$KB_PY" /app/bin/facts/extract "$@" ;; - audit) exec "$KB_PY" /app/bin/facts/audit "$@" ;; - *) echo "unknown command: $CMD" >&2; exit 2 ;; + shell) exec bash ;; + search) exec "$KB_PY" /app/bin/kb/search "$@" ;; + index) exec "$KB_PY" /app/bin/kb/index --with-mail "$@" ;; + watch) exec /app/bin/watch "$@" ;; + serve) exec /app/bin/serve "$@" ;; + extract) exec "$KB_PY" /app/bin/facts/extract "$@" ;; + audit) exec "$KB_PY" /app/bin/facts/audit "$@" ;; + *) echo "unknown command: $CMD" >&2; exit 2 ;; esac diff --git a/bin/kb/search b/bin/kb/search index 5b4af3d..bd847cc 100755 --- a/bin/kb/search +++ b/bin/kb/search @@ -1,10 +1,9 @@ #!/usr/bin/env bash # bin/kb/search — deprecated wrapper. Use bin/brain/search.go. -# Sets CGO for ladybug, builds a binary (embed daemon needs a real executable), -# then execs it. Prints one deprecation line. +# CGO via Zig (bin/cgo/zig), not gcc. Builds a binary then execs it. set -euo pipefail -ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +ROOT="$(CDPATH= cd -- "$(dirname "$0")/../.." && pwd)" BIN="$ROOT/var/bin/brain-search" SRC="$ROOT/internal/brain" CMD="$ROOT/bin/brain" @@ -24,11 +23,10 @@ else fi if [ "$need_build" -eq 1 ]; then - echo "Building brain/search..." >&2 + echo "Building brain/search (zig cc)..." >&2 ( cd "$ROOT" && - CGO_CFLAGS="-I$ROOT/lib-ladybug" \ - CGO_LDFLAGS="-L$ROOT/lib-ladybug -Wl,-rpath,$ROOT/lib-ladybug" \ + eval "$("$ROOT/bin/cgo/zig" env)" && go build -tags system_ladybug -o "$BIN" ./bin/brain ) || exit 1 fi diff --git a/bin/tools/test_bin_layout.py b/bin/tools/test_bin_layout.py index 197b22f..32bd70e 100644 --- a/bin/tools/test_bin_layout.py +++ b/bin/tools/test_bin_layout.py @@ -1,6 +1,7 @@ """D14 layout: bin/{subject}/{method}.go, libs in internal/, one go.mod.""" from __future__ import annotations +import os import unittest from pathlib import Path @@ -157,3 +158,21 @@ class BinLayoutTest(unittest.TestCase): for line in first.splitlines(): if "go-git/go-git" in line: self.assertNotIn("indirect", line) + + def test_cgo_uses_zig_not_gcc(self) -> None: + for rel in ("bin/cgo/zig", "bin/cgo/zcc", "bin/cgo/zc++"): + p = ROOT / rel + self.assertTrue(p.is_file(), f"missing {rel}") + self.assertTrue( + os.access(p, os.X_OK), + f"{rel} must be executable", + ) + zig = (ROOT / "bin" / "cgo" / "zig").read_text() + self.assertIn("zig cc", zig) + self.assertIn("0.14.1", zig) + zcc = (ROOT / "bin" / "cgo" / "zcc").read_text() + self.assertIn('exec "$ZIG" cc', zcc) + self.assertNotIn("command -v gcc", zcc) + search = (ROOT / "bin" / "kb" / "search").read_text() + self.assertIn("bin/cgo/zig", search) + self.assertNotIn("command -v gcc", search) diff --git a/bin/tools/test_published_docs.py b/bin/tools/test_published_docs.py index d5510d0..8d0918a 100644 --- a/bin/tools/test_published_docs.py +++ b/bin/tools/test_published_docs.py @@ -89,6 +89,20 @@ class PublishedDocsTest(unittest.TestCase): self.assertFalse((ROOT / "skills" / "db-yaml").exists()) self.assertTrue((ROOT / "skills" / "postgres" / "SKILL.md").is_file()) + def test_cgo_zig_and_index_profile(self) -> None: + plan = (ROOT / "PLAN.md").read_text() + self.assertIn("D21", plan) + self.assertIn("zig cc", plan) + dockerfile = (ROOT / "Dockerfile").read_text() + self.assertIn("bin/cgo/zcc", dockerfile) + self.assertIn("FROM debian:bookworm-slim AS api", dockerfile) + self.assertIn("FROM python:3.12-slim AS index", dockerfile) + api = dockerfile[dockerfile.index("FROM debian:bookworm-slim AS api") :] + self.assertNotIn("pip install", api) + compose = (ROOT / "compose.yaml").read_text() + self.assertIn('profiles: ["index"]', compose) + self.assertIn("target: api", compose) + def test_readme_search_escalates_web(self) -> None: text = (ROOT / "README.md").read_text() self.assertIn("--no-web", text) diff --git a/compose.yaml b/compose.yaml index 90cce69..1eb006f 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,66 +1,86 @@ # 2dph — docker composition # -# docker compose run --rm brain index # rebuild graph -# docker compose run --rm brain search "Matrix fed" # one-shot query -# docker compose run --rm brain serve # async Go server -# docker compose up brain-watch # auto re-index +# docker compose up -d brain # API (Zig CGO serve) +# docker compose --profile index run --rm index # Python rebuild +# docker compose --profile picoclaw up brain-mcp +# docker compose --profile searxng up -d # -# Caching: the 128M model (HF_HOME) and kb.lbug (VAR_DIR) live in named -# volumes, so rebuilds never redownload the model or re-derive the graph. -# Secrets are never baked into the image: search.env + db-profiles.yml mount -# read-only from ~/.config/brain. +# Secrets never baked in: search.env + db-profiles.yml from ~/.config/brain. name: 2dph services: brain: - image: ghcr.io/eslider/2dph:latest + image: ghcr.io/eslider/2dph:api build: context: . dockerfile: Dockerfile + target: api cache_from: - ghcr.io/eslider/2dph:cache - command: ["brain", "search", "help"] + command: ["serve"] environment: &env HF_HOME: /data/hf BRAIN_SEARCH_CACHE: /data/cache/web-search.sqlite BRAIN_DB_PROFILES: /secret/db-profiles.yml BRAIN_SEARCH_ENV: /secret/search.env - KB_SEARCH_CMD: /app/bin/kb/search + KB_ROOT: /data KB_WORKERS: "4" + KB_PORT: "8630" volumes: - kb-model:/data/hf - kb-var:/data - # corpus is read-only on the host, never written from the container - - ..:/corpus:ro - ~/.config/brain:/secret:ro + ports: + - "127.0.0.1:8630:8630" read_only: true tmpfs: - /tmp healthcheck: - test: ["CMD", "python3", "-c", "import ladybug, model2vec, mistune; print('ok')"] + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8630/health"] interval: 30s timeout: 5s retries: 3 restart: unless-stopped stop_grace_period: 20s - # watcher: re-index on corpus file change (watchdog script) brain-watch: - image: ghcr.io/eslider/2dph:latest + image: ghcr.io/eslider/2dph:api environment: *env volumes: - kb-model:/data/hf - kb-var:/data - ..:/corpus:ro - ~/.config/brain:/secret:ro - command: ["brain", "watch", "/corpus"] + command: ["watch", "/corpus"] read_only: true tmpfs: - /tmp restart: unless-stopped stop_grace_period: 20s + # Python write path (Ladybug rebuild). Not in the API image. + # docker compose --profile index run --rm index + index: + profiles: ["index"] + image: ghcr.io/eslider/2dph:index + build: + context: . + dockerfile: Dockerfile + target: index + environment: + HF_HOME: /data/hf + KB_PY: python3 + volumes: + - kb-model:/data/hf + - kb-var:/app/var + - ..:/corpus:ro + - ~/.config/brain:/secret:ro + command: ["index"] + read_only: true + tmpfs: + - /tmp + # Optional local SearXNG (D3). Skip if BRAIN_SEARCH_URL already points at a # live instance — do not run a second copy on that host. # SEARXNG_SECRET=$(openssl rand -hex 32) docker compose --profile searxng up -d @@ -78,16 +98,15 @@ services: # MCP endpoint for an external agent (PicoClaw is not shipped here). # docker compose --profile picoclaw up brain-mcp - # Point the agent at http://127.0.0.1:8630/mcp (see deploy/picoclaw/). brain-mcp: profiles: ["picoclaw"] - image: ghcr.io/eslider/2dph:latest + image: ghcr.io/eslider/2dph:api environment: *env volumes: - kb-model:/data/hf - kb-var:/data - ~/.config/brain:/secret:ro - command: ["brain", "serve"] + command: ["serve"] ports: - "127.0.0.1:8630:8630" read_only: true @@ -97,4 +116,4 @@ services: volumes: kb-model: - kb-var: \ No newline at end of file + kb-var: diff --git a/docs/design.md b/docs/design.md index 02f1cdb..220e071 100644 --- a/docs/design.md +++ b/docs/design.md @@ -66,10 +66,12 @@ Conflicting pairings (≥2 yes vs ≥2 no) = hypothesis (OQ1 → v2 resolution). ## Read path `bin/brain/get.go`, `stats.go`, and `eval.go` call `internal/brain` with cgo -(`system_ladybug`). They do not exec Python. Control questions for recall@5 -live in `internal/brain/rank` so CI can test the table without libladybug. +(`system_ladybug`), compiled by **Zig** (`bin/cgo/zcc`, D21), not gcc. +They do not exec Python. Control questions for recall@5 live in +`internal/brain/rank` so CI can test the table without libladybug. Python `bin/kb/{get,stats,eval}` remain for GitHub Actions until the runner -has ladybug cgo. Index/write is still `bin/kb/index`. +fetches Zig + libs (`bin/cgo/zig`). Index/write is still `bin/kb/index` +(`docker compose --profile index`). ## Agent API (D20)