feat: DuckDB quantiles in-process (gcc CGO), not Ladybug.
OQ3: internal/duckstats + bin/qa/stats.go. Zig stays Ladybug-only. mikefarah/yq for small structured slices. Gitea #30.
This commit is contained in:
Executable
+73
@@ -0,0 +1,73 @@
|
||||
//usr/bin/env go run -tags=qa_stats "$0" "$@"; exit
|
||||
//go:build qa_stats
|
||||
//
|
||||
// bin/qa/stats.go - DuckDB quantiles over a JSON number array or JSONL count.
|
||||
//
|
||||
// ./bin/qa/stats.go <<< '[1,2,3,4,5]'
|
||||
// ./bin/qa/stats.go --jsonl rows.jsonl
|
||||
//
|
||||
// NOTE: never run `gofmt -w` on this file — it breaks the shebang.
|
||||
// DuckDB CGO needs gcc/g++ (not Zig). After eval "$(bin/cgo/zig env)":
|
||||
// CC=gcc CXX=g++ CGO_CFLAGS= CGO_LDFLAGS= ./bin/qa/stats.go
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/eSlider/2dph/internal/duckstats"
|
||||
)
|
||||
|
||||
func main() {
|
||||
os.Exit(run(os.Args[1:]))
|
||||
}
|
||||
|
||||
func run(args []string) int {
|
||||
jsonl := ""
|
||||
for i := 0; i < len(args); i++ {
|
||||
a := args[i]
|
||||
switch {
|
||||
case a == "--jsonl" && i+1 < len(args):
|
||||
i++
|
||||
jsonl = args[i]
|
||||
case strings.HasPrefix(a, "--jsonl="):
|
||||
jsonl = strings.TrimPrefix(a, "--jsonl=")
|
||||
case a == "-h" || a == "--help":
|
||||
fmt.Fprintln(os.Stderr, "bin/qa/stats.go [--jsonl FILE] # stdin = JSON [float,…]")
|
||||
return 0
|
||||
default:
|
||||
fmt.Fprintln(os.Stderr, "unknown arg:", a)
|
||||
return 2
|
||||
}
|
||||
}
|
||||
if jsonl != "" {
|
||||
n, err := duckstats.CountJSONL(jsonl)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 1
|
||||
}
|
||||
fmt.Printf("n: %d\n", n)
|
||||
return 0
|
||||
}
|
||||
raw, err := io.ReadAll(os.Stdin)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 1
|
||||
}
|
||||
var samples []float64
|
||||
if err := json.Unmarshal(raw, &samples); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 1
|
||||
}
|
||||
s, err := duckstats.Quantiles(samples)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 1
|
||||
}
|
||||
fmt.Printf("n: %d\nmin: %g\np50: %g\np95: %g\nmax: %g\navg: %g\n",
|
||||
s.N, s.Min, s.P50, s.P95, s.Max, s.Avg)
|
||||
return 0
|
||||
}
|
||||
+12
-1
@@ -7,7 +7,7 @@
|
||||
// ./bin/reasoner/bakeoff.go --model MichelRosselli/bonsai-27b:Q1_0 --json
|
||||
//
|
||||
// Measures OpenAI tool_calls (search/get/audit) and RSS from Ollama /api/ps, not VRAM.
|
||||
// PicoClaw is not in this repo; the tool names match internal/httpapi MCP ops.
|
||||
// PicoClaw is compose profile picoclaw; tool names match internal/httpapi MCP ops.
|
||||
// NOTE: never run `gofmt -w` on this file — it breaks the shebang.
|
||||
package main
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/eSlider/2dph/internal/duckstats"
|
||||
"github.com/eSlider/2dph/internal/reasoner"
|
||||
)
|
||||
|
||||
@@ -61,6 +62,14 @@ func run(args []string) int {
|
||||
}
|
||||
c := reasoner.Client{BaseURL: base, Model: model, Device: device}
|
||||
rep := reasoner.Run(c)
|
||||
lat := make([]float64, 0, len(rep.Prompts))
|
||||
for _, p := range rep.Prompts {
|
||||
lat = append(lat, float64(p.LatencyMS))
|
||||
}
|
||||
if st, err := duckstats.Quantiles(lat); err == nil {
|
||||
rep.LatencyP50MS = st.P50
|
||||
rep.LatencyP95MS = st.P95
|
||||
}
|
||||
raw, err := json.MarshalIndent(rep, "", " ")
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
@@ -76,6 +85,8 @@ func run(args []string) int {
|
||||
fmt.Printf("xml_leak: %d\n", rep.XMLLeak)
|
||||
fmt.Printf("rss_mb: %d\n", rep.RSSMB)
|
||||
fmt.Printf("vram_mb: %d\n", rep.VRAMMB)
|
||||
fmt.Printf("latency_p50_ms: %g\n", rep.LatencyP50MS)
|
||||
fmt.Printf("latency_p95_ms: %g\n", rep.LatencyP95MS)
|
||||
for _, p := range rep.Prompts {
|
||||
status := "fail"
|
||||
if p.OK {
|
||||
|
||||
@@ -217,6 +217,32 @@ class BinLayoutTest(unittest.TestCase):
|
||||
if "go-git/go-git" in line:
|
||||
self.assertNotIn("indirect", line)
|
||||
|
||||
def test_duckdb_go_is_direct_require(self) -> None:
|
||||
text = (ROOT / "go.mod").read_text()
|
||||
first = text.split("require (")[1].split(")")[0]
|
||||
self.assertRegex(first, r"github.com/duckdb/duckdb-go/v2\s+v")
|
||||
for line in first.splitlines():
|
||||
if "duckdb/duckdb-go" in line:
|
||||
self.assertNotIn("indirect", line)
|
||||
skill = (ROOT / "skills" / "duckdb" / "SKILL.md").read_text()
|
||||
self.assertIn("github.com/duckdb/duckdb-go", skill)
|
||||
self.assertIn("Ladybug", skill)
|
||||
self.assertIn("sqlite", skill.lower())
|
||||
self.assertIn("gcc", skill.lower())
|
||||
self.assertIn("Zig", skill)
|
||||
plan = (ROOT / "PLAN.md").read_text()
|
||||
self.assertIn("D22", plan)
|
||||
self.assertIn("duckdb-go", plan)
|
||||
self._assert_shebang("bin/qa/stats.go")
|
||||
reasoner = (ROOT / "internal" / "reasoner" / "client.go").read_text()
|
||||
self.assertNotIn("duckdb", reasoner)
|
||||
self.assertNotIn("duckstats", reasoner)
|
||||
bakeoff = (ROOT / "bin" / "reasoner" / "bakeoff.go").read_text()
|
||||
self.assertIn("internal/duckstats", bakeoff)
|
||||
webcache = (ROOT / "internal" / "websearch" / "cache.go").read_text()
|
||||
self.assertNotIn("duckdb", webcache)
|
||||
self.assertIn("modernc.org/sqlite", webcache)
|
||||
|
||||
def test_cgo_uses_zig_not_gcc(self) -> None:
|
||||
for rel in ("bin/cgo/zig", "bin/cgo/zcc", "bin/cgo/zc++"):
|
||||
p = ROOT / rel
|
||||
|
||||
@@ -121,7 +121,7 @@ class PublishedDocsTest(unittest.TestCase):
|
||||
self.assertIn("D18", plan)
|
||||
self.assertIn("Qwen/Qwen3.5-9B", plan)
|
||||
compose = (ROOT / "compose.yaml").read_text()
|
||||
self.assertIn('profiles: ["reasoner"]', compose)
|
||||
self.assertIn('"reasoner"', compose)
|
||||
self.assertIn("OLLAMA_NUM_GPU", compose)
|
||||
self.assertIn("127.0.0.1:11435", compose)
|
||||
dockerfile = (ROOT / "Dockerfile").read_text()
|
||||
|
||||
@@ -47,3 +47,17 @@ class SkillsTest(unittest.TestCase):
|
||||
self.assertIn("throttled", skill.lower())
|
||||
self.assertIn("not a negative finding", agents)
|
||||
self.assertIn("Fact-check every", agents)
|
||||
|
||||
def test_yq_is_mikefarah_for_structured_data(self) -> None:
|
||||
skill = (ROOT / "skills" / "yq" / "SKILL.md").read_text()
|
||||
self.assertIn("https://github.com/mikefarah/yq", skill)
|
||||
for fmt in ("YAML", "JSON", "XML", "CSV", "TOML", "HCL"):
|
||||
self.assertIn(fmt, skill)
|
||||
self.assertIn("not kislyuk", skill.lower())
|
||||
plan = (ROOT / "PLAN.md").read_text()
|
||||
self.assertIn("mikefarah/yq", plan)
|
||||
agents = (ROOT / "AGENTS.md").read_text()
|
||||
self.assertIn("mikefarah/yq", agents)
|
||||
web = (ROOT / "skills" / "web-search" / "SKILL.md").read_text()
|
||||
self.assertIn("| yq ", web)
|
||||
self.assertNotIn("| jq ", web)
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
"""qa/system_perf.py is an offline-gated system test (no live brain in CI)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
class SystemPerfScriptTest(unittest.TestCase):
|
||||
def test_script_compiles_and_is_read_only(self) -> None:
|
||||
path = ROOT / "qa" / "system_perf.py"
|
||||
src = path.read_text()
|
||||
compile(src, str(path), "exec")
|
||||
self.assertIn("--json", src)
|
||||
self.assertIn("qwen3.5:9b", src)
|
||||
self.assertIn("--picoclaw", src)
|
||||
self.assertIn("BRAIN_URL", src)
|
||||
self.assertIn("tools/list", src)
|
||||
self.assertIn("tools/call", src)
|
||||
self.assertIn("GATE_HEALTH_MS", src)
|
||||
self.assertIn("GATE_GET_P50_MS", src)
|
||||
self.assertNotIn("kb.lbug", src)
|
||||
self.assertNotIn("password", src.lower())
|
||||
self.assertNotIn("token", src.lower())
|
||||
|
||||
def test_script_does_not_write_ladybug(self) -> None:
|
||||
tree = ast.parse((ROOT / "qa" / "system_perf.py").read_text())
|
||||
writes = [
|
||||
n.func.attr
|
||||
for n in ast.walk(tree)
|
||||
if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute)
|
||||
and n.func.attr in {"write_text", "write_bytes", "dump"}
|
||||
]
|
||||
self.assertEqual(writes, [], f"system_perf must not write files: {writes}")
|
||||
Reference in New Issue
Block a user