feat: DuckDB quantiles in-process (gcc CGO), not Ladybug. (#34)
OQ3: internal/duckstats + bin/qa/stats.go. Zig stays Ladybug-only. mikefarah/yq for small structured slices. Gitea #30.
This commit is contained in:
@@ -0,0 +1,47 @@
|
||||
// Package duckstats runs in-process DuckDB for columnar aggregates.
|
||||
// Graph facts stay in Ladybug. Web-search KV cache stays modernc sqlite.
|
||||
package duckstats
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
|
||||
_ "github.com/duckdb/duckdb-go/v2"
|
||||
)
|
||||
|
||||
type Stats struct {
|
||||
N int `json:"n"`
|
||||
Min float64 `json:"min"`
|
||||
P50 float64 `json:"p50"`
|
||||
P95 float64 `json:"p95"`
|
||||
Max float64 `json:"max"`
|
||||
Avg float64 `json:"avg"`
|
||||
}
|
||||
|
||||
func Quantiles(samples []float64) (Stats, error) {
|
||||
if len(samples) == 0 {
|
||||
return Stats{}, fmt.Errorf("duckstats: empty samples")
|
||||
}
|
||||
db, err := sql.Open("duckdb", "")
|
||||
if err != nil {
|
||||
return Stats{}, err
|
||||
}
|
||||
defer db.Close()
|
||||
var s Stats
|
||||
err = db.QueryRow(`
|
||||
SELECT count(v), min(v), quantile_cont(v, 0.5), quantile_cont(v, 0.95), max(v), avg(v)
|
||||
FROM (SELECT unnest(?) AS v)`, samples).Scan(
|
||||
&s.N, &s.Min, &s.P50, &s.P95, &s.Max, &s.Avg)
|
||||
return s, err
|
||||
}
|
||||
|
||||
func CountJSONL(path string) (int64, error) {
|
||||
db, err := sql.Open("duckdb", "")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer db.Close()
|
||||
var n int64
|
||||
err = db.QueryRow(`SELECT count(*) FROM read_json_auto(?)`, path).Scan(&n)
|
||||
return n, err
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package duckstats
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestQuantilesEmpty(t *testing.T) {
|
||||
_, err := Quantiles(nil)
|
||||
if err == nil {
|
||||
t.Fatal("empty slice must error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuantilesOdd(t *testing.T) {
|
||||
s, err := Quantiles([]float64{1, 2, 3, 4, 5})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s.N != 5 {
|
||||
t.Fatalf("n=%d", s.N)
|
||||
}
|
||||
if s.Min != 1 || s.Max != 5 {
|
||||
t.Fatalf("min=%v max=%v", s.Min, s.Max)
|
||||
}
|
||||
if s.P50 != 3 {
|
||||
t.Fatalf("p50=%v want 3", s.P50)
|
||||
}
|
||||
if s.Avg != 3 {
|
||||
t.Fatalf("avg=%v want 3", s.Avg)
|
||||
}
|
||||
if s.P95 < 4.5 || s.P95 > 5 {
|
||||
t.Fatalf("p95=%v want in [4.5,5]", s.P95)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCountJSONL(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
p := dir + "/rows.jsonl"
|
||||
body := "{\"ms\":1}\n{\"ms\":2}\n{\"ms\":3}\n"
|
||||
if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n, err := CountJSONL(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 3 {
|
||||
t.Fatalf("count=%d want 3", n)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user