build(ci): uv toolchain, release-please semver; feat(skills): vendor tools self-contained (no symlinks, relative refs)
- bin/db/psql-yq + bin/web/search + tools/{yamlout,websearch} vendored as real files
- bin/db/ssh-tunnel added (OnlyOffice VM pg on 5433)
- skills reference local bin/ paths; no agent-skills/abs links in git
- pyproject.toml + uv.lock; CI installs via uv sync --frozen
- release-please auto-tags semver from conventional commits when green
- LICENSE MIT, badges/mermaid README
This commit is contained in:
@@ -0,0 +1,161 @@
|
||||
"""SearXNG client that is safe for agents to share.
|
||||
|
||||
Three things make this more than a curl wrapper:
|
||||
|
||||
* An empty result set from this instance usually means "throttled", not "no
|
||||
matches". Reporting it as absence would make an agent state something false,
|
||||
so `classify` never returns a word that sounds like a negative finding.
|
||||
* Queries leave the host, so `phi_reason` refuses anything that smells like
|
||||
client data before it reaches an external engine.
|
||||
* Results are cached and calls are serialised, because the instance suspends
|
||||
engines under load.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
SNIPPET_CHARS = 150
|
||||
DEFAULT_LIMIT = 5
|
||||
MIN_INTERVAL = 10.0
|
||||
CACHE_TTL = 7 * 24 * 3600
|
||||
RETRY_BACKOFF = (20.0, 60.0)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# response handling
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def classify(payload: dict) -> str:
|
||||
"""`ok` when at least one engine answered, `throttled` otherwise."""
|
||||
return "ok" if payload.get("results") else "throttled"
|
||||
|
||||
|
||||
def project(payload: dict, limit: int = DEFAULT_LIMIT,
|
||||
snippet_chars: int = SNIPPET_CHARS) -> dict:
|
||||
"""Keep the few fields worth spending context on."""
|
||||
status = classify(payload)
|
||||
results = []
|
||||
for rank, item in enumerate(payload.get("results", [])[:limit], start=1):
|
||||
snippet = re.sub(r"\s+", " ", item.get("content") or "").strip()
|
||||
if len(snippet) > snippet_chars:
|
||||
snippet = snippet[:snippet_chars].rstrip() + "..."
|
||||
results.append({
|
||||
"rank": rank,
|
||||
"title": item.get("title", ""),
|
||||
"url": item.get("url", ""),
|
||||
"snippet": snippet,
|
||||
"engine": item.get("engine", ""),
|
||||
})
|
||||
out = {
|
||||
"query": payload.get("query", ""),
|
||||
"status": status,
|
||||
"results": results,
|
||||
}
|
||||
unresponsive = [f"{name}: {reason}" for name, reason in
|
||||
payload.get("unresponsive_engines", [])]
|
||||
if unresponsive:
|
||||
out["unresponsive"] = unresponsive
|
||||
if status == "throttled":
|
||||
out["note"] = ("no engine answered - this is a throttled instance, "
|
||||
"not evidence that nothing exists")
|
||||
return out
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# cache key and throttling
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def cache_key(query: str, params: dict) -> str:
|
||||
norm = " ".join(query.lower().split())
|
||||
stable = json.dumps(params, sort_keys=True, ensure_ascii=False)
|
||||
return hashlib.sha256(f"{norm}\x00{stable}".encode()).hexdigest()
|
||||
|
||||
|
||||
def wait_for(last: float | None, now: float, interval: float = MIN_INTERVAL) -> float:
|
||||
"""Seconds to sleep so that calls stay `interval` apart."""
|
||||
if last is None:
|
||||
return 0.0
|
||||
return max(0.0, interval - (now - last))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# PII guard
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
PII_PATTERNS = [
|
||||
(re.compile(r"\d{6,}"), "a run of six or more digits looks like an ID"),
|
||||
(re.compile(r"\bpersonalnummer\b", re.I), "Personalnummer is staff data"),
|
||||
(re.compile(r"\bkv[-\s]?nr\b", re.I), "KV-Nr is an insurance number"),
|
||||
(re.compile(r"\bversichertennummer\b", re.I), "insurance number"),
|
||||
(re.compile(r"\b[A-Za-zÄÖÜäöüß]+(?:stra(?:ss|ß)e|str\.)\s*\d+", re.I),
|
||||
"a street with a house number looks like an address"),
|
||||
(re.compile(r"\bgeb(?:urtsdatum)?\.?\s*\d{1,2}[./]\d{1,2}[./]\d{2,4}", re.I),
|
||||
"a date of birth"),
|
||||
]
|
||||
|
||||
|
||||
def phi_reason(query: str) -> str | None:
|
||||
"""Why this query must not be sent, or None when it is safe."""
|
||||
for pattern, reason in PII_PATTERNS:
|
||||
if pattern.search(query):
|
||||
return reason
|
||||
return None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# cache storage
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
CACHE_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS responses (
|
||||
key TEXT PRIMARY KEY,
|
||||
fetched REAL NOT NULL,
|
||||
payload TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value REAL NOT NULL
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
def open_cache(path: Path) -> sqlite3.Connection:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(path, timeout=30)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.executescript(CACHE_SCHEMA)
|
||||
return conn
|
||||
|
||||
|
||||
def cache_get(conn: sqlite3.Connection, key: str, ttl: float = CACHE_TTL,
|
||||
now: float | None = None) -> dict | None:
|
||||
now = time.time() if now is None else now
|
||||
row = conn.execute("SELECT fetched, payload FROM responses WHERE key = ?",
|
||||
(key,)).fetchone()
|
||||
if row is None or now - row["fetched"] > ttl:
|
||||
return None
|
||||
return json.loads(row["payload"])
|
||||
|
||||
|
||||
def cache_put(conn: sqlite3.Connection, key: str, payload: dict,
|
||||
now: float | None = None) -> None:
|
||||
now = time.time() if now is None else now
|
||||
conn.execute("INSERT OR REPLACE INTO responses (key, fetched, payload) VALUES (?, ?, ?)",
|
||||
(key, now, json.dumps(payload, ensure_ascii=False)))
|
||||
conn.commit()
|
||||
|
||||
|
||||
def last_call(conn: sqlite3.Connection) -> float | None:
|
||||
row = conn.execute("SELECT value FROM meta WHERE key = 'last_call'").fetchone()
|
||||
return row["value"] if row else None
|
||||
|
||||
|
||||
def mark_call(conn: sqlite3.Connection, now: float | None = None) -> None:
|
||||
now = time.time() if now is None else now
|
||||
conn.execute("INSERT OR REPLACE INTO meta (key, value) VALUES ('last_call', ?)", (now,))
|
||||
conn.commit()
|
||||
Reference in New Issue
Block a user