"""SearXNG client that is safe for agents to share. Three things make this more than a curl wrapper: * An empty result set from this instance usually means "throttled", not "no matches". Reporting it as absence would make an agent state something false, so `classify` never returns a word that sounds like a negative finding. * Queries leave the host, so `phi_reason` refuses anything that smells like client data before it reaches an external engine. * Results are cached and calls are serialised, because the instance suspends engines under load. """ from __future__ import annotations import hashlib import json import re import sqlite3 import time from pathlib import Path SNIPPET_CHARS = 150 DEFAULT_LIMIT = 5 MIN_INTERVAL = 10.0 CACHE_TTL = 7 * 24 * 3600 RETRY_BACKOFF = (20.0, 60.0) # -------------------------------------------------------------------------- # response handling # -------------------------------------------------------------------------- def classify(payload: dict) -> str: """`ok` when at least one engine answered, `throttled` otherwise.""" return "ok" if payload.get("results") else "throttled" def project(payload: dict, limit: int = DEFAULT_LIMIT, snippet_chars: int = SNIPPET_CHARS) -> dict: """Keep the few fields worth spending context on.""" status = classify(payload) results = [] for rank, item in enumerate(payload.get("results", [])[:limit], start=1): snippet = re.sub(r"\s+", " ", item.get("content") or "").strip() if len(snippet) > snippet_chars: snippet = snippet[:snippet_chars].rstrip() + "..." results.append({ "rank": rank, "title": item.get("title", ""), "url": item.get("url", ""), "snippet": snippet, "engine": item.get("engine", ""), }) out = { "query": payload.get("query", ""), "status": status, "results": results, } unresponsive = [f"{name}: {reason}" for name, reason in payload.get("unresponsive_engines", [])] if unresponsive: out["unresponsive"] = unresponsive if status == "throttled": out["note"] = ("no engine answered - this is a throttled instance, " "not evidence that nothing exists") return out # -------------------------------------------------------------------------- # cache key and throttling # -------------------------------------------------------------------------- def cache_key(query: str, params: dict) -> str: norm = " ".join(query.lower().split()) stable = json.dumps(params, sort_keys=True, ensure_ascii=False) return hashlib.sha256(f"{norm}\x00{stable}".encode()).hexdigest() def wait_for(last: float | None, now: float, interval: float = MIN_INTERVAL) -> float: """Seconds to sleep so that calls stay `interval` apart.""" if last is None: return 0.0 return max(0.0, interval - (now - last)) # -------------------------------------------------------------------------- # PII guard # -------------------------------------------------------------------------- PII_PATTERNS = [ (re.compile(r"\d{6,}"), "a run of six or more digits looks like an ID"), (re.compile(r"\bpersonalnummer\b", re.I), "Personalnummer is staff data"), (re.compile(r"\bkv[-\s]?nr\b", re.I), "KV-Nr is an insurance number"), (re.compile(r"\bversichertennummer\b", re.I), "insurance number"), (re.compile(r"\b[A-Za-zÄÖÜäöüß]+(?:stra(?:ss|ß)e|str\.)\s*\d+", re.I), "a street with a house number looks like an address"), (re.compile(r"\bgeb(?:urtsdatum)?\.?\s*\d{1,2}[./]\d{1,2}[./]\d{2,4}", re.I), "a date of birth"), ] def phi_reason(query: str) -> str | None: """Why this query must not be sent, or None when it is safe.""" for pattern, reason in PII_PATTERNS: if pattern.search(query): return reason return None # -------------------------------------------------------------------------- # cache storage # -------------------------------------------------------------------------- CACHE_SCHEMA = """ CREATE TABLE IF NOT EXISTS responses ( key TEXT PRIMARY KEY, fetched REAL NOT NULL, payload TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS meta ( key TEXT PRIMARY KEY, value REAL NOT NULL ); """ def open_cache(path: Path) -> sqlite3.Connection: path.parent.mkdir(parents=True, exist_ok=True) conn = sqlite3.connect(path, timeout=30) conn.row_factory = sqlite3.Row conn.executescript(CACHE_SCHEMA) return conn def cache_get(conn: sqlite3.Connection, key: str, ttl: float = CACHE_TTL, now: float | None = None) -> dict | None: now = time.time() if now is None else now row = conn.execute("SELECT fetched, payload FROM responses WHERE key = ?", (key,)).fetchone() if row is None or now - row["fetched"] > ttl: return None return json.loads(row["payload"]) def cache_put(conn: sqlite3.Connection, key: str, payload: dict, now: float | None = None) -> None: now = time.time() if now is None else now conn.execute("INSERT OR REPLACE INTO responses (key, fetched, payload) VALUES (?, ?, ?)", (key, now, json.dumps(payload, ensure_ascii=False))) conn.commit() def last_call(conn: sqlite3.Connection) -> float | None: row = conn.execute("SELECT value FROM meta WHERE key = 'last_call'").fetchone() return row["value"] if row else None def mark_call(conn: sqlite3.Connection, now: float | None = None) -> None: now = time.time() if now is None else now conn.execute("INSERT OR REPLACE INTO meta (key, value) VALUES ('last_call', ?)", (now,)) conn.commit()