refactor(tools): bin/{subject}/{method} layout; Go serve+watch modules
Move serve/ (module) -> bin/server, tools/ -> bin/tools, replace bin/kb-watch bash with bin/watch Go package; self-executing Go shebangs bin/serve.go and bin/kb/watch.go; Docker + CI + git/import + docs repointed. Multi-stage image builds static serve+watch binaries (no Go runtime in container).
This commit is contained in:
File diff suppressed because one or more lines are too long
@@ -0,0 +1 @@
|
||||
{"query": "Pflegegrad Test 4", "results": [], "answers": [], "corrections": [], "infoboxes": [], "suggestions": [], "unresponsive_engines": [["brave", "Suspended: too many requests"], ["duckduckgo", "CAPTCHA"], ["startpage", "Suspended: CAPTCHA"]]}
|
||||
@@ -0,0 +1,110 @@
|
||||
"""Tests for the SearXNG client. No network: two recorded responses stand in.
|
||||
|
||||
Run: python3 -m unittest discover -s tools -t .
|
||||
"""
|
||||
import sys
|
||||
import json
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
import websearch as ws
|
||||
|
||||
FIXTURES = Path(__file__).resolve().parent / "fixtures"
|
||||
HEALTHY = json.loads((FIXTURES / "healthy.json").read_text())
|
||||
THROTTLED = json.loads((FIXTURES / "throttled.json").read_text())
|
||||
|
||||
|
||||
class Classify(unittest.TestCase):
|
||||
"""An empty result set is not evidence of absence.
|
||||
|
||||
The instance answers 200 with `results: []` when it throttles us, so calling
|
||||
that "no matches" would make an agent conclude something false.
|
||||
"""
|
||||
|
||||
def test_healthy_response_is_ok(self):
|
||||
self.assertEqual(ws.classify(HEALTHY), "ok")
|
||||
|
||||
def test_empty_response_is_throttled_not_empty(self):
|
||||
self.assertEqual(ws.classify(THROTTLED), "throttled")
|
||||
|
||||
def test_status_is_never_the_word_empty(self):
|
||||
self.assertNotIn(ws.classify(THROTTLED), ("empty", "no_results"))
|
||||
|
||||
|
||||
class Project(unittest.TestCase):
|
||||
def test_keeps_only_the_fields_worth_context(self):
|
||||
out = ws.project(HEALTHY, limit=3)
|
||||
self.assertEqual(out["status"], "ok")
|
||||
self.assertEqual(len(out["results"]), 3)
|
||||
self.assertEqual(set(out["results"][0]), {"rank", "title", "url", "snippet", "engine"})
|
||||
|
||||
def test_snippet_is_trimmed(self):
|
||||
out = ws.project(HEALTHY, limit=5, snippet_chars=40)
|
||||
self.assertTrue(all(len(r["snippet"]) <= 43 for r in out["results"]))
|
||||
|
||||
def test_projection_is_far_cheaper_than_the_raw_payload(self):
|
||||
raw = len(json.dumps(HEALTHY))
|
||||
small = len(json.dumps(ws.project(HEALTHY, limit=5)))
|
||||
self.assertLess(small * 3, raw)
|
||||
|
||||
def test_throttled_projection_carries_the_engine_reasons(self):
|
||||
out = ws.project(THROTTLED, limit=5)
|
||||
self.assertEqual(out["status"], "throttled")
|
||||
self.assertEqual(out["results"], [])
|
||||
self.assertTrue(out["unresponsive"])
|
||||
|
||||
|
||||
class CacheKey(unittest.TestCase):
|
||||
def test_same_question_same_key(self):
|
||||
self.assertEqual(ws.cache_key("Pflegegrad", {}), ws.cache_key("Pflegegrad", {}))
|
||||
|
||||
def test_case_and_padding_do_not_matter(self):
|
||||
self.assertEqual(ws.cache_key(" Pflegegrad ", {}), ws.cache_key("pflegegrad", {}))
|
||||
|
||||
def test_parameters_change_the_key(self):
|
||||
self.assertNotEqual(ws.cache_key("x", {"lang": "de"}), ws.cache_key("x", {}))
|
||||
|
||||
def test_parameter_order_does_not_change_the_key(self):
|
||||
self.assertEqual(ws.cache_key("x", {"a": "1", "b": "2"}),
|
||||
ws.cache_key("x", {"b": "2", "a": "1"}))
|
||||
|
||||
|
||||
class PhiGuard(unittest.TestCase):
|
||||
"""The query leaves this host, so client data must never reach it."""
|
||||
|
||||
def test_plain_technical_query_passes(self):
|
||||
self.assertIsNone(ws.phi_reason("Pflegegrad SGB XI Einstufung"))
|
||||
self.assertIsNone(ws.phi_reason("site:ticket.detective.de Toureffizienz"))
|
||||
|
||||
def test_long_digit_run_is_refused(self):
|
||||
self.assertIsNotNone(ws.phi_reason("Kunde 4711220385 Adresse"))
|
||||
|
||||
def test_insurance_number_is_refused(self):
|
||||
self.assertIsNotNone(ws.phi_reason("KV-Nr A123456789"))
|
||||
|
||||
def test_street_with_house_number_is_refused(self):
|
||||
self.assertIsNotNone(ws.phi_reason("Hauptstraße 14 Berlin"))
|
||||
self.assertIsNotNone(ws.phi_reason("Lindenstr. 7"))
|
||||
|
||||
def test_personalnummer_is_refused(self):
|
||||
self.assertIsNotNone(ws.phi_reason("Personalnummer 12"))
|
||||
|
||||
def test_short_numbers_are_fine(self):
|
||||
self.assertIsNone(ws.phi_reason("SGB XI Paragraph 45b"))
|
||||
|
||||
|
||||
class Throttle(unittest.TestCase):
|
||||
def test_waits_the_remainder_of_the_interval(self):
|
||||
self.assertAlmostEqual(ws.wait_for(last=100.0, now=104.0, interval=10.0), 6.0)
|
||||
|
||||
def test_no_wait_once_the_interval_passed(self):
|
||||
self.assertEqual(ws.wait_for(last=100.0, now=130.0, interval=10.0), 0.0)
|
||||
|
||||
def test_no_wait_on_a_first_call(self):
|
||||
self.assertEqual(ws.wait_for(last=None, now=130.0, interval=10.0), 0.0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,161 @@
|
||||
"""SearXNG client that is safe for agents to share.
|
||||
|
||||
Three things make this more than a curl wrapper:
|
||||
|
||||
* An empty result set from this instance usually means "throttled", not "no
|
||||
matches". Reporting it as absence would make an agent state something false,
|
||||
so `classify` never returns a word that sounds like a negative finding.
|
||||
* Queries leave the host, so `phi_reason` refuses anything that smells like
|
||||
client data before it reaches an external engine.
|
||||
* Results are cached and calls are serialised, because the instance suspends
|
||||
engines under load.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
SNIPPET_CHARS = 150
|
||||
DEFAULT_LIMIT = 5
|
||||
MIN_INTERVAL = 10.0
|
||||
CACHE_TTL = 7 * 24 * 3600
|
||||
RETRY_BACKOFF = (20.0, 60.0)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# response handling
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def classify(payload: dict) -> str:
|
||||
"""`ok` when at least one engine answered, `throttled` otherwise."""
|
||||
return "ok" if payload.get("results") else "throttled"
|
||||
|
||||
|
||||
def project(payload: dict, limit: int = DEFAULT_LIMIT,
|
||||
snippet_chars: int = SNIPPET_CHARS) -> dict:
|
||||
"""Keep the few fields worth spending context on."""
|
||||
status = classify(payload)
|
||||
results = []
|
||||
for rank, item in enumerate(payload.get("results", [])[:limit], start=1):
|
||||
snippet = re.sub(r"\s+", " ", item.get("content") or "").strip()
|
||||
if len(snippet) > snippet_chars:
|
||||
snippet = snippet[:snippet_chars].rstrip() + "..."
|
||||
results.append({
|
||||
"rank": rank,
|
||||
"title": item.get("title", ""),
|
||||
"url": item.get("url", ""),
|
||||
"snippet": snippet,
|
||||
"engine": item.get("engine", ""),
|
||||
})
|
||||
out = {
|
||||
"query": payload.get("query", ""),
|
||||
"status": status,
|
||||
"results": results,
|
||||
}
|
||||
unresponsive = [f"{name}: {reason}" for name, reason in
|
||||
payload.get("unresponsive_engines", [])]
|
||||
if unresponsive:
|
||||
out["unresponsive"] = unresponsive
|
||||
if status == "throttled":
|
||||
out["note"] = ("no engine answered - this is a throttled instance, "
|
||||
"not evidence that nothing exists")
|
||||
return out
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# cache key and throttling
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def cache_key(query: str, params: dict) -> str:
|
||||
norm = " ".join(query.lower().split())
|
||||
stable = json.dumps(params, sort_keys=True, ensure_ascii=False)
|
||||
return hashlib.sha256(f"{norm}\x00{stable}".encode()).hexdigest()
|
||||
|
||||
|
||||
def wait_for(last: float | None, now: float, interval: float = MIN_INTERVAL) -> float:
|
||||
"""Seconds to sleep so that calls stay `interval` apart."""
|
||||
if last is None:
|
||||
return 0.0
|
||||
return max(0.0, interval - (now - last))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# PII guard
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
PII_PATTERNS = [
|
||||
(re.compile(r"\d{6,}"), "a run of six or more digits looks like an ID"),
|
||||
(re.compile(r"\bpersonalnummer\b", re.I), "Personalnummer is staff data"),
|
||||
(re.compile(r"\bkv[-\s]?nr\b", re.I), "KV-Nr is an insurance number"),
|
||||
(re.compile(r"\bversichertennummer\b", re.I), "insurance number"),
|
||||
(re.compile(r"\b[A-Za-zÄÖÜäöüß]+(?:stra(?:ss|ß)e|str\.)\s*\d+", re.I),
|
||||
"a street with a house number looks like an address"),
|
||||
(re.compile(r"\bgeb(?:urtsdatum)?\.?\s*\d{1,2}[./]\d{1,2}[./]\d{2,4}", re.I),
|
||||
"a date of birth"),
|
||||
]
|
||||
|
||||
|
||||
def phi_reason(query: str) -> str | None:
|
||||
"""Why this query must not be sent, or None when it is safe."""
|
||||
for pattern, reason in PII_PATTERNS:
|
||||
if pattern.search(query):
|
||||
return reason
|
||||
return None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# cache storage
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
CACHE_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS responses (
|
||||
key TEXT PRIMARY KEY,
|
||||
fetched REAL NOT NULL,
|
||||
payload TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value REAL NOT NULL
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
def open_cache(path: Path) -> sqlite3.Connection:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(path, timeout=30)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.executescript(CACHE_SCHEMA)
|
||||
return conn
|
||||
|
||||
|
||||
def cache_get(conn: sqlite3.Connection, key: str, ttl: float = CACHE_TTL,
|
||||
now: float | None = None) -> dict | None:
|
||||
now = time.time() if now is None else now
|
||||
row = conn.execute("SELECT fetched, payload FROM responses WHERE key = ?",
|
||||
(key,)).fetchone()
|
||||
if row is None or now - row["fetched"] > ttl:
|
||||
return None
|
||||
return json.loads(row["payload"])
|
||||
|
||||
|
||||
def cache_put(conn: sqlite3.Connection, key: str, payload: dict,
|
||||
now: float | None = None) -> None:
|
||||
now = time.time() if now is None else now
|
||||
conn.execute("INSERT OR REPLACE INTO responses (key, fetched, payload) VALUES (?, ?, ?)",
|
||||
(key, now, json.dumps(payload, ensure_ascii=False)))
|
||||
conn.commit()
|
||||
|
||||
|
||||
def last_call(conn: sqlite3.Connection) -> float | None:
|
||||
row = conn.execute("SELECT value FROM meta WHERE key = 'last_call'").fetchone()
|
||||
return row["value"] if row else None
|
||||
|
||||
|
||||
def mark_call(conn: sqlite3.Connection, now: float | None = None) -> None:
|
||||
now = time.time() if now is None else now
|
||||
conn.execute("INSERT OR REPLACE INTO meta (key, value) VALUES ('last_call', ?)", (now,))
|
||||
conn.commit()
|
||||
Reference in New Issue
Block a user