feat: Go SearXNG client; throttled is not absence (#16)
Tests / Test (push) Skipped
Tests / Release (semver) (push) Skipped
Tests / Test (push) Skipped
Tests / Release (semver) (push) Skipped
This commit is contained in:
committed by
GitHub
GitHub
co-authored by
GitHub
GitHub
parent
ba5cc3a6e2
commit
39ae2abe8d
+12
-136
@@ -1,150 +1,26 @@
|
||||
#!/usr/bin/env python3
|
||||
"""web/search - web search through the self-hosted SearXNG at search.ops.io.
|
||||
"""web/search — deprecated. Use bin/web/search.go (SearXNG, no Python client).
|
||||
|
||||
bin/web/search "LadybugDB vector search"
|
||||
bin/web/search "model2vec multilingual" --site github.com
|
||||
bin/web/search "uclancy" --category it -n 3 --json | jq -r '.results[].url'
|
||||
bin/web/search "sqlite-vec" --refresh # ignore the cached answer
|
||||
|
||||
This complements bin/kb/search: the knowledge base holds our own facts, this
|
||||
reaches the public web. Use it as the second, independent source that the
|
||||
detective method asks for.
|
||||
|
||||
Exit codes: 0 results, 2 refused as possible PII, 3 throttled (not "nothing
|
||||
found" - the instance answers 200 with an empty list when it throttles).
|
||||
bin/web/search.go QUERY [--json] [-n N] [--site HOST]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
TOOLS = Path(__file__).resolve().parents[1] / "tools"
|
||||
sys.path.insert(0, str(TOOLS))
|
||||
sys.path.insert(0, str(TOOLS / "web-search"))
|
||||
|
||||
import websearch as ws # noqa: E402
|
||||
from yamlout import to_yaml # noqa: E402
|
||||
|
||||
CONFIG = Path(os.environ.get("BRAIN_SEARCH_ENV", Path.home() / ".config/brain/search.env"))
|
||||
CACHE = Path(os.environ.get("BRAIN_SEARCH_CACHE", Path.home() / ".cache/brain/web-search.sqlite"))
|
||||
LOCK = CACHE.with_suffix(".lock")
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def load_config() -> dict:
|
||||
if not CONFIG.exists():
|
||||
sys.exit(f"no credentials at {CONFIG} (mode 600, BRAIN_SEARCH_URL/USER/PASS)")
|
||||
conf = {}
|
||||
for line in CONFIG.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
conf[key.strip()] = value.strip().strip("\"'")
|
||||
missing = {"BRAIN_SEARCH_URL", "BRAIN_SEARCH_USER", "BRAIN_SEARCH_PASS"} - conf.keys()
|
||||
if missing:
|
||||
sys.exit(f"{CONFIG} is missing {', '.join(sorted(missing))}")
|
||||
return conf
|
||||
|
||||
|
||||
def fetch(conf: dict, query: str, params: dict, timeout: int) -> dict:
|
||||
args = {"q": query, "format": "json", **params}
|
||||
url = f"{conf['BRAIN_SEARCH_URL'].rstrip('/')}/search?{urllib.parse.urlencode(args)}"
|
||||
request = urllib.request.Request(url)
|
||||
token = f"{conf['BRAIN_SEARCH_USER']}:{conf['BRAIN_SEARCH_PASS']}".encode()
|
||||
import base64
|
||||
request.add_header("Authorization", "Basic " + base64.b64encode(token).decode())
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
return json.loads(response.read().decode())
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="web search via SearXNG")
|
||||
parser.add_argument("query")
|
||||
parser.add_argument("-n", "--limit", type=int, default=ws.DEFAULT_LIMIT)
|
||||
parser.add_argument("--site", help="restrict to one domain")
|
||||
parser.add_argument("--lang", help="language code, e.g. de")
|
||||
parser.add_argument("--fresh", choices=["day", "week", "month", "year"],
|
||||
help="time range")
|
||||
parser.add_argument("--category", help="SearXNG category, e.g. it, science, news")
|
||||
parser.add_argument("--engines", help="comma separated engine list")
|
||||
parser.add_argument("--json", action="store_true")
|
||||
parser.add_argument("--refresh", action="store_true", help="bypass the cache")
|
||||
parser.add_argument("--ttl", type=float, default=ws.CACHE_TTL)
|
||||
parser.add_argument("--timeout", type=int, default=25)
|
||||
parser.add_argument("--force", action="store_true",
|
||||
help="send even if the query looks like PII")
|
||||
args = parser.parse_args()
|
||||
|
||||
query = f"site:{args.site} {args.query}" if args.site else args.query
|
||||
|
||||
reason = ws.phi_reason(query)
|
||||
if reason and not args.force:
|
||||
print(f"refused: {reason}. This query would leave the host.", file=sys.stderr)
|
||||
print("Rephrase without identifiers, or pass --force if it is genuinely public.",
|
||||
file=sys.stderr)
|
||||
return 2
|
||||
|
||||
params = {}
|
||||
if args.lang:
|
||||
params["language"] = args.lang
|
||||
if args.fresh:
|
||||
params["time_range"] = args.fresh
|
||||
if args.category:
|
||||
params["categories"] = args.category
|
||||
if args.engines:
|
||||
params["engines"] = args.engines
|
||||
|
||||
key = ws.cache_key(query, params)
|
||||
conn = ws.open_cache(CACHE)
|
||||
|
||||
if not args.refresh:
|
||||
cached = ws.cache_get(conn, key, ttl=args.ttl)
|
||||
if cached is not None:
|
||||
out = ws.project(cached, limit=args.limit)
|
||||
out["cached"] = True
|
||||
sys.stdout.write(json.dumps(out, indent=2, ensure_ascii=False) + "\n"
|
||||
if args.json else to_yaml(out))
|
||||
return 0
|
||||
|
||||
conf = load_config()
|
||||
LOCK.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# One request at a time across every agent on this host: the instance
|
||||
# suspends engines for minutes when several of us ask at once.
|
||||
with open(LOCK, "w") as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
|
||||
payload = None
|
||||
for attempt in range(1 + len(ws.RETRY_BACKOFF)):
|
||||
delay = ws.wait_for(ws.last_call(conn), time.time())
|
||||
if delay:
|
||||
time.sleep(delay)
|
||||
ws.mark_call(conn)
|
||||
try:
|
||||
payload = fetch(conf, query, params, args.timeout)
|
||||
except Exception as error: # noqa: BLE001 - report, do not crash
|
||||
print(f"request failed: {error}", file=sys.stderr)
|
||||
return 3
|
||||
if ws.classify(payload) == "ok":
|
||||
break
|
||||
if attempt < len(ws.RETRY_BACKOFF):
|
||||
time.sleep(ws.RETRY_BACKOFF[attempt])
|
||||
|
||||
if ws.classify(payload) == "ok":
|
||||
ws.cache_put(conn, key, payload)
|
||||
|
||||
out = ws.project(payload, limit=args.limit)
|
||||
sys.stdout.write(json.dumps(out, indent=2, ensure_ascii=False) + "\n"
|
||||
if args.json else to_yaml(out))
|
||||
return 0 if out["status"] == "ok" else 3
|
||||
def main(argv: list[str]) -> int:
|
||||
print(
|
||||
"bin/web/search is deprecated; use bin/web/search.go",
|
||||
file=sys.stderr,
|
||||
)
|
||||
target = ROOT / "bin" / "web" / "search.go"
|
||||
os.execvp("go", ["go", "run", str(target), *argv])
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
|
||||
Executable
+232
@@ -0,0 +1,232 @@
|
||||
//usr/bin/env go run "$0" "$@"; exit
|
||||
//
|
||||
// bin/web/search.go - SearXNG as the second independent source (D3).
|
||||
//
|
||||
// ./bin/web/search.go "LadybugDB vector search"
|
||||
// ./bin/web/search.go "model2vec" --category it --json
|
||||
// ./bin/web/search.go "postgres" --site github.com --fresh year
|
||||
//
|
||||
// Empty results mean throttled, not "nothing exists". Exit 2 = PII refuse, 3 = throttled.
|
||||
// Config: $BRAIN_SEARCH_ENV (default $HOME/.config/brain/search.env).
|
||||
// NOTE: never run `gofmt -w` on this file — it breaks the shebang.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/eSlider/2dph/internal/websearch"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
func main() {
|
||||
os.Exit(run(os.Args[1:]))
|
||||
}
|
||||
|
||||
func run(args []string) int {
|
||||
var (
|
||||
query, site, lang, fresh, category, engines string
|
||||
limit = websearch.DefaultLimit
|
||||
jsonOut, refresh, force bool
|
||||
ttl = float64(websearch.CacheTTL)
|
||||
timeout = 25
|
||||
)
|
||||
i := 0
|
||||
for i < len(args) {
|
||||
a := args[i]
|
||||
switch {
|
||||
case a == "--json":
|
||||
jsonOut = true
|
||||
case a == "--refresh":
|
||||
refresh = true
|
||||
case a == "--force":
|
||||
force = true
|
||||
case (a == "-n" || a == "--limit") && i+1 < len(args):
|
||||
i++
|
||||
n, err := strconv.Atoi(args[i])
|
||||
if err != nil || n < 0 {
|
||||
fmt.Fprintln(os.Stderr, "web/search: --limit must be a non-negative integer")
|
||||
return 2
|
||||
}
|
||||
limit = n
|
||||
case a == "--site" && i+1 < len(args):
|
||||
i++
|
||||
site = args[i]
|
||||
case a == "--lang" && i+1 < len(args):
|
||||
i++
|
||||
lang = args[i]
|
||||
case a == "--fresh" && i+1 < len(args):
|
||||
i++
|
||||
fresh = args[i]
|
||||
case a == "--category" && i+1 < len(args):
|
||||
i++
|
||||
category = args[i]
|
||||
case a == "--engines" && i+1 < len(args):
|
||||
i++
|
||||
engines = args[i]
|
||||
case a == "--ttl" && i+1 < len(args):
|
||||
i++
|
||||
v, err := strconv.ParseFloat(args[i], 64)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "web/search: --ttl must be a number")
|
||||
return 2
|
||||
}
|
||||
ttl = v
|
||||
case a == "--timeout" && i+1 < len(args):
|
||||
i++
|
||||
n, err := strconv.Atoi(args[i])
|
||||
if err != nil || n <= 0 {
|
||||
fmt.Fprintln(os.Stderr, "web/search: --timeout must be a positive integer")
|
||||
return 2
|
||||
}
|
||||
timeout = n
|
||||
case a == "-h" || a == "--help":
|
||||
fmt.Fprintln(os.Stderr, `usage: bin/web/search.go QUERY [--json] [-n N] [--site HOST] [--lang LANG] [--fresh day|week|month|year] [--category CAT] [--engines LIST] [--refresh] [--force]`)
|
||||
return 0
|
||||
case len(a) > 0 && a[0] != '-' && query == "":
|
||||
query = a
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "web/search: unknown flag %s\n", a)
|
||||
return 2
|
||||
}
|
||||
i++
|
||||
}
|
||||
if query == "" {
|
||||
fmt.Fprintln(os.Stderr, "web/search: query required")
|
||||
return 2
|
||||
}
|
||||
if site != "" {
|
||||
query = "site:" + site + " " + query
|
||||
}
|
||||
if reason := websearch.PHIReason(query); reason != "" && !force {
|
||||
fmt.Fprintf(os.Stderr, "refused: %s. This query would leave the host.\n", reason)
|
||||
fmt.Fprintln(os.Stderr, "Rephrase without identifiers, or pass --force if it is genuinely public.")
|
||||
return 2
|
||||
}
|
||||
|
||||
params := map[string]string{}
|
||||
if lang != "" {
|
||||
params["language"] = lang
|
||||
}
|
||||
if fresh != "" {
|
||||
params["time_range"] = fresh
|
||||
}
|
||||
if category != "" {
|
||||
params["categories"] = category
|
||||
}
|
||||
if engines != "" {
|
||||
params["engines"] = engines
|
||||
}
|
||||
|
||||
cachePath := os.Getenv("BRAIN_SEARCH_CACHE")
|
||||
if cachePath == "" {
|
||||
cachePath = os.Getenv("HOME") + "/.cache/brain/web-search.sqlite"
|
||||
}
|
||||
cache, err := websearch.OpenCache(cachePath)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "web/search: cache: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
defer cache.Close()
|
||||
|
||||
key := websearch.CacheKey(query, params)
|
||||
now := float64(time.Now().Unix())
|
||||
if !refresh {
|
||||
if cached, err := cache.Get(key, ttl, now); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "web/search: cache: %v\n", err)
|
||||
return 1
|
||||
} else if cached != nil {
|
||||
out := websearch.Project(*cached, limit, websearch.DefaultSnippetChars)
|
||||
out.Cached = true
|
||||
return writeOut(out, jsonOut)
|
||||
}
|
||||
}
|
||||
|
||||
envPath := os.Getenv("BRAIN_SEARCH_ENV")
|
||||
if envPath == "" {
|
||||
envPath = os.Getenv("HOME") + "/.config/brain/search.env"
|
||||
}
|
||||
conf, err := websearch.LoadConfig(envPath)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "web/search: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
|
||||
lockPath := cachePath + ".lock"
|
||||
lock, err := os.OpenFile(lockPath, os.O_CREATE|os.O_RDWR, 0o600)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "web/search: lock: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
defer lock.Close()
|
||||
if err := unix.Flock(int(lock.Fd()), unix.LOCK_EX); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "web/search: lock: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
defer unix.Flock(int(lock.Fd()), unix.LOCK_UN)
|
||||
|
||||
var payload websearch.Payload
|
||||
attempts := 1 + len(websearch.RetryBackoff)
|
||||
client := &http.Client{}
|
||||
for attempt := 0; attempt < attempts; attempt++ {
|
||||
last, err := cache.LastCall()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "web/search: cache: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
if delay := websearch.WaitFor(last, float64(time.Now().Unix()), websearch.MinInterval); delay > 0 {
|
||||
time.Sleep(time.Duration(delay * float64(time.Second)))
|
||||
}
|
||||
if err := cache.MarkCall(float64(time.Now().Unix())); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "web/search: cache: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
payload, err = websearch.Fetch(client, conf, query, params, time.Duration(timeout)*time.Second)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "request failed: %v\n", err)
|
||||
return 3
|
||||
}
|
||||
if websearch.Classify(payload) == websearch.StatusOK {
|
||||
break
|
||||
}
|
||||
if attempt < len(websearch.RetryBackoff) {
|
||||
time.Sleep(time.Duration(websearch.RetryBackoff[attempt] * float64(time.Second)))
|
||||
}
|
||||
}
|
||||
|
||||
if websearch.Classify(payload) == websearch.StatusOK {
|
||||
if err := cache.Put(key, payload, float64(time.Now().Unix())); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "web/search: cache: %v\n", err)
|
||||
}
|
||||
}
|
||||
out := websearch.Project(payload, limit, websearch.DefaultSnippetChars)
|
||||
code := writeOut(out, jsonOut)
|
||||
if out.Status != websearch.StatusOK && code == 0 {
|
||||
return 3
|
||||
}
|
||||
return code
|
||||
}
|
||||
|
||||
func writeOut(out websearch.Output, jsonOut bool) int {
|
||||
if jsonOut {
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
enc.SetEscapeHTML(false)
|
||||
if err := enc.Encode(out); err != nil {
|
||||
return 1
|
||||
}
|
||||
if out.Status != websearch.StatusOK {
|
||||
return 3
|
||||
}
|
||||
return 0
|
||||
}
|
||||
fmt.Print(out.YAML())
|
||||
if out.Status != websearch.StatusOK {
|
||||
return 3
|
||||
}
|
||||
return 0
|
||||
}
|
||||
Reference in New Issue
Block a user