eSlider/2dph
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

build(ci): uv toolchain, release-please semver; feat(skills): vendor tools self-contained (no symlinks, relative refs)

Browse Source 
- bin/db/psql-yq + bin/web/search + tools/{yamlout,websearch} vendored as real files
- bin/db/ssh-tunnel added (OnlyOffice VM pg on 5433)
- skills reference local bin/ paths; no agent-skills/abs links in git
- pyproject.toml + uv.lock; CI installs via uv sync --frozen
- release-please auto-tags semver from conventional commits when green
- LICENSE MIT, badges/mermaid README
This commit is contained in:
eSlider
2026-08-10 21:24:42 +01:00
parent d4a88eead7
commit 63d3be0e19
31 changed files with 1795 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/db/psql-yq
-1
View File
@@ -1 +0,0 @@
/mnt/8TB/projects/ai/agent-skills/bin/db/psql-yq
bin/db/psql-yq
Executable
+116
View File
@@ -0,0 +1,116 @@
#!/usr/bin/env bash
# db/psql-yq - read any Postgres as YAML, cheaply and read-only.
#
# db/psql-yq --profile onlyoffice -t document_asset -l 20 # table sample
# db/psql-yq --profile onlyoffice -s document_asset # column list
# db/psql-yq --profile onlyoffice -c 'SELECT ...' # query -> YAML
# db/psql-yq --container my-pg --db app -c '...' # ad-hoc container
# db/psql-yq --dsn 'postgres://u@h:5432/db' -c '...'
# db/psql-yq -r 'SELECT ...' # raw rows, no YAML
#
# Profiles live in ~/.config/brain/db-profiles.yml so that credentials and
# hostnames stay out of every project repo. A profile is:
#
# onlyoffice:
# host: 127.0.0.1
# port: 5433
# user: onlyoffice
# db: onlyoffice
# password_env_file: /home/ano/.config/ops/onlyoffice.env
#
# The onlyoffice Postgres listens inside the QEMU VM (host port 32, SSH only).
# Credentials come from /etc/onlyoffice/documentserver/local.json inside the
# VM (dbUser/dbPass), read via db/ssh-tunnel, never committed.
# db/ssh-tunnel opens 127.0.0.1:5433 -> vm:5432 before querying.
#
# Read-only guard: any DML or DDL keyword is rejected, whatever the profile.
set -euo pipefail
PROFILES="${BRAIN_DB_PROFILES:-$HOME/.config/brain/db-profiles.yml}"
IMAGE="${BRAIN_PSQL_IMAGE:-postgres:13}"
LIMIT="${PSQLYQ_LIMIT:-50}"
MODE=c
ARG=""
PROFILE=""
CONTAINER=""
DSN=""
DB=""
USER_NAME=""
usage() { sed -n '2,26p' "$0" | sed 's/^# \{0,1\}//'; }
[[ $# -eq 0 ]] && { usage; exit 1; }
while [[ $# -gt 0 ]]; do
case "$1" in
--profile) PROFILE="$2"; shift 2 ;;
--container) CONTAINER="$2"; shift 2 ;;
--dsn) DSN="$2"; shift 2 ;;
--db) DB="$2"; shift 2 ;;
--user) USER_NAME="$2"; shift 2 ;;
-c) MODE=c; ARG="$2"; shift 2 ;;
-t) MODE=t; ARG="$2"; shift 2 ;;
-s) MODE=s; ARG="$2"; shift 2 ;;
-r) MODE=r; ARG="$2"; shift 2 ;;
-l) LIMIT="$2"; shift 2 ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown arg: $1" >&2; usage; exit 1 ;;
esac
done
assert_readonly() {
local s="${1,,}"
if [[ "$s" =~ (^|[^a-z])(insert|update|delete|drop|truncate|alter|create|grant|revoke|vacuum|copy)[[:space:]] ]]; then
echo "db/psql-yq: read-only, query rejected" >&2
exit 3
fi
}
prof() { yq -r ".\"$PROFILE\".$1 // \"\"" "$PROFILES" 2>/dev/null; }
build_client() {
if [[ -n "$DSN" ]]; then
PGCLI=(docker run --rm "$IMAGE" psql "$DSN" -X -A -t)
return
fi
if [[ -n "$CONTAINER" ]]; then
PGCLI=(docker exec "$CONTAINER" psql -U "${USER_NAME:-postgres}" -d "${DB:?--db required with --container}" -X -A -t)
return
fi
[[ -n "$PROFILE" ]] || { echo "need --profile, --container or --dsn" >&2; exit 2; }
[[ -f "$PROFILES" ]] || { echo "no profiles file: $PROFILES" >&2; exit 2; }
[[ "$(yq -r "has(\"$PROFILE\")" "$PROFILES")" == "true" ]] \
|| { echo "unknown profile '$PROFILE' in $PROFILES" >&2; exit 2; }
local container host port user db network envfile
container="$(prof container)"; host="$(prof host)"; port="$(prof port)"
user="$(prof user)"; db="$(prof db)"; network="$(prof network)"
envfile="$(prof password_env_file)"
if [[ -n "$container" ]]; then
PGCLI=(docker exec "$container" psql -U "${user:-postgres}" -d "$db" -X -A -t)
else
if [[ -n "$envfile" ]]; then
[[ -f "$envfile" ]] || { echo "password_env_file missing: $envfile" >&2; exit 2; }
# shellcheck disable=SC1090
set -a; . "$envfile"; set +a
fi
local net=()
[[ -n "$network" ]] && net=(--network "$network")
PGCLI=(docker run --rm "${net[@]}" -e PGPASSWORD="${PGPASSWORD:?password unset for profile $PROFILE}" \
"$IMAGE" psql -h "$host" -p "${port:-5432}" -U "$user" -d "$db" -X -A -t)
fi
}
build_client
case "$MODE" in
r) assert_readonly "$ARG"; "${PGCLI[@]}" -c "$ARG" ;;
t) assert_readonly "SELECT * FROM $ARG"
"${PGCLI[@]}" -c "SELECT json_agg(row_to_json(t)) FROM (SELECT * FROM $ARG LIMIT $LIMIT) t" | yq -P . ;;
s) "${PGCLI[@]}" -c "SELECT column_name || ':' || data_type FROM information_schema.columns WHERE table_name='$ARG' ORDER BY ordinal_position" ;;
c) assert_readonly "$ARG"
"${PGCLI[@]}" -c "SELECT json_agg(row_to_json(t)) FROM ($ARG) t" | yq -P . ;;
esac
bin/db/ssh-tunnel
Executable
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env bash
# db/ssh-tunnel - open 127.0.0.1:5433 -> onlyoffice VM Postgres (host port 32).
#
# db/ssh-tunnel # background tunnel, ready for db/psql-yq
# db/ssh-tunnel --check # exit 0 if a tunnel is already up
# db/ssh-tunnel --stop # kill any tunnel owned by this tool
#
# OnlyOffice runs as a QEMU VM (docker network office_default), SSH on host
# port 32. Its Postgres listens on 127.0.0.1:5432 inside the VM and is not
# exposed. This forwards localhost:5433 to it so db/psql-yq can query with
# the onlyoffice profile. Credentials are never part of the tunnel.
set -euo pipefail
SRC="${BRAIN_TUNNEL_LOCAL:-127.0.0.1:5433}"
DST="${BRAIN_TUNNEL_REMOTE:-127.0.0.1:5432}"
SSH_PORT="${BRAIN_TUNNEL_SSH_PORT:-32}"
SSH_USER="${BRAIN_TUNNEL_SSH_USER:-root}"
SSH_HOST="${BRAIN_TUNNEL_SSH_HOST:-127.0.0.1}"
MARKER="2dph-ssh-tunnel"
case "${1:-}" in
--check)
ss -tln 2>/dev/null | grep -q "${SRC%:*}:${SRC#*:}" && exit 0
pgrep -f "${MARKER}" >/dev/null && exit 0
exit 1
;;
--stop)
pkill -f "${MARKER}" && echo "tunnel stopped" || echo "no tunnel running"
exit 0
;;
-h|--help)
sed -n '2,9p' "$0" | sed 's/^# \{0,1\}//'
exit 0
;;
"")
[ -f "$HOME/.ssh/config" ] || { echo "db/ssh-tunnel: ~/.ssh/config missing" >&2; exit 1; }
if db/ssh-tunnel --check; then
echo "tunnel already up on ${SRC}"
exit 0
fi
ssh -f -N -M -S "$HOME/.ssh/2dph-tunnel.sock" \
-L "${SRC}:${DST}" -p "$SSH_PORT" "${SSH_USER}@${SSH_HOST}" \
&& echo "tunnel up on ${SRC} (-> vm:${DST})"
exit 0
;;
*)
echo "unknown arg: $1" >&2
exit 1
;;
esac
bin/web/search
-1
View File
@@ -1 +0,0 @@
/mnt/8TB/projects/ai/agent-skills/bin/web/search
bin/web/search
Executable
+150
View File
@@ -0,0 +1,150 @@
#!/usr/bin/env python3
"""web/search - web search through the self-hosted SearXNG at search.ops.io.
bin/web/search "LadybugDB vector search"
bin/web/search "model2vec multilingual" --site github.com
bin/web/search "uclancy" --category it -n 3 --json | jq -r '.results[].url'
bin/web/search "sqlite-vec" --refresh # ignore the cached answer
This complements bin/kb/search: the knowledge base holds our own facts, this
reaches the public web. Use it as the second, independent source that the
detective method asks for.
Exit codes: 0 results, 2 refused as possible PII, 3 throttled (not "nothing
found" - the instance answers 200 with an empty list when it throttles).
"""
from __future__ import annotations
import argparse
import fcntl
import json
import os
import sys
import time
import urllib.parse
import urllib.request
from pathlib import Path
TOOLS = Path(__file__).resolve().parents[1].parent / "tools"
sys.path.insert(0, str(TOOLS))
sys.path.insert(0, str(TOOLS / "web-search"))
import websearch as ws # noqa: E402
from yamlout import to_yaml # noqa: E402
CONFIG = Path(os.environ.get("BRAIN_SEARCH_ENV", Path.home() / ".config/brain/search.env"))
CACHE = Path(os.environ.get("BRAIN_SEARCH_CACHE", Path.home() / ".cache/brain/web-search.sqlite"))
LOCK = CACHE.with_suffix(".lock")
def load_config() -> dict:
if not CONFIG.exists():
sys.exit(f"no credentials at {CONFIG} (mode 600, BRAIN_SEARCH_URL/USER/PASS)")
conf = {}
for line in CONFIG.read_text().splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, _, value = line.partition("=")
conf[key.strip()] = value.strip().strip("\"'")
missing = {"BRAIN_SEARCH_URL", "BRAIN_SEARCH_USER", "BRAIN_SEARCH_PASS"} - conf.keys()
if missing:
sys.exit(f"{CONFIG} is missing {', '.join(sorted(missing))}")
return conf
def fetch(conf: dict, query: str, params: dict, timeout: int) -> dict:
args = {"q": query, "format": "json", **params}
url = f"{conf['BRAIN_SEARCH_URL'].rstrip('/')}/search?{urllib.parse.urlencode(args)}"
request = urllib.request.Request(url)
token = f"{conf['BRAIN_SEARCH_USER']}:{conf['BRAIN_SEARCH_PASS']}".encode()
import base64
request.add_header("Authorization", "Basic " + base64.b64encode(token).decode())
with urllib.request.urlopen(request, timeout=timeout) as response:
return json.loads(response.read().decode())
def main() -> int:
parser = argparse.ArgumentParser(description="web search via SearXNG")
parser.add_argument("query")
parser.add_argument("-n", "--limit", type=int, default=ws.DEFAULT_LIMIT)
parser.add_argument("--site", help="restrict to one domain")
parser.add_argument("--lang", help="language code, e.g. de")
parser.add_argument("--fresh", choices=["day", "week", "month", "year"],
help="time range")
parser.add_argument("--category", help="SearXNG category, e.g. it, science, news")
parser.add_argument("--engines", help="comma separated engine list")
parser.add_argument("--json", action="store_true")
parser.add_argument("--refresh", action="store_true", help="bypass the cache")
parser.add_argument("--ttl", type=float, default=ws.CACHE_TTL)
parser.add_argument("--timeout", type=int, default=25)
parser.add_argument("--force", action="store_true",
help="send even if the query looks like PII")
args = parser.parse_args()
query = f"site:{args.site} {args.query}" if args.site else args.query
reason = ws.phi_reason(query)
if reason and not args.force:
print(f"refused: {reason}. This query would leave the host.", file=sys.stderr)
print("Rephrase without identifiers, or pass --force if it is genuinely public.",
file=sys.stderr)
return 2
params = {}
if args.lang:
params["language"] = args.lang
if args.fresh:
params["time_range"] = args.fresh
if args.category:
params["categories"] = args.category
if args.engines:
params["engines"] = args.engines
key = ws.cache_key(query, params)
conn = ws.open_cache(CACHE)
if not args.refresh:
cached = ws.cache_get(conn, key, ttl=args.ttl)
if cached is not None:
out = ws.project(cached, limit=args.limit)
out["cached"] = True
sys.stdout.write(json.dumps(out, indent=2, ensure_ascii=False) + "\n"
if args.json else to_yaml(out))
return 0
conf = load_config()
LOCK.parent.mkdir(parents=True, exist_ok=True)
# One request at a time across every agent on this host: the instance
# suspends engines for minutes when several of us ask at once.
with open(LOCK, "w") as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
payload = None
for attempt in range(1 + len(ws.RETRY_BACKOFF)):
delay = ws.wait_for(ws.last_call(conn), time.time())
if delay:
time.sleep(delay)
ws.mark_call(conn)
try:
payload = fetch(conf, query, params, args.timeout)
except Exception as error: # noqa: BLE001 - report, do not crash
print(f"request failed: {error}", file=sys.stderr)
return 3
if ws.classify(payload) == "ok":
break
if attempt < len(ws.RETRY_BACKOFF):
time.sleep(ws.RETRY_BACKOFF[attempt])
if ws.classify(payload) == "ok":
ws.cache_put(conn, key, payload)
out = ws.project(payload, limit=args.limit)
sys.stdout.write(json.dumps(out, indent=2, ensure_ascii=False) + "\n"
if args.json else to_yaml(out))
return 0 if out["status"] == "ok" else 3
if __name__ == "__main__":
sys.exit(main())