feat(simulator): web data-plane explorer
Single-page view over the Parquet lake: live partition tree with rolled-up sizes and a SQL console gated by the same read-only rules as the peer query channel. Ships in the monitoring profile.
This commit is contained in:
@@ -0,0 +1,160 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Swarm data-plane explorer</title>
|
||||
<style>
|
||||
:root {
|
||||
--bg: #0e1116; --panel: #161b22; --border: #2b3240;
|
||||
--fg: #d7dde6; --dim: #8b95a5; --accent: #4ec9b0; --warn: #e5c07b;
|
||||
color-scheme: dark;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body {
|
||||
margin: 0; background: var(--bg); color: var(--fg);
|
||||
font: 14px/1.5 "JetBrains Mono", ui-monospace, monospace;
|
||||
display: grid; grid-template-columns: 380px 1fr; height: 100vh;
|
||||
}
|
||||
header {
|
||||
grid-column: 1 / -1; padding: 10px 16px; border-bottom: 1px solid var(--border);
|
||||
display: flex; align-items: baseline; gap: 14px;
|
||||
}
|
||||
header h1 { margin: 0; font-size: 15px; font-weight: 600; }
|
||||
header .totals { color: var(--dim); font-size: 12px; }
|
||||
body { grid-template-rows: auto 1fr; }
|
||||
aside {
|
||||
border-right: 1px solid var(--border); overflow: auto; padding: 10px 8px;
|
||||
background: var(--panel);
|
||||
}
|
||||
main { display: flex; flex-direction: column; overflow: hidden; }
|
||||
details { margin-left: 12px; }
|
||||
details > summary {
|
||||
cursor: pointer; padding: 1px 4px; border-radius: 4px; white-space: nowrap;
|
||||
}
|
||||
details > summary:hover { background: #1f2733; }
|
||||
summary .meta { color: var(--dim); font-size: 11px; margin-left: 6px; }
|
||||
.leaf { margin-left: 30px; color: var(--dim); font-size: 12px; white-space: nowrap; }
|
||||
.leaf.current { color: var(--warn); }
|
||||
.sql-box { padding: 12px 14px; border-bottom: 1px solid var(--border); }
|
||||
textarea {
|
||||
width: 100%; height: 92px; background: var(--panel); color: var(--fg);
|
||||
border: 1px solid var(--border); border-radius: 6px; padding: 8px 10px;
|
||||
font: 13px/1.45 inherit; resize: vertical;
|
||||
}
|
||||
textarea:focus { outline: 1px solid var(--accent); }
|
||||
.row { display: flex; gap: 8px; margin-top: 8px; align-items: center; flex-wrap: wrap; }
|
||||
button {
|
||||
background: var(--accent); color: #0b0e12; border: 0; border-radius: 6px;
|
||||
padding: 5px 14px; font: 600 13px inherit; cursor: pointer;
|
||||
}
|
||||
button.sample {
|
||||
background: transparent; color: var(--dim); border: 1px solid var(--border);
|
||||
font-weight: 400; font-size: 12px;
|
||||
}
|
||||
button.sample:hover { color: var(--fg); border-color: var(--dim); }
|
||||
.hint { color: var(--dim); font-size: 12px; }
|
||||
#results { flex: 1; overflow: auto; padding: 12px 14px; }
|
||||
table { border-collapse: collapse; font-size: 12.5px; }
|
||||
th, td { border: 1px solid var(--border); padding: 3px 9px; text-align: left; white-space: nowrap; }
|
||||
th { background: var(--panel); position: sticky; top: 0; }
|
||||
.error { color: #e06c75; white-space: pre-wrap; }
|
||||
.note { color: var(--dim); font-size: 12px; margin: 6px 0; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>Swarm data-plane explorer</h1>
|
||||
<span class="totals" id="totals">scanning…</span>
|
||||
<span class="totals" style="margin-left:auto">read-only SQL · same gate as the peer query channel</span>
|
||||
</header>
|
||||
|
||||
<aside id="tree"></aside>
|
||||
|
||||
<main>
|
||||
<div class="sql-box">
|
||||
<textarea id="sql" spellcheck="false">SELECT sensor, count(*) AS rows, count(DISTINCT drone) AS drones
|
||||
FROM telemetry GROUP BY sensor ORDER BY rows DESC</textarea>
|
||||
<div class="row">
|
||||
<button id="run">Run</button>
|
||||
<span class="hint">views: <b>telemetry</b>, <b>detections</b>, <b>state</b> · Ctrl+Enter to run</span>
|
||||
</div>
|
||||
<div class="row" id="samples"></div>
|
||||
</div>
|
||||
<div id="results"><p class="note">Run a query to see results. The tree on the left refreshes every 10 s while the swarm writes.</p></div>
|
||||
</main>
|
||||
|
||||
<script>
|
||||
const SAMPLES = [
|
||||
["fleet overview", "SELECT drone, min(to_timestamp(ts_ns/1e9)) AS first_seen, max(to_timestamp(ts_ns/1e9)) AS last_seen, count(*) AS rows FROM telemetry GROUP BY drone ORDER BY drone"],
|
||||
["battery timeline", "SELECT drone, time_bucket(INTERVAL 10 seconds, to_timestamp(ts_ns/1e9)) AS t, avg(level_pct) AS battery FROM telemetry WHERE sensor='battery' GROUP BY 1,2 ORDER BY 2,1 LIMIT 100"],
|
||||
["detections by class", "SELECT cls, count(*) AS n, round(avg(confidence),3) AS avg_conf FROM detections GROUP BY cls ORDER BY n DESC"],
|
||||
["peer link quality", "SELECT drone, peer_id, round(avg(rssi_dbm),1) AS rssi, round(avg(distance_m),1) AS dist_m FROM telemetry WHERE sensor='rssi' GROUP BY 1,2 ORDER BY 1,2"],
|
||||
["partitions", "SELECT dataset, drone, sensor, count(*) AS rows FROM telemetry GROUP BY ALL ORDER BY 2,3"],
|
||||
["schema", "DESCRIBE telemetry"],
|
||||
];
|
||||
|
||||
function fmtBytes(b) {
|
||||
if (b == null) return "";
|
||||
const u = ["B","KB","MB","GB"]; let i = 0;
|
||||
while (b >= 1024 && i < u.length - 1) { b /= 1024; i++; }
|
||||
return b.toFixed(i ? 1 : 0) + " " + u[i];
|
||||
}
|
||||
|
||||
function renderNode(name, node, open) {
|
||||
const children = node.children || {};
|
||||
const keys = Object.keys(children);
|
||||
if (!keys.length) {
|
||||
const cls = name.includes("current") || name.endsWith(".tmp") ? "leaf current" : "leaf";
|
||||
return `<div class="${cls}">${name} <span class="meta">${fmtBytes(node.bytes)}</span></div>`;
|
||||
}
|
||||
const meta = `<span class="meta">${node.files ?? ""} files · ${fmtBytes(node.bytes)}</span>`;
|
||||
const inner = keys.map(k => renderNode(k, children[k], false)).join("");
|
||||
return `<details ${open ? "open" : ""}><summary>${name} ${meta}</summary>${inner}</details>`;
|
||||
}
|
||||
|
||||
async function refreshTree() {
|
||||
try {
|
||||
const r = await (await fetch("/api/tree")).json();
|
||||
document.getElementById("totals").textContent =
|
||||
`${r.total_files} parquet files · ${fmtBytes(r.total_bytes)}`;
|
||||
const children = (r.tree.children) || {};
|
||||
document.getElementById("tree").innerHTML =
|
||||
Object.keys(children).sort().map(k => renderNode(k, children[k], true)).join("")
|
||||
|| '<p class="note">no parquet files yet</p>';
|
||||
} catch { /* transient; try again on next tick */ }
|
||||
}
|
||||
|
||||
async function run() {
|
||||
const sql = document.getElementById("sql").value;
|
||||
const out = document.getElementById("results");
|
||||
out.innerHTML = '<p class="note">running…</p>';
|
||||
const r = await (await fetch("/api/query", {
|
||||
method: "POST", headers: {"Content-Type": "application/json"},
|
||||
body: JSON.stringify({sql}),
|
||||
})).json();
|
||||
if (r.error) { out.innerHTML = `<p class="error">${r.error}</p>`; return; }
|
||||
const head = r.columns.map(c => `<th>${c}</th>`).join("");
|
||||
const body = r.rows.map(row =>
|
||||
`<tr>${row.map(v => `<td>${v === null ? "<i>null</i>" : String(v)}</td>`).join("")}</tr>`).join("");
|
||||
out.innerHTML =
|
||||
`<p class="note">${r.rows.length} rows${r.truncated ? " (truncated)" : ""}</p>` +
|
||||
`<table><thead><tr>${head}</tr></thead><tbody>${body}</tbody></table>`;
|
||||
}
|
||||
|
||||
document.getElementById("run").onclick = run;
|
||||
document.getElementById("sql").addEventListener("keydown", e => {
|
||||
if ((e.ctrlKey || e.metaKey) && e.key === "Enter") run();
|
||||
});
|
||||
document.getElementById("samples").innerHTML =
|
||||
SAMPLES.map(([label], i) => `<button class="sample" data-i="${i}">${label}</button>`).join("");
|
||||
document.getElementById("samples").onclick = e => {
|
||||
const i = e.target.dataset.i;
|
||||
if (i != null) { document.getElementById("sql").value = SAMPLES[i][1]; run(); }
|
||||
};
|
||||
|
||||
refreshTree();
|
||||
setInterval(refreshTree, 10000);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,151 @@
|
||||
"""Data-plane explorer: a web view over the Hive-partitioned Parquet lake.
|
||||
|
||||
Serves three things:
|
||||
/ single-page UI (partition tree + read-only SQL console)
|
||||
/api/tree partition hierarchy with file counts and bytes, live
|
||||
/api/query gated read-only DuckDB SQL, same statement rules as the
|
||||
peer query channel (SELECT/WITH only, single statement)
|
||||
|
||||
The point is doctrinal, not just convenient: the explorer reuses the exact
|
||||
read-only SQL contract that drones expose to each other, so "looking at the
|
||||
data plane" on the bench exercises the same path a peer would use in flight.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
import duckdb
|
||||
|
||||
DATA_DIR = Path(os.environ.get("DATA_DIR", "./data"))
|
||||
PORT = int(os.environ.get("EXPLORER_PORT", "8088"))
|
||||
ROW_LIMIT = int(os.environ.get("ROW_LIMIT", "500"))
|
||||
STATIC_DIR = Path(__file__).parent
|
||||
|
||||
# Same spirit as the forced-command gate on a real drone: one statement,
|
||||
# must be a read, no statement that could write, configure, or reach out.
|
||||
_ALLOWED_START = re.compile(r"^\s*(SELECT|WITH|DESCRIBE|SUMMARIZE|SHOW)\b", re.IGNORECASE)
|
||||
_FORBIDDEN = re.compile(
|
||||
r"\b(INSERT|UPDATE|DELETE|CREATE|DROP|ALTER|ATTACH|DETACH|COPY|EXPORT"
|
||||
r"|INSTALL|LOAD|SET|RESET|PRAGMA|CALL|IMPORT|VACUUM|CHECKPOINT)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def gate(sql: str) -> str | None:
|
||||
"""Return a rejection reason, or None if the statement passes."""
|
||||
stripped = re.sub(r"--[^\n]*|/\*.*?\*/", " ", sql, flags=re.DOTALL).strip().rstrip(";")
|
||||
if not stripped:
|
||||
return "empty statement"
|
||||
if ";" in stripped:
|
||||
return "multiple statements are not allowed"
|
||||
if not _ALLOWED_START.match(stripped):
|
||||
return "only SELECT / WITH / DESCRIBE / SUMMARIZE / SHOW are allowed"
|
||||
if _FORBIDDEN.search(stripped):
|
||||
return "statement contains a forbidden keyword"
|
||||
return None
|
||||
|
||||
|
||||
def connect() -> duckdb.DuckDBPyConnection:
|
||||
"""Fresh connection with the three datasets pre-registered as views."""
|
||||
con = duckdb.connect()
|
||||
for ds in ("telemetry", "detections", "state"):
|
||||
pattern = f"{DATA_DIR}/dataset={ds}/**/*.parquet"
|
||||
try:
|
||||
con.execute(
|
||||
f"CREATE VIEW {ds} AS SELECT * FROM read_parquet("
|
||||
f"'{pattern}', hive_partitioning=true, union_by_name=true)"
|
||||
)
|
||||
except duckdb.Error:
|
||||
pass # dataset not written yet; view simply won't exist
|
||||
return con
|
||||
|
||||
|
||||
def tree() -> dict:
|
||||
"""Partition hierarchy: dataset -> flight -> drone -> leafs, with sizes."""
|
||||
root: dict = {}
|
||||
total_bytes = 0
|
||||
total_files = 0
|
||||
for f in sorted(DATA_DIR.rglob("*.parquet")):
|
||||
rel = f.relative_to(DATA_DIR)
|
||||
size = f.stat().st_size
|
||||
total_bytes += size
|
||||
total_files += 1
|
||||
node = root
|
||||
for part in rel.parts[:-1]:
|
||||
node = node.setdefault("children", {}).setdefault(part, {})
|
||||
leaf = node.setdefault("children", {}).setdefault(rel.parts[-1], {})
|
||||
leaf["bytes"] = size
|
||||
# roll sizes up the tree
|
||||
node = root
|
||||
node["bytes"] = node.get("bytes", 0) + size
|
||||
node["files"] = node.get("files", 0) + 1
|
||||
for part in rel.parts[:-1]:
|
||||
node = node["children"][part]
|
||||
node["bytes"] = node.get("bytes", 0) + size
|
||||
node["files"] = node.get("files", 0) + 1
|
||||
return {"tree": root, "total_bytes": total_bytes, "total_files": total_files}
|
||||
|
||||
|
||||
def run_query(sql: str) -> dict:
|
||||
reason = gate(sql)
|
||||
if reason:
|
||||
return {"error": f"rejected by read-only gate: {reason}"}
|
||||
con = connect()
|
||||
try:
|
||||
cur = con.sql(sql)
|
||||
columns = cur.columns
|
||||
rows = cur.fetchmany(ROW_LIMIT)
|
||||
return {
|
||||
"columns": columns,
|
||||
"rows": [[repr(v) if isinstance(v, bytes) else v for v in row] for row in rows],
|
||||
"truncated": len(rows) == ROW_LIMIT,
|
||||
}
|
||||
except duckdb.Error as exc:
|
||||
return {"error": str(exc)}
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def _send(self, code: int, body: bytes, ctype: str) -> None:
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", ctype)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _json(self, payload: dict, code: int = 200) -> None:
|
||||
self._send(code, json.dumps(payload, default=str).encode(), "application/json")
|
||||
|
||||
def do_GET(self) -> None: # noqa: N802 — http.server API
|
||||
if self.path in ("/", "/index.html"):
|
||||
self._send(200, (STATIC_DIR / "index.html").read_bytes(), "text/html; charset=utf-8")
|
||||
elif self.path == "/api/tree":
|
||||
self._json(tree())
|
||||
else:
|
||||
self._send(404, b"not found", "text/plain")
|
||||
|
||||
def do_POST(self) -> None: # noqa: N802 — http.server API
|
||||
if self.path != "/api/query":
|
||||
self._send(404, b"not found", "text/plain")
|
||||
return
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
try:
|
||||
sql = json.loads(self.rfile.read(length)).get("sql", "")
|
||||
except (json.JSONDecodeError, AttributeError):
|
||||
self._json({"error": "invalid request body"}, 400)
|
||||
return
|
||||
self._json(run_query(sql))
|
||||
|
||||
def log_message(self, *_args: object) -> None:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(f"data-plane explorer on :{PORT}, reading {DATA_DIR}")
|
||||
ThreadingHTTPServer(("", PORT), Handler).serve_forever()
|
||||
Reference in New Issue
Block a user