perf: bound lake scans and add platform Grafana dashboard
Explorer and exporter were scanning the full Parquet lake every 1–5s (~2000 files, 200MB+), driving ~2.2 CPU cores. Limit metrics to the last N flights, cache DuckDB views and the partition tree, slow live polls to 2s, and keep drones alive after seal to stop restart churn. Add node-exporter, scan-duration metrics, and a Swarm Platform Grafana dashboard for node CPU/memory and scan health.
This commit is contained in:
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import random
|
||||
import select
|
||||
import time
|
||||
@@ -108,6 +109,10 @@ def run(cfg: Config) -> None:
|
||||
writer.seal()
|
||||
print(f"[{cfg.drone_id}] done: {frames_sent} state frames sent, "
|
||||
f"{len(peers_seen)} peers seen {sorted(peers_seen)}; sealed to {root}")
|
||||
if os.environ.get("KEEP_ALIVE", "0") == "1":
|
||||
print(f"[{cfg.drone_id}] KEEP_ALIVE=1 — idle after seal (no pod restart churn)")
|
||||
while True:
|
||||
time.sleep(3600)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user