perf: bound lake scans and add platform Grafana dashboard
Explorer and exporter were scanning the full Parquet lake every 1–5s (~2000 files, 200MB+), driving ~2.2 CPU cores. Limit metrics to the last N flights, cache DuckDB views and the partition tree, slow live polls to 2s, and keep drones alive after seal to stop restart churn. Add node-exporter, scan-duration metrics, and a Swarm Platform Grafana dashboard for node CPU/memory and scan health.
This commit is contained in:
@@ -63,6 +63,14 @@ resource "kubernetes_stateful_set" "drone" {
|
||||
name = "DATA_DIR"
|
||||
value = "/data"
|
||||
}
|
||||
env {
|
||||
name = "KEEP_ALIVE"
|
||||
value = "1"
|
||||
}
|
||||
env {
|
||||
name = "IMU_HZ"
|
||||
value = "20"
|
||||
}
|
||||
|
||||
volume_mount {
|
||||
name = "lake"
|
||||
@@ -118,9 +126,21 @@ resource "kubernetes_deployment" "exporter" {
|
||||
name = "EXPORTER_PORT"
|
||||
value = "9105"
|
||||
}
|
||||
env {
|
||||
name = "SCAN_INTERVAL_S"
|
||||
value = "30"
|
||||
}
|
||||
env {
|
||||
name = "METRIC_FLIGHT_WINDOW"
|
||||
value = "5"
|
||||
}
|
||||
port {
|
||||
container_port = 9105
|
||||
}
|
||||
resources {
|
||||
requests = { cpu = "50m", memory = "64Mi" }
|
||||
limits = { cpu = "500m", memory = "256Mi" }
|
||||
}
|
||||
volume_mount {
|
||||
name = "lake"
|
||||
mount_path = "/data"
|
||||
@@ -182,9 +202,25 @@ resource "kubernetes_deployment" "explorer" {
|
||||
name = "EXPLORER_PORT"
|
||||
value = "8088"
|
||||
}
|
||||
env {
|
||||
name = "VIEW_REFRESH_S"
|
||||
value = "30"
|
||||
}
|
||||
env {
|
||||
name = "TREE_CACHE_S"
|
||||
value = "15"
|
||||
}
|
||||
env {
|
||||
name = "METRIC_FLIGHT_WINDOW"
|
||||
value = "5"
|
||||
}
|
||||
port {
|
||||
container_port = 8088
|
||||
}
|
||||
resources {
|
||||
requests = { cpu = "50m", memory = "64Mi" }
|
||||
limits = { cpu = "750m", memory = "256Mi" }
|
||||
}
|
||||
volume_mount {
|
||||
name = "lake"
|
||||
mount_path = "/data"
|
||||
@@ -226,11 +262,17 @@ resource "kubernetes_config_map" "prometheus" {
|
||||
data = {
|
||||
"prometheus.yml" = <<-EOT
|
||||
global:
|
||||
scrape_interval: 5s
|
||||
scrape_interval: 15s
|
||||
scrape_configs:
|
||||
- job_name: swarm
|
||||
static_configs:
|
||||
- targets: ["exporter:9105"]
|
||||
- job_name: explorer
|
||||
static_configs:
|
||||
- targets: ["explorer:8088"]
|
||||
- job_name: node
|
||||
static_configs:
|
||||
- targets: ["node-exporter:9100"]
|
||||
EOT
|
||||
}
|
||||
}
|
||||
@@ -327,7 +369,91 @@ resource "kubernetes_config_map" "grafana_dashboard" {
|
||||
namespace = kubernetes_namespace.swarm.metadata[0].name
|
||||
}
|
||||
data = {
|
||||
"swarm.json" = file("${path.module}/../../../simulator/monitoring/grafana/dashboards/swarm.json")
|
||||
"swarm.json" = file("${path.module}/../../../simulator/monitoring/grafana/dashboards/swarm.json")
|
||||
"swarm-platform.json" = file("${path.module}/../../../simulator/monitoring/grafana/dashboards/swarm-platform.json")
|
||||
}
|
||||
}
|
||||
|
||||
# Host metrics for the k3d node (CPU / memory on the platform dashboard)
|
||||
resource "kubernetes_deployment" "node_exporter" {
|
||||
metadata {
|
||||
name = "node-exporter"
|
||||
namespace = kubernetes_namespace.swarm.metadata[0].name
|
||||
labels = local.labels
|
||||
}
|
||||
|
||||
spec {
|
||||
replicas = 1
|
||||
selector {
|
||||
match_labels = { app = "node-exporter" }
|
||||
}
|
||||
template {
|
||||
metadata {
|
||||
labels = merge(local.labels, { app = "node-exporter" })
|
||||
}
|
||||
spec {
|
||||
host_network = true
|
||||
host_pid = true
|
||||
container {
|
||||
name = "node-exporter"
|
||||
image = "prom/node-exporter:v1.8.2"
|
||||
args = [
|
||||
"--path.procfs=/host/proc",
|
||||
"--path.sysfs=/host/sys",
|
||||
"--path.rootfs=/host/root",
|
||||
"--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)",
|
||||
]
|
||||
port {
|
||||
container_port = 9100
|
||||
}
|
||||
volume_mount {
|
||||
name = "proc"
|
||||
mount_path = "/host/proc"
|
||||
read_only = true
|
||||
}
|
||||
volume_mount {
|
||||
name = "sys"
|
||||
mount_path = "/host/sys"
|
||||
read_only = true
|
||||
}
|
||||
volume_mount {
|
||||
name = "root"
|
||||
mount_path = "/host/root"
|
||||
read_only = true
|
||||
}
|
||||
resources {
|
||||
requests = { cpu = "20m", memory = "32Mi" }
|
||||
limits = { cpu = "200m", memory = "64Mi" }
|
||||
}
|
||||
}
|
||||
volume {
|
||||
name = "proc"
|
||||
host_path { path = "/proc" }
|
||||
}
|
||||
volume {
|
||||
name = "sys"
|
||||
host_path { path = "/sys" }
|
||||
}
|
||||
volume {
|
||||
name = "root"
|
||||
host_path { path = "/" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "kubernetes_service" "node_exporter" {
|
||||
metadata {
|
||||
name = "node-exporter"
|
||||
namespace = kubernetes_namespace.swarm.metadata[0].name
|
||||
}
|
||||
spec {
|
||||
selector = { app = "node-exporter" }
|
||||
port {
|
||||
port = 9100
|
||||
target_port = 9100
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user