diff --git a/.gitignore b/.gitignore index 28d98fe..a989123 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,9 @@ data/ +simulator/data/ +var/t1/** +var/t3/** +!var/t1/.gitkeep +!var/t3/.gitkeep __pycache__/ *.pyc .venv/ diff --git a/infra/ansible/README.md b/infra/ansible/README.md new file mode 100644 index 0000000..5e00f2b --- /dev/null +++ b/infra/ansible/README.md @@ -0,0 +1,39 @@ +# Ansible — host preparation + +Ansible owns everything that happens **on a host before workloads run**. +Terraform ([`../terraform/`](../terraform/)) owns the workloads themselves. +The boundary is deliberate and documented in +[06 — Environments](../../docs/06-environments.md#iac-boundaries). + +| Playbook | Target | What it does | +| --- | --- | --- | +| `drone-provision.yml` | Drone (on the bench, before a mission) | Identity keys, peer trust with forced commands, WireGuard, Compose bundle from the fleet release manifest | +| `sim-cluster.yml` | Local workstation / CI host | k3d cluster that miniatures the ground segment; shared data volume; simulator image import | + +## Provision a drone + +```bash +ansible-playbook -i inventory.example.yml drone-provision.yml +``` + +The playbook is idempotent and runs only over the bench network — drones +are never provisioned in flight (see [05 — Network & security](../../docs/05-network-security.md)): + +1. **Identity**: generate the drone's ed25519 keypair if absent. +2. **Peer trust**: install every fleet member's public key into + `authorized_keys`, each pinned to the read-only SQL forced command — + the only thing a peer can execute. +3. **WireGuard**: render `wg0.conf` with the drone's address and peers. +4. **Data plane**: lay down the Compose bundle referenced by the fleet + release manifest and enable it as a systemd unit. + +## Create the simulation cluster + +```bash +ansible-playbook sim-cluster.yml # creates k3d cluster 'swarm-sim' +cd ../terraform/sim-env && terraform init && terraform apply +``` + +The cluster mounts a shared host directory as `/data` on the (single) +node, so drone pods, the exporter, and the explorer see one Parquet lake — +the same contract as the on-board NVMe layout. diff --git a/infra/ansible/drone-provision.yml b/infra/ansible/drone-provision.yml new file mode 100644 index 0000000..fc3400e --- /dev/null +++ b/infra/ansible/drone-provision.yml @@ -0,0 +1,101 @@ +# Provision a drone on the bench: identity, peer trust, WireGuard, data plane. +# Idempotent; never runs over the mission radio (bench network only). +--- +- name: Provision drone data plane + hosts: drones + become: true + vars: + swarm_user: swarm + swarm_home: /opt/swarm + forced_command: "{{ swarm_home }}/bin/peer-query.sh" + + tasks: + - name: Create swarm service user + ansible.builtin.user: + name: "{{ swarm_user }}" + home: "{{ swarm_home }}" + shell: /usr/sbin/nologin + system: true + + # --- 1. Identity ----------------------------------------------------- + - name: Generate drone ed25519 identity key + community.crypto.openssh_keypair: + path: "{{ swarm_home }}/.ssh/id_ed25519" + type: ed25519 + comment: "{{ inventory_hostname }}" + owner: "{{ swarm_user }}" + mode: "0600" + register: identity + + - name: Collect fleet public keys + ansible.builtin.slurp: + src: "{{ swarm_home }}/.ssh/id_ed25519.pub" + register: pubkey + + # --- 2. Peer trust: forced command only ------------------------------ + # Every peer may connect, but the only thing it can execute is the + # read-only SQL wrapper (docs/04). No shell, no forwarding, no pty. + - name: Authorize fleet peers with read-only forced command + ansible.posix.authorized_key: + user: "{{ swarm_user }}" + key: "{{ hostvars[item].pubkey.content | b64decode }}" + key_options: >- + command="{{ forced_command }}",no-port-forwarding,no-agent-forwarding,no-X11-forwarding,no-pty + loop: "{{ groups['drones'] | difference([inventory_hostname]) }}" + when: hostvars[item].pubkey is defined + + - name: Install peer query wrapper + ansible.builtin.copy: + dest: "{{ forced_command }}" + mode: "0755" + content: | + #!/bin/sh + # Forced command: read-only DuckDB SQL over the sealed lake. + # SSH_ORIGINAL_COMMAND carries the statement; the gate rejects + # anything that is not a single read statement. + exec {{ swarm_home }}/bin/sql-gate --read-only --data /data "$SSH_ORIGINAL_COMMAND" + + # --- 3. WireGuard swarm plane ---------------------------------------- + - name: Render WireGuard configuration + ansible.builtin.template: + src: templates/wg0.conf.j2 + dest: /etc/wireguard/wg0.conf + mode: "0600" + notify: Restart wireguard + + - name: Enable WireGuard interface + ansible.builtin.systemd: + name: wg-quick@wg0 + enabled: true + state: started + + # --- 4. Data plane from the fleet release manifest -------------------- + - name: Read fleet release manifest + ansible.builtin.include_vars: + file: "{{ fleet_manifest }}" + name: manifest + + - name: Lay down Compose bundle for release {{ manifest.release }} + ansible.builtin.unarchive: + src: "{{ playbook_dir }}/../../.tmp/{{ manifest.artifacts[0].name }}" + dest: "{{ swarm_home }}/release" + owner: "{{ swarm_user }}" + + - name: Enable data plane unit + ansible.builtin.template: + src: templates/swarm-data-plane.service.j2 + dest: /etc/systemd/system/swarm-data-plane.service + mode: "0644" + notify: Restart data plane + + handlers: + - name: Restart wireguard + ansible.builtin.systemd: + name: wg-quick@wg0 + state: restarted + + - name: Restart data plane + ansible.builtin.systemd: + name: swarm-data-plane + state: restarted + daemon_reload: true diff --git a/infra/ansible/inventory.example.yml b/infra/ansible/inventory.example.yml new file mode 100644 index 0000000..2109067 --- /dev/null +++ b/infra/ansible/inventory.example.yml @@ -0,0 +1,19 @@ +# Example fleet inventory. Real inventories are generated from the fleet +# release manifest and live outside the repository. +all: + children: + drones: + hosts: + dr-01: + ansible_host: 10.44.0.11 + wg_address: 10.44.0.11/24 + dr-02: + ansible_host: 10.44.0.12 + wg_address: 10.44.0.12/24 + dr-03: + ansible_host: 10.44.0.13 + wg_address: 10.44.0.13/24 + vars: + ansible_user: ops + wg_port: 51820 + fleet_manifest: ../../.tmp/manifest.yaml diff --git a/infra/ansible/sim-cluster.yml b/infra/ansible/sim-cluster.yml new file mode 100644 index 0000000..8f7df5b --- /dev/null +++ b/infra/ansible/sim-cluster.yml @@ -0,0 +1,125 @@ +# Create the local k3d simulation cluster — an executable miniature of the +# ground segment (k3s) from docs/06. Terraform then owns the workloads. +--- +- name: Simulation cluster (k3d) + hosts: localhost + connection: local + gather_facts: false + vars: + cluster_name: swarm-sim + repo_root: "{{ playbook_dir }}/../.." + data_dir: "{{ lookup('env', 'SWARM_SIM_DATA') | default(repo_root + '/var/t1', true) }}" + warehouse_dir: "{{ lookup('env', 'SWARM_WAREHOUSE_DATA') | default(repo_root + '/var/t3', true) }}" + api_port: 6560 + # Host ports for services exposed by the Terraform modules (NodePorts) + port_explorer: 30088 + port_grafana: 30300 + port_prometheus: 30990 + port_warehouse_explorer: 30089 + port_minio_console: 30901 + simulator_image: swarm-house/simulator:dev + + tasks: + - name: Check k3d is installed + ansible.builtin.command: k3d version + changed_when: false + + - name: Create shared data directory (the pods' /data lake) + ansible.builtin.file: + path: "{{ data_dir }}" + state: directory + mode: "0777" + + - name: Create warehouse directory (T3 host path inside the k3d node) + ansible.builtin.file: + path: "{{ warehouse_dir }}" + state: directory + mode: "0777" + + - name: List existing clusters + ansible.builtin.command: k3d cluster list -o json + register: clusters + changed_when: false + + - name: Create cluster {{ cluster_name }} + ansible.builtin.command: >- + k3d cluster create {{ cluster_name }} + --api-port {{ api_port }} + --servers 1 --agents 0 + --volume {{ data_dir }}:/data@server:0 + --volume {{ warehouse_dir }}:/warehouse@server:0 + --port {{ port_explorer }}:{{ port_explorer }}@server:0 + --port {{ port_grafana }}:{{ port_grafana }}@server:0 + --port {{ port_prometheus }}:{{ port_prometheus }}@server:0 + --port {{ port_warehouse_explorer }}:{{ port_warehouse_explorer }}@server:0 + --port {{ port_minio_console }}:{{ port_minio_console }}@server:0 + --k3s-arg "--disable=traefik@server:0" + --wait + when: clusters.stdout | from_json | selectattr('name', 'equalto', cluster_name) | list | length == 0 + + - name: Build simulator image + ansible.builtin.command: + cmd: docker build -t {{ simulator_image }} . + chdir: "{{ playbook_dir }}/../../simulator" + register: build + changed_when: "'Using cache' not in build.stderr" + + - name: Import simulator image into the cluster + ansible.builtin.command: k3d image import {{ simulator_image }} -c {{ cluster_name }} + changed_when: true + + - name: Merge k3d kubeconfig into the default context + ansible.builtin.command: k3d kubeconfig merge {{ cluster_name }} --kubeconfig-merge-default + changed_when: false + + - name: Check whether Flux CLI is available + ansible.builtin.command: flux version --client + register: flux_cli + changed_when: false + failed_when: false + + - name: Install Flux CLI (local user bin) + when: flux_cli.rc != 0 + block: + - name: Download Flux release archive + ansible.builtin.get_url: + url: https://github.com/fluxcd/flux2/releases/download/v2.4.0/flux_2.4.0_linux_amd64.tar.gz + dest: /tmp/flux.tar.gz + mode: "0644" + - name: Extract flux binary + ansible.builtin.unarchive: + src: /tmp/flux.tar.gz + dest: "{{ lookup('env', 'HOME') }}/.local/bin" + remote_src: true + include: + - flux + extra_opts: + - --no-same-owner + ignore_errors: true + - name: Ensure flux is executable + ansible.builtin.file: + path: "{{ lookup('env', 'HOME') }}/.local/bin/flux" + mode: "0755" + state: file + + - name: Install Flux controllers into the cluster + ansible.builtin.command: flux install --namespace=flux-system --components=source-controller,kustomize-controller + environment: + PATH: "{{ lookup('env', 'HOME') }}/.local/bin:{{ lookup('env', 'PATH') }}" + KUBECONFIG: "{{ lookup('env', 'HOME') }}/.kube/config" + register: flux_install + changed_when: "'installed' in (flux_install.stdout | default(''))" + + - name: Apply Flux GitOps CRs (GitRepository + Kustomization) + ansible.builtin.command: kubectl apply -k {{ repo_root }}/infra/gitops/flux + changed_when: true + + - name: Seed ground GitOps resources locally (works before first git push) + ansible.builtin.command: kubectl apply -k {{ repo_root }}/infra/gitops/ground + changed_when: true + + - name: Show kubeconfig hint + ansible.builtin.debug: + msg: >- + Cluster ready. Workloads: cd ../terraform/sim-env && terraform init && terraform apply. + kubeconfig: k3d kubeconfig get {{ cluster_name }} diff --git a/infra/ansible/templates/swarm-data-plane.service.j2 b/infra/ansible/templates/swarm-data-plane.service.j2 new file mode 100644 index 0000000..ee9d71f --- /dev/null +++ b/infra/ansible/templates/swarm-data-plane.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=Swarm data plane (Compose bundle, release {{ manifest.release }}) +After=docker.service wg-quick@wg0.service +Requires=docker.service + +[Service] +Type=oneshot +RemainAfterExit=yes +WorkingDirectory={{ swarm_home }}/release +ExecStart=/usr/bin/docker compose up -d +ExecStop=/usr/bin/docker compose down +TimeoutStartSec=300 + +[Install] +WantedBy=multi-user.target diff --git a/infra/ansible/templates/wg0.conf.j2 b/infra/ansible/templates/wg0.conf.j2 new file mode 100644 index 0000000..b3c8334 --- /dev/null +++ b/infra/ansible/templates/wg0.conf.j2 @@ -0,0 +1,12 @@ +# WireGuard swarm plane — rendered by drone-provision.yml +[Interface] +Address = {{ wg_address }} +ListenPort = {{ wg_port }} +PrivateKey = __REPLACED_FROM_HSM_AT_SEALING__ + +{% for peer in groups['drones'] | difference([inventory_hostname]) %} +[Peer] +# {{ peer }} +PublicKey = __PEER_{{ peer | upper | replace('-', '_') }}_PUBKEY__ +AllowedIPs = {{ hostvars[peer].wg_address | ansible.utils.ipaddr('address') }}/32 +{% endfor %} diff --git a/infra/gitops/README.md b/infra/gitops/README.md new file mode 100644 index 0000000..dbabcf6 --- /dev/null +++ b/infra/gitops/README.md @@ -0,0 +1,52 @@ +# GitOps — ground segment (Flux) + +Flux reconciles **long-lived ground configuration** from this repository. It does +not run on drones and does not replace the fleet release manifest — those are +different lifecycles by design ([11 — CI/CD](../../docs/11-cicd-delivery.md)). + +## Boundary: Terraform vs Flux vs fleet manifest + +| Layer | Tool | Owns | +| --- | --- | --- | +| Cluster + first boot | **Terraform** ([`../terraform/`](../terraform/)) | Namespaces, PVCs, Deployments, CronJobs, NodePorts — the shape of the simulation | +| Ongoing ground config | **Flux** (this directory) | Policy labels, dashboard bundles, offload knobs — things that change without reprovisioning PVCs | +| Drones | **Fleet release manifest** | Compose bundle digests, model weights, peer registry — atomic, dock-only delivery | + +Drones are **outside GitOps**: there is no reconciler in flight. A dock applies +the pinned manifest once; mid-mission drift is impossible because the update +endpoint does not exist in the radio profile. + +## Layout + +``` +infra/gitops/ + flux/ Flux GitRepository + Kustomization CRs (install into flux-system) + ground/ Kustomize overlay reconciled into the ground namespace +``` + +## Bootstrap on the k3d simulation cluster + +`infra/ansible/sim-cluster.yml` installs Flux controllers and applies the CRs +below. After the first `git push`, Flux polls `origin` and reconciles +`infra/gitops/ground/` into the `ground` namespace. + +```bash +# Manual bootstrap (if you skipped Ansible): +flux install --namespace=flux-system +kubectl apply -k infra/gitops/flux +``` + +To reconcile immediately without waiting for the poll interval: + +```bash +flux reconcile source git swarm-house -n flux-system +flux reconcile kustomization ground-segment -n flux-system +``` + +## What Flux manages here (example) + +The [`ground/`](ground/) overlay currently carries a **GitOps-managed ConfigMap** +that tags the warehouse segment with fleet policy metadata. In production this +pattern extends to Grafana dashboard bundles, Prometheus rule files, and +offload-schedule ConfigMaps — all versioned in git, all auditable, none of +them requiring a `terraform apply` to tweak a label. diff --git a/infra/gitops/flux/gitrepository.yaml b/infra/gitops/flux/gitrepository.yaml new file mode 100644 index 0000000..419f89c --- /dev/null +++ b/infra/gitops/flux/gitrepository.yaml @@ -0,0 +1,10 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: swarm-house + namespace: flux-system +spec: + interval: 1m + url: https://git.produktor.io/eSlider/swarm-house.git + ref: + branch: main diff --git a/infra/gitops/flux/kustomization-ground.yaml b/infra/gitops/flux/kustomization-ground.yaml new file mode 100644 index 0000000..51c9460 --- /dev/null +++ b/infra/gitops/flux/kustomization-ground.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: ground-segment + namespace: flux-system +spec: + interval: 2m + sourceRef: + kind: GitRepository + name: swarm-house + path: ./infra/gitops/ground + prune: true + wait: true + timeout: 3m diff --git a/infra/gitops/flux/kustomization.yaml b/infra/gitops/flux/kustomization.yaml new file mode 100644 index 0000000..37de77f --- /dev/null +++ b/infra/gitops/flux/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - gitrepository.yaml + - kustomization-ground.yaml diff --git a/infra/gitops/ground/configmap-fleet-policy.yaml b/infra/gitops/ground/configmap-fleet-policy.yaml new file mode 100644 index 0000000..0774188 --- /dev/null +++ b/infra/gitops/ground/configmap-fleet-policy.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ground-fleet-policy + namespace: ground + labels: + app.kubernetes.io/part-of: swarm-ground + app.kubernetes.io/managed-by: flux +data: + offload_schedule: "*/2 * * * *" + retention_policy: "keep-all-flights" + explorer_mode: "read-only-sql" diff --git a/infra/gitops/ground/kustomization.yaml b/infra/gitops/ground/kustomization.yaml new file mode 100644 index 0000000..d14ef74 --- /dev/null +++ b/infra/gitops/ground/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: ground +resources: + - namespace.yaml + - configmap-fleet-policy.yaml diff --git a/infra/gitops/ground/namespace.yaml b/infra/gitops/ground/namespace.yaml new file mode 100644 index 0000000..cd8f082 --- /dev/null +++ b/infra/gitops/ground/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ground + labels: + app.kubernetes.io/part-of: swarm-ground diff --git a/infra/terraform/README.md b/infra/terraform/README.md new file mode 100644 index 0000000..d508982 --- /dev/null +++ b/infra/terraform/README.md @@ -0,0 +1,34 @@ +## Recovering from partial state + +If resources were created outside Terraform (or state was lost), import them +before `terraform apply`: + +```bash +# sim-env (namespace: swarm) +terraform import kubernetes_namespace.swarm swarm +terraform import kubernetes_stateful_set.drone swarm/drone +terraform import kubernetes_deployment.exporter swarm/exporter +terraform import kubernetes_deployment.explorer swarm/explorer +terraform import kubernetes_deployment.prometheus swarm/prometheus +terraform import kubernetes_deployment.grafana swarm/grafana +terraform import kubernetes_service.exporter swarm/exporter +terraform import kubernetes_service.explorer swarm/explorer +terraform import kubernetes_service.prometheus swarm/prometheus +terraform import kubernetes_service.grafana swarm/grafana +terraform import kubernetes_config_map.prometheus swarm/prometheus-config +terraform import kubernetes_config_map.grafana_provisioning swarm/grafana-provisioning +terraform import kubernetes_config_map.grafana_dashboard swarm/grafana-dashboard + +# ground (namespace: ground) +terraform import kubernetes_namespace.ground ground +terraform import kubernetes_persistent_volume_claim.minio ground/minio-data +terraform import kubernetes_secret.minio ground/minio-credentials +terraform import kubernetes_deployment.minio ground/minio +terraform import kubernetes_deployment.warehouse_explorer ground/warehouse-explorer +terraform import kubernetes_service.minio ground/minio +terraform import kubernetes_service.warehouse_explorer ground/warehouse-explorer +terraform import kubernetes_cron_job_v1.offload ground/offload +``` + +Then `terraform apply` reconciles drift (for example pinning `EXPLORER_PORT` +and `EXPORTER_PORT` so Kubernetes service env injection does not break startup). diff --git a/infra/terraform/ground/.terraform.lock.hcl b/infra/terraform/ground/.terraform.lock.hcl new file mode 100644 index 0000000..a7e5860 --- /dev/null +++ b/infra/terraform/ground/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.38.0" + constraints = "~> 2.33" + hashes = [ + "h1:5CkveFo5ynsLdzKk+Kv+r7+U9rMrNjfZPT3a0N/fhgE=", + "zh:0af928d776eb269b192dc0ea0f8a3f0f5ec117224cd644bdacdc682300f84ba0", + "zh:1be998e67206f7cfc4ffe77c01a09ac91ce725de0abaec9030b22c0a832af44f", + "zh:326803fe5946023687d603f6f1bab24de7af3d426b01d20e51d4e6fbe4e7ec1b", + "zh:4a99ec8d91193af961de1abb1f824be73df07489301d62e6141a656b3ebfff12", + "zh:5136e51765d6a0b9e4dbcc3b38821e9736bd2136cf15e9aac11668f22db117d2", + "zh:63fab47349852d7802fb032e4f2b6a101ee1ce34b62557a9ad0f0f0f5b6ecfdc", + "zh:924fb0257e2d03e03e2bfe9c7b99aa73c195b1f19412ca09960001bee3c50d15", + "zh:b63a0be5e233f8f6727c56bed3b61eb9456ca7a8bb29539fba0837f1badf1396", + "zh:d39861aa21077f1bc899bc53e7233262e530ba8a3a2d737449b100daeb303e4d", + "zh:de0805e10ebe4c83ce3b728a67f6b0f9d18be32b25146aa89116634df5145ad4", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:faf23e45f0090eef8ba28a8aac7ec5d4fdf11a36c40a8d286304567d71c1e7db", + ] +} diff --git a/infra/terraform/ground/.terraform/providers/registry.terraform.io/hashicorp/kubernetes/2.38.0/linux_amd64/LICENSE.txt b/infra/terraform/ground/.terraform/providers/registry.terraform.io/hashicorp/kubernetes/2.38.0/linux_amd64/LICENSE.txt new file mode 100644 index 0000000..b9ac071 --- /dev/null +++ b/infra/terraform/ground/.terraform/providers/registry.terraform.io/hashicorp/kubernetes/2.38.0/linux_amd64/LICENSE.txt @@ -0,0 +1,375 @@ +Copyright (c) 2017 HashiCorp, Inc. + +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/infra/terraform/ground/.terraform/providers/registry.terraform.io/hashicorp/kubernetes/2.38.0/linux_amd64/terraform-provider-kubernetes_v2.38.0_x5 b/infra/terraform/ground/.terraform/providers/registry.terraform.io/hashicorp/kubernetes/2.38.0/linux_amd64/terraform-provider-kubernetes_v2.38.0_x5 new file mode 100755 index 0000000..a02e3e5 Binary files /dev/null and b/infra/terraform/ground/.terraform/providers/registry.terraform.io/hashicorp/kubernetes/2.38.0/linux_amd64/terraform-provider-kubernetes_v2.38.0_x5 differ diff --git a/infra/terraform/ground/README.md b/infra/terraform/ground/README.md new file mode 100644 index 0000000..e6fc8c6 --- /dev/null +++ b/infra/terraform/ground/README.md @@ -0,0 +1,38 @@ +# Terraform — ground warehouse (T3) + +The stationary half of the system: the historical warehouse that receives +sealed partitions after every flight ([03 — Storage tiers](../../../docs/03-storage-design.md)). +It is the longest-lived, most conventional infrastructure in the design — +and therefore the most natural Terraform territory. + +Runs in the same k3d cluster as [`../sim-env`](../sim-env/) but in its own +namespace with its own state: the fleet is disposable, the warehouse is not, +and the two lifecycles must never share a `terraform destroy`. + +| Resource | Purpose | +| --- | --- | +| Deployment `minio` + PVC + NodePort 30901 | Object-store facade of the warehouse for downstream consumers (training pipelines, replay) | +| CronJob `offload` | Post-flight offload: copies Hive partitions verbatim from the fleet lake (T1) into the warehouse (T3), then mirrors into MinIO | +| Deployment `warehouse-explorer` + NodePort 30089 | Second explorer instance over the warehouse path — historical read-only SQL across **all** flights | + +## Usage + +```bash +terraform init +terraform apply +terraform output +``` + +After a couple of offload runs, the warehouse explorer shows the same +partition tree as the live lake but accumulated across flights — the T1 +layout **is** the T3 layout, which is the whole point: no transform step, +no schema drift, DuckDB queries work identically on both ends. + +## Simulation vs production + +- The CronJob compresses "drone lands, docks, offloads, prunes" into a + periodic rsync-style copy. Production offload is event-driven per docking + and verifies checksums from the flight's partition manifest before pruning + the on-board lake. +- MinIO credentials here are throwaway defaults; production credentials are + sealed per site and never in state or VCS. diff --git a/infra/terraform/ground/main.tf b/infra/terraform/ground/main.tf new file mode 100644 index 0000000..cc5e4df --- /dev/null +++ b/infra/terraform/ground/main.tf @@ -0,0 +1,273 @@ +# Ground warehouse segment (T3, docs/03 + docs/06) — the stationary, +# long-lifecycle half of the system and therefore the most natural Terraform +# territory. Runs in the same k3d cluster as the fleet, in its own namespace +# and with its own state. + +locals { + labels = { "app.kubernetes.io/part-of" = "swarm-ground" } +} + +resource "kubernetes_namespace" "ground" { + metadata { + name = var.namespace + labels = local.labels + } +} + +# --- MinIO object store ------------------------------------------------------- + +resource "kubernetes_secret" "minio" { + metadata { + name = "minio-credentials" + namespace = kubernetes_namespace.ground.metadata[0].name + } + data = { + MINIO_ROOT_USER = var.minio_root_user + MINIO_ROOT_PASSWORD = var.minio_root_password + } +} + +resource "kubernetes_persistent_volume_claim" "minio" { + metadata { + name = "minio-data" + namespace = kubernetes_namespace.ground.metadata[0].name + } + spec { + access_modes = ["ReadWriteOnce"] + resources { + requests = { storage = var.minio_storage } + } + } +} + +resource "kubernetes_deployment" "minio" { + metadata { + name = "minio" + namespace = kubernetes_namespace.ground.metadata[0].name + labels = local.labels + } + + spec { + replicas = 1 + strategy { + type = "Recreate" # RWO volume + } + selector { + match_labels = { app = "minio" } + } + template { + metadata { + labels = merge(local.labels, { app = "minio" }) + } + spec { + container { + name = "minio" + image = "minio/minio:RELEASE.2024-06-13T22-53-53Z" + args = ["server", "/data", "--console-address", ":9001"] + + env_from { + secret_ref { + name = kubernetes_secret.minio.metadata[0].name + } + } + # Air-gap hygiene: no update checks + env { + name = "MINIO_UPDATE" + value = "off" + } + + port { + container_port = 9000 + } + port { + container_port = 9001 + } + + volume_mount { + name = "store" + mount_path = "/data" + } + } + volume { + name = "store" + persistent_volume_claim { + claim_name = kubernetes_persistent_volume_claim.minio.metadata[0].name + } + } + } + } + } +} + +resource "kubernetes_service" "minio" { + metadata { + name = "minio" + namespace = kubernetes_namespace.ground.metadata[0].name + } + spec { + type = "NodePort" + selector = { app = "minio" } + port { + name = "api" + port = 9000 + target_port = 9000 + } + port { + name = "console" + port = 9001 + target_port = 9001 + node_port = var.node_ports.minio_console + } + } +} + +# --- Post-flight offload: T1 lake -> T3 warehouse ------------------------------ +# In production this runs on the docking station after a drone lands: only +# sealed partitions move, checksums are verified, then the on-board lake is +# pruned. The simulation compresses that into a periodic rsync-style copy +# plus an object-store mirror. + +resource "kubernetes_cron_job_v1" "offload" { + metadata { + name = "offload" + namespace = kubernetes_namespace.ground.metadata[0].name + labels = local.labels + } + + spec { + schedule = var.offload_schedule + concurrency_policy = "Forbid" + successful_jobs_history_limit = 3 + failed_jobs_history_limit = 3 + + job_template { + metadata { + labels = local.labels + } + spec { + backoff_limit = 1 + template { + metadata { + labels = local.labels + } + spec { + restart_policy = "Never" + + # Step 1: copy Hive partitions verbatim (T1 layout == T3 layout) + init_container { + name = "copy" + image = "busybox:1.36" + command = ["sh", "-c", "cp -ru /lake/flight_id=* /warehouse/ 2>/dev/null; ls /warehouse | wc -l"] + volume_mount { + name = "lake" + mount_path = "/lake" + read_only = true + } + volume_mount { + name = "warehouse" + mount_path = "/warehouse" + } + } + + # Step 2: mirror the warehouse into the object store for + # downstream consumers (training pipelines, replay tooling) + container { + name = "mirror" + image = "minio/mc:RELEASE.2024-06-12T14-34-03Z" + command = ["sh", "-c", "mc alias set store http://minio:9000 \"$MINIO_ROOT_USER\" \"$MINIO_ROOT_PASSWORD\" && mc mb -p store/warehouse && mc mirror --overwrite /warehouse store/warehouse"] + env_from { + secret_ref { + name = kubernetes_secret.minio.metadata[0].name + } + } + volume_mount { + name = "warehouse" + mount_path = "/warehouse" + read_only = true + } + } + + volume { + name = "lake" + host_path { + path = var.lake_host_path + } + } + volume { + name = "warehouse" + host_path { + path = var.warehouse_host_path + type = "DirectoryOrCreate" + } + } + } + } + } + } + } +} + +# --- Warehouse explorer: historical read-only SQL over all flights ------------- + +resource "kubernetes_deployment" "warehouse_explorer" { + metadata { + name = "warehouse-explorer" + namespace = kubernetes_namespace.ground.metadata[0].name + labels = local.labels + } + + spec { + replicas = 1 + selector { + match_labels = { app = "warehouse-explorer" } + } + template { + metadata { + labels = merge(local.labels, { app = "warehouse-explorer" }) + } + spec { + container { + name = "explorer" + image = var.simulator_image + image_pull_policy = "Never" + command = ["python", "explorer/server.py"] + env { + name = "DATA_DIR" + value = "/warehouse" + } + port { + container_port = 8088 + } + volume_mount { + name = "warehouse" + mount_path = "/warehouse" + read_only = true + } + } + volume { + name = "warehouse" + host_path { + path = var.warehouse_host_path + type = "DirectoryOrCreate" + } + } + } + } + } +} + +resource "kubernetes_service" "warehouse_explorer" { + metadata { + name = "warehouse-explorer" + namespace = kubernetes_namespace.ground.metadata[0].name + } + spec { + type = "NodePort" + selector = { app = "warehouse-explorer" } + port { + port = 8088 + target_port = 8088 + node_port = var.node_ports.warehouse_explorer + } + } +} diff --git a/infra/terraform/ground/outputs.tf b/infra/terraform/ground/outputs.tf new file mode 100644 index 0000000..f970785 --- /dev/null +++ b/infra/terraform/ground/outputs.tf @@ -0,0 +1,14 @@ +output "warehouse_explorer_url" { + description = "Historical read-only SQL over all offloaded flights" + value = "http://localhost:${var.node_ports.warehouse_explorer}" +} + +output "minio_console_url" { + description = "MinIO console (object-store view of the warehouse)" + value = "http://localhost:${var.node_ports.minio_console}" +} + +output "offload_schedule" { + description = "Cron schedule of the T1 -> T3 offload job" + value = var.offload_schedule +} diff --git a/infra/terraform/ground/terraform.tfstate b/infra/terraform/ground/terraform.tfstate new file mode 100644 index 0000000..8f134f8 --- /dev/null +++ b/infra/terraform/ground/terraform.tfstate @@ -0,0 +1,1132 @@ +{ + "version": 4, + "terraform_version": "1.15.6", + "serial": 11, + "lineage": "df6d346f-1b17-4b8f-6165-3536b2b272fe", + "outputs": { + "minio_console_url": { + "value": "http://localhost:30901", + "type": "string" + }, + "offload_schedule": { + "value": "*/2 * * * *", + "type": "string" + }, + "warehouse_explorer_url": { + "value": "http://localhost:30089", + "type": "string" + } + }, + "resources": [ + { + "mode": "managed", + "type": "kubernetes_cron_job_v1", + "name": "offload", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ground/offload", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 1, + "labels": { + "app.kubernetes.io/part-of": "swarm-ground" + }, + "name": "offload", + "namespace": "ground", + "resource_version": "6337", + "uid": "a1b55fa7-eb9f-46d6-bddf-82576be4c872" + } + ], + "spec": [ + { + "concurrency_policy": "Forbid", + "failed_jobs_history_limit": 3, + "job_template": [ + { + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": { + "app.kubernetes.io/part-of": "swarm-ground" + }, + "name": "", + "namespace": "", + "resource_version": "", + "uid": "" + } + ], + "spec": [ + { + "active_deadline_seconds": 0, + "backoff_limit": 1, + "backoff_limit_per_index": 0, + "completion_mode": "", + "completions": 1, + "manual_selector": false, + "max_failed_indexes": 0, + "parallelism": 1, + "pod_failure_policy": [], + "selector": [], + "template": [ + { + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": { + "app.kubernetes.io/part-of": "swarm-ground" + }, + "name": "", + "resource_version": "", + "uid": "" + } + ], + "spec": [ + { + "active_deadline_seconds": 0, + "affinity": [], + "automount_service_account_token": true, + "container": [ + { + "args": [], + "command": [ + "sh", + "-c", + "mc alias set store http://minio:9000 \"$MINIO_ROOT_USER\" \"$MINIO_ROOT_PASSWORD\" \u0026\u0026 mc mb -p store/warehouse \u0026\u0026 mc mirror --overwrite /warehouse store/warehouse" + ], + "env": [], + "env_from": [ + { + "config_map_ref": [], + "prefix": "", + "secret_ref": [ + { + "name": "minio-credentials", + "optional": false + } + ] + } + ], + "image": "minio/mc:RELEASE.2024-06-12T14-34-03Z", + "image_pull_policy": "IfNotPresent", + "lifecycle": [], + "liveness_probe": [], + "name": "mirror", + "port": [], + "readiness_probe": [], + "resources": [ + { + "limits": {}, + "requests": {} + } + ], + "security_context": [], + "startup_probe": [], + "stdin": false, + "stdin_once": false, + "termination_message_path": "/dev/termination-log", + "termination_message_policy": "File", + "tty": false, + "volume_device": [], + "volume_mount": [ + { + "mount_path": "/warehouse", + "mount_propagation": "None", + "name": "warehouse", + "read_only": true, + "sub_path": "", + "sub_path_expr": "" + } + ], + "working_dir": "" + } + ], + "dns_config": [], + "dns_policy": "ClusterFirst", + "enable_service_links": true, + "host_aliases": [], + "host_ipc": false, + "host_network": false, + "host_pid": false, + "hostname": "", + "image_pull_secrets": [], + "init_container": [ + { + "args": [], + "command": [ + "sh", + "-c", + "cp -ru /lake/flight_id=* /warehouse/ 2\u003e/dev/null; ls /warehouse | wc -l" + ], + "env": [], + "env_from": [], + "image": "busybox:1.36", + "image_pull_policy": "IfNotPresent", + "lifecycle": [], + "liveness_probe": [], + "name": "copy", + "port": [], + "readiness_probe": [], + "resources": [ + { + "limits": {}, + "requests": {} + } + ], + "security_context": [], + "startup_probe": [], + "stdin": false, + "stdin_once": false, + "termination_message_path": "/dev/termination-log", + "termination_message_policy": "File", + "tty": false, + "volume_device": [], + "volume_mount": [ + { + "mount_path": "/lake", + "mount_propagation": "None", + "name": "lake", + "read_only": true, + "sub_path": "", + "sub_path_expr": "" + }, + { + "mount_path": "/warehouse", + "mount_propagation": "None", + "name": "warehouse", + "read_only": false, + "sub_path": "", + "sub_path_expr": "" + } + ], + "working_dir": "" + } + ], + "node_name": "", + "node_selector": {}, + "os": [], + "priority_class_name": "", + "readiness_gate": [], + "restart_policy": "Never", + "runtime_class_name": "", + "scheduler_name": "default-scheduler", + "security_context": [], + "service_account_name": "", + "share_process_namespace": false, + "subdomain": "", + "termination_grace_period_seconds": 30, + "toleration": [], + "topology_spread_constraint": [], + "volume": [ + { + "aws_elastic_block_store": [], + "azure_disk": [], + "azure_file": [], + "ceph_fs": [], + "cinder": [], + "config_map": [], + "csi": [], + "downward_api": [], + "empty_dir": [], + "ephemeral": [], + "fc": [], + "flex_volume": [], + "flocker": [], + "gce_persistent_disk": [], + "git_repo": [], + "glusterfs": [], + "host_path": [ + { + "path": "/data", + "type": "" + } + ], + "iscsi": [], + "local": [], + "name": "lake", + "nfs": [], + "persistent_volume_claim": [], + "photon_persistent_disk": [], + "projected": [], + "quobyte": [], + "rbd": [], + "secret": [], + "vsphere_volume": [] + }, + { + "aws_elastic_block_store": [], + "azure_disk": [], + "azure_file": [], + "ceph_fs": [], + "cinder": [], + "config_map": [], + "csi": [], + "downward_api": [], + "empty_dir": [], + "ephemeral": [], + "fc": [], + "flex_volume": [], + "flocker": [], + "gce_persistent_disk": [], + "git_repo": [], + "glusterfs": [], + "host_path": [ + { + "path": "/data/.warehouse", + "type": "DirectoryOrCreate" + } + ], + "iscsi": [], + "local": [], + "name": "warehouse", + "nfs": [], + "persistent_volume_claim": [], + "photon_persistent_disk": [], + "projected": [], + "quobyte": [], + "rbd": [], + "secret": [], + "vsphere_volume": [] + } + ] + } + ] + } + ], + "ttl_seconds_after_finished": "" + } + ] + } + ], + "schedule": "*/2 * * * *", + "starting_deadline_seconds": 0, + "successful_jobs_history_limit": 3, + "suspend": false, + "timezone": "" + } + ], + "timeouts": null + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "batch/v1", + "kind": "CronJob", + "name": "offload", + "namespace": "ground" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjo2MDAwMDAwMDAwMH19", + "dependencies": [ + "kubernetes_namespace.ground", + "kubernetes_secret.minio" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_deployment", + "name": "minio", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "id": "ground/minio", + "metadata": [ + { + "annotations": null, + "generate_name": "", + "generation": 1, + "labels": { + "app.kubernetes.io/part-of": "swarm-ground" + }, + "name": "minio", + "namespace": "ground", + "resource_version": "6405", + "uid": "31963382-0a1d-4d43-a221-1a14845948d5" + } + ], + "spec": [ + { + "min_ready_seconds": 0, + "paused": false, + "progress_deadline_seconds": 600, + "replicas": "1", + "revision_history_limit": 10, + "selector": [ + { + "match_expressions": [], + "match_labels": { + "app": "minio" + } + } + ], + "strategy": [ + { + "rolling_update": [], + "type": "Recreate" + } + ], + "template": [ + { + "metadata": [ + { + "annotations": null, + "generate_name": "", + "generation": 0, + "labels": { + "app": "minio", + "app.kubernetes.io/part-of": "swarm-ground" + }, + "name": "", + "namespace": "", + "resource_version": "", + "uid": "" + } + ], + "spec": [ + { + "active_deadline_seconds": 0, + "affinity": [], + "automount_service_account_token": true, + "container": [ + { + "args": [ + "server", + "/data", + "--console-address", + ":9001" + ], + "command": null, + "env": [ + { + "name": "MINIO_UPDATE", + "value": "off", + "value_from": [] + } + ], + "env_from": [ + { + "config_map_ref": [], + "prefix": "", + "secret_ref": [ + { + "name": "minio-credentials", + "optional": false + } + ] + } + ], + "image": "minio/minio:RELEASE.2024-06-13T22-53-53Z", + "image_pull_policy": "IfNotPresent", + "lifecycle": [], + "liveness_probe": [], + "name": "minio", + "port": [ + { + "container_port": 9000, + "host_ip": "", + "host_port": 0, + "name": "", + "protocol": "TCP" + }, + { + "container_port": 9001, + "host_ip": "", + "host_port": 0, + "name": "", + "protocol": "TCP" + } + ], + "readiness_probe": [], + "resources": [ + { + "limits": {}, + "requests": {} + } + ], + "security_context": [], + "startup_probe": [], + "stdin": false, + "stdin_once": false, + "termination_message_path": "/dev/termination-log", + "termination_message_policy": "File", + "tty": false, + "volume_device": [], + "volume_mount": [ + { + "mount_path": "/data", + "mount_propagation": "None", + "name": "store", + "read_only": false, + "sub_path": "", + "sub_path_expr": "" + } + ], + "working_dir": "" + } + ], + "dns_config": [], + "dns_policy": "ClusterFirst", + "enable_service_links": true, + "host_aliases": [], + "host_ipc": false, + "host_network": false, + "host_pid": false, + "hostname": "", + "image_pull_secrets": [], + "init_container": [], + "node_name": "", + "node_selector": null, + "os": [], + "priority_class_name": "", + "readiness_gate": [], + "restart_policy": "Always", + "runtime_class_name": "", + "scheduler_name": "default-scheduler", + "security_context": [], + "service_account_name": "", + "share_process_namespace": false, + "subdomain": "", + "termination_grace_period_seconds": 30, + "toleration": [], + "topology_spread_constraint": [], + "volume": [ + { + "aws_elastic_block_store": [], + "azure_disk": [], + "azure_file": [], + "ceph_fs": [], + "cinder": [], + "config_map": [], + "csi": [], + "downward_api": [], + "empty_dir": [], + "ephemeral": [], + "fc": [], + "flex_volume": [], + "flocker": [], + "gce_persistent_disk": [], + "git_repo": [], + "glusterfs": [], + "host_path": [], + "iscsi": [], + "local": [], + "name": "store", + "nfs": [], + "persistent_volume_claim": [ + { + "claim_name": "minio-data", + "read_only": false + } + ], + "photon_persistent_disk": [], + "projected": [], + "quobyte": [], + "rbd": [], + "secret": [], + "vsphere_volume": [] + } + ] + } + ] + } + ] + } + ], + "timeouts": null, + "wait_for_rollout": true + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "apps/v1", + "kind": "Deployment", + "name": "minio", + "namespace": "ground" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "kubernetes_namespace.ground", + "kubernetes_persistent_volume_claim.minio", + "kubernetes_secret.minio" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_deployment", + "name": "warehouse_explorer", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "id": "ground/warehouse-explorer", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 1, + "labels": { + "app.kubernetes.io/part-of": "swarm-ground" + }, + "name": "warehouse-explorer", + "namespace": "ground", + "resource_version": "5964", + "uid": "f85535d1-d718-4969-80dd-70d7b4544360" + } + ], + "spec": [ + { + "min_ready_seconds": 0, + "paused": false, + "progress_deadline_seconds": 600, + "replicas": "1", + "revision_history_limit": 10, + "selector": [ + { + "match_expressions": [], + "match_labels": { + "app": "warehouse-explorer" + } + } + ], + "strategy": [ + { + "rolling_update": [ + { + "max_surge": "25%", + "max_unavailable": "25%" + } + ], + "type": "RollingUpdate" + } + ], + "template": [ + { + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": { + "app": "warehouse-explorer", + "app.kubernetes.io/part-of": "swarm-ground" + }, + "name": "", + "namespace": "", + "resource_version": "", + "uid": "" + } + ], + "spec": [ + { + "active_deadline_seconds": 0, + "affinity": [], + "automount_service_account_token": true, + "container": [ + { + "args": [], + "command": [ + "python", + "explorer/server.py" + ], + "env": [ + { + "name": "DATA_DIR", + "value": "/warehouse", + "value_from": [] + } + ], + "env_from": [], + "image": "swarm-house/simulator:dev", + "image_pull_policy": "Never", + "lifecycle": [], + "liveness_probe": [], + "name": "explorer", + "port": [ + { + "container_port": 8088, + "host_ip": "", + "host_port": 0, + "name": "", + "protocol": "TCP" + } + ], + "readiness_probe": [], + "resources": [ + { + "limits": {}, + "requests": {} + } + ], + "security_context": [], + "startup_probe": [], + "stdin": false, + "stdin_once": false, + "termination_message_path": "/dev/termination-log", + "termination_message_policy": "File", + "tty": false, + "volume_device": [], + "volume_mount": [ + { + "mount_path": "/warehouse", + "mount_propagation": "None", + "name": "warehouse", + "read_only": true, + "sub_path": "", + "sub_path_expr": "" + } + ], + "working_dir": "" + } + ], + "dns_config": [], + "dns_policy": "ClusterFirst", + "enable_service_links": true, + "host_aliases": [], + "host_ipc": false, + "host_network": false, + "host_pid": false, + "hostname": "", + "image_pull_secrets": [], + "init_container": [], + "node_name": "", + "node_selector": {}, + "os": [], + "priority_class_name": "", + "readiness_gate": [], + "restart_policy": "Always", + "runtime_class_name": "", + "scheduler_name": "default-scheduler", + "security_context": [], + "service_account_name": "", + "share_process_namespace": false, + "subdomain": "", + "termination_grace_period_seconds": 30, + "toleration": [], + "topology_spread_constraint": [], + "volume": [ + { + "aws_elastic_block_store": [], + "azure_disk": [], + "azure_file": [], + "ceph_fs": [], + "cinder": [], + "config_map": [], + "csi": [], + "downward_api": [], + "empty_dir": [], + "ephemeral": [], + "fc": [], + "flex_volume": [], + "flocker": [], + "gce_persistent_disk": [], + "git_repo": [], + "glusterfs": [], + "host_path": [ + { + "path": "/data/.warehouse", + "type": "DirectoryOrCreate" + } + ], + "iscsi": [], + "local": [], + "name": "warehouse", + "nfs": [], + "persistent_volume_claim": [], + "photon_persistent_disk": [], + "projected": [], + "quobyte": [], + "rbd": [], + "secret": [], + "vsphere_volume": [] + } + ] + } + ] + } + ] + } + ], + "timeouts": null, + "wait_for_rollout": true + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "apps/v1", + "kind": "Deployment", + "name": "warehouse-explorer", + "namespace": "ground" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "kubernetes_namespace.ground" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_namespace", + "name": "ground", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ground", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": { + "app.kubernetes.io/part-of": "swarm-ground" + }, + "name": "ground", + "resource_version": "5926", + "uid": "e5be5fa0-11b9-4659-9bc1-f6b376f388bf" + } + ], + "timeouts": null, + "wait_for_default_service_account": false + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "v1", + "kind": "Namespace", + "name": "ground" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjozMDAwMDAwMDAwMDB9fQ==" + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_persistent_volume_claim", + "name": "minio", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ground/minio-data", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": {}, + "name": "minio-data", + "namespace": "ground", + "resource_version": "5947", + "uid": "480bcab4-d670-4f27-8909-9b1272d0005f" + } + ], + "spec": [ + { + "access_modes": [ + "ReadWriteOnce" + ], + "resources": [ + { + "limits": {}, + "requests": { + "storage": "2Gi" + } + } + ], + "selector": [], + "storage_class_name": "local-path", + "volume_mode": "Filesystem", + "volume_name": "" + } + ], + "timeouts": null, + "wait_until_bound": true + }, + "sensitive_attributes": [], + "identity_schema_version": 0, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjAifQ==", + "dependencies": [ + "kubernetes_namespace.ground" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_secret", + "name": "minio", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "binary_data": null, + "binary_data_wo": null, + "binary_data_wo_revision": null, + "data": { + "MINIO_ROOT_PASSWORD": "warehouse-sim-only", + "MINIO_ROOT_USER": "warehouse" + }, + "data_wo": null, + "data_wo_revision": null, + "id": "ground/minio-credentials", + "immutable": false, + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": {}, + "name": "minio-credentials", + "namespace": "ground", + "resource_version": "5929", + "uid": "94e85160-0dba-4570-8e61-92e299760ce3" + } + ], + "timeouts": null, + "type": "Opaque", + "wait_for_service_account_token": true + }, + "sensitive_attributes": [ + [ + { + "type": "get_attr", + "value": "binary_data" + } + ], + [ + { + "type": "get_attr", + "value": "data" + } + ], + [ + { + "type": "get_attr", + "value": "data" + }, + { + "type": "index", + "value": { + "value": "MINIO_ROOT_PASSWORD", + "type": "string" + } + } + ] + ], + "identity_schema_version": 1, + "identity": { + "api_version": "v1", + "kind": "Secret", + "name": "minio-credentials", + "namespace": "ground" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMH19", + "dependencies": [ + "kubernetes_namespace.ground" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_service", + "name": "minio", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "id": "ground/minio", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": {}, + "name": "minio", + "namespace": "ground", + "resource_version": "5937", + "uid": "f3412ffd-efb1-4ac6-a8f1-a4a5be7e7b26" + } + ], + "spec": [ + { + "allocate_load_balancer_node_ports": true, + "cluster_ip": "10.43.22.201", + "cluster_ips": [ + "10.43.22.201" + ], + "external_ips": [], + "external_name": "", + "external_traffic_policy": "Cluster", + "health_check_node_port": 0, + "internal_traffic_policy": "Cluster", + "ip_families": [ + "IPv4" + ], + "ip_family_policy": "SingleStack", + "load_balancer_class": "", + "load_balancer_ip": "", + "load_balancer_source_ranges": [], + "port": [ + { + "app_protocol": "", + "name": "api", + "node_port": 32746, + "port": 9000, + "protocol": "TCP", + "target_port": "9000" + }, + { + "app_protocol": "", + "name": "console", + "node_port": 30901, + "port": 9001, + "protocol": "TCP", + "target_port": "9001" + } + ], + "publish_not_ready_addresses": false, + "selector": { + "app": "minio" + }, + "session_affinity": "None", + "session_affinity_config": [], + "type": "NodePort" + } + ], + "status": [ + { + "load_balancer": [ + { + "ingress": [] + } + ] + } + ], + "timeouts": null, + "wait_for_load_balancer": true + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "v1", + "kind": "Service", + "name": "minio", + "namespace": "ground" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "kubernetes_namespace.ground" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_service", + "name": "warehouse_explorer", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "id": "ground/warehouse-explorer", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": {}, + "name": "warehouse-explorer", + "namespace": "ground", + "resource_version": "5933", + "uid": "55457495-82fb-4def-ac18-730652f75507" + } + ], + "spec": [ + { + "allocate_load_balancer_node_ports": true, + "cluster_ip": "10.43.212.60", + "cluster_ips": [ + "10.43.212.60" + ], + "external_ips": [], + "external_name": "", + "external_traffic_policy": "Cluster", + "health_check_node_port": 0, + "internal_traffic_policy": "Cluster", + "ip_families": [ + "IPv4" + ], + "ip_family_policy": "SingleStack", + "load_balancer_class": "", + "load_balancer_ip": "", + "load_balancer_source_ranges": [], + "port": [ + { + "app_protocol": "", + "name": "", + "node_port": 30089, + "port": 8088, + "protocol": "TCP", + "target_port": "8088" + } + ], + "publish_not_ready_addresses": false, + "selector": { + "app": "warehouse-explorer" + }, + "session_affinity": "None", + "session_affinity_config": [], + "type": "NodePort" + } + ], + "status": [ + { + "load_balancer": [ + { + "ingress": [] + } + ] + } + ], + "timeouts": null, + "wait_for_load_balancer": true + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "v1", + "kind": "Service", + "name": "warehouse-explorer", + "namespace": "ground" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "kubernetes_namespace.ground" + ] + } + ] + } + ], + "check_results": null +} diff --git a/infra/terraform/ground/terraform.tfstate.backup b/infra/terraform/ground/terraform.tfstate.backup new file mode 100644 index 0000000..5bc321e --- /dev/null +++ b/infra/terraform/ground/terraform.tfstate.backup @@ -0,0 +1,897 @@ +{ + "version": 4, + "terraform_version": "1.15.6", + "serial": 9, + "lineage": "df6d346f-1b17-4b8f-6165-3536b2b272fe", + "outputs": { + "minio_console_url": { + "value": "http://localhost:30901", + "type": "string" + }, + "offload_schedule": { + "value": "*/2 * * * *", + "type": "string" + }, + "warehouse_explorer_url": { + "value": "http://localhost:30089", + "type": "string" + } + }, + "resources": [ + { + "mode": "managed", + "type": "kubernetes_cron_job_v1", + "name": "offload", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ground/offload", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 1, + "labels": { + "app.kubernetes.io/part-of": "swarm-ground" + }, + "name": "offload", + "namespace": "ground", + "resource_version": "6337", + "uid": "a1b55fa7-eb9f-46d6-bddf-82576be4c872" + } + ], + "spec": [ + { + "concurrency_policy": "Forbid", + "failed_jobs_history_limit": 3, + "job_template": [ + { + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": { + "app.kubernetes.io/part-of": "swarm-ground" + }, + "name": "", + "namespace": "", + "resource_version": "", + "uid": "" + } + ], + "spec": [ + { + "active_deadline_seconds": 0, + "backoff_limit": 1, + "backoff_limit_per_index": 0, + "completion_mode": "", + "completions": 1, + "manual_selector": false, + "max_failed_indexes": 0, + "parallelism": 1, + "pod_failure_policy": [], + "selector": [], + "template": [ + { + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": { + "app.kubernetes.io/part-of": "swarm-ground" + }, + "name": "", + "resource_version": "", + "uid": "" + } + ], + "spec": [ + { + "active_deadline_seconds": 0, + "affinity": [], + "automount_service_account_token": true, + "container": [ + { + "args": [], + "command": [ + "sh", + "-c", + "mc alias set store http://minio:9000 \"$MINIO_ROOT_USER\" \"$MINIO_ROOT_PASSWORD\" \u0026\u0026 mc mb -p store/warehouse \u0026\u0026 mc mirror --overwrite /warehouse store/warehouse" + ], + "env": [], + "env_from": [ + { + "config_map_ref": [], + "prefix": "", + "secret_ref": [ + { + "name": "minio-credentials", + "optional": false + } + ] + } + ], + "image": "minio/mc:RELEASE.2024-06-12T14-34-03Z", + "image_pull_policy": "IfNotPresent", + "lifecycle": [], + "liveness_probe": [], + "name": "mirror", + "port": [], + "readiness_probe": [], + "resources": [ + { + "limits": {}, + "requests": {} + } + ], + "security_context": [], + "startup_probe": [], + "stdin": false, + "stdin_once": false, + "termination_message_path": "/dev/termination-log", + "termination_message_policy": "File", + "tty": false, + "volume_device": [], + "volume_mount": [ + { + "mount_path": "/warehouse", + "mount_propagation": "None", + "name": "warehouse", + "read_only": true, + "sub_path": "", + "sub_path_expr": "" + } + ], + "working_dir": "" + } + ], + "dns_config": [], + "dns_policy": "ClusterFirst", + "enable_service_links": true, + "host_aliases": [], + "host_ipc": false, + "host_network": false, + "host_pid": false, + "hostname": "", + "image_pull_secrets": [], + "init_container": [ + { + "args": [], + "command": [ + "sh", + "-c", + "cp -ru /lake/flight_id=* /warehouse/ 2\u003e/dev/null; ls /warehouse | wc -l" + ], + "env": [], + "env_from": [], + "image": "busybox:1.36", + "image_pull_policy": "IfNotPresent", + "lifecycle": [], + "liveness_probe": [], + "name": "copy", + "port": [], + "readiness_probe": [], + "resources": [ + { + "limits": {}, + "requests": {} + } + ], + "security_context": [], + "startup_probe": [], + "stdin": false, + "stdin_once": false, + "termination_message_path": "/dev/termination-log", + "termination_message_policy": "File", + "tty": false, + "volume_device": [], + "volume_mount": [ + { + "mount_path": "/lake", + "mount_propagation": "None", + "name": "lake", + "read_only": true, + "sub_path": "", + "sub_path_expr": "" + }, + { + "mount_path": "/warehouse", + "mount_propagation": "None", + "name": "warehouse", + "read_only": false, + "sub_path": "", + "sub_path_expr": "" + } + ], + "working_dir": "" + } + ], + "node_name": "", + "node_selector": {}, + "os": [], + "priority_class_name": "", + "readiness_gate": [], + "restart_policy": "Never", + "runtime_class_name": "", + "scheduler_name": "default-scheduler", + "security_context": [], + "service_account_name": "", + "share_process_namespace": false, + "subdomain": "", + "termination_grace_period_seconds": 30, + "toleration": [], + "topology_spread_constraint": [], + "volume": [ + { + "aws_elastic_block_store": [], + "azure_disk": [], + "azure_file": [], + "ceph_fs": [], + "cinder": [], + "config_map": [], + "csi": [], + "downward_api": [], + "empty_dir": [], + "ephemeral": [], + "fc": [], + "flex_volume": [], + "flocker": [], + "gce_persistent_disk": [], + "git_repo": [], + "glusterfs": [], + "host_path": [ + { + "path": "/data", + "type": "" + } + ], + "iscsi": [], + "local": [], + "name": "lake", + "nfs": [], + "persistent_volume_claim": [], + "photon_persistent_disk": [], + "projected": [], + "quobyte": [], + "rbd": [], + "secret": [], + "vsphere_volume": [] + }, + { + "aws_elastic_block_store": [], + "azure_disk": [], + "azure_file": [], + "ceph_fs": [], + "cinder": [], + "config_map": [], + "csi": [], + "downward_api": [], + "empty_dir": [], + "ephemeral": [], + "fc": [], + "flex_volume": [], + "flocker": [], + "gce_persistent_disk": [], + "git_repo": [], + "glusterfs": [], + "host_path": [ + { + "path": "/data/.warehouse", + "type": "DirectoryOrCreate" + } + ], + "iscsi": [], + "local": [], + "name": "warehouse", + "nfs": [], + "persistent_volume_claim": [], + "photon_persistent_disk": [], + "projected": [], + "quobyte": [], + "rbd": [], + "secret": [], + "vsphere_volume": [] + } + ] + } + ] + } + ], + "ttl_seconds_after_finished": "" + } + ] + } + ], + "schedule": "*/2 * * * *", + "starting_deadline_seconds": 0, + "successful_jobs_history_limit": 3, + "suspend": false, + "timezone": "" + } + ], + "timeouts": null + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "batch/v1", + "kind": "CronJob", + "name": "offload", + "namespace": "ground" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjo2MDAwMDAwMDAwMH19", + "dependencies": [ + "kubernetes_namespace.ground", + "kubernetes_secret.minio" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_deployment", + "name": "warehouse_explorer", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "id": "ground/warehouse-explorer", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 1, + "labels": { + "app.kubernetes.io/part-of": "swarm-ground" + }, + "name": "warehouse-explorer", + "namespace": "ground", + "resource_version": "5964", + "uid": "f85535d1-d718-4969-80dd-70d7b4544360" + } + ], + "spec": [ + { + "min_ready_seconds": 0, + "paused": false, + "progress_deadline_seconds": 600, + "replicas": "1", + "revision_history_limit": 10, + "selector": [ + { + "match_expressions": [], + "match_labels": { + "app": "warehouse-explorer" + } + } + ], + "strategy": [ + { + "rolling_update": [ + { + "max_surge": "25%", + "max_unavailable": "25%" + } + ], + "type": "RollingUpdate" + } + ], + "template": [ + { + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": { + "app": "warehouse-explorer", + "app.kubernetes.io/part-of": "swarm-ground" + }, + "name": "", + "namespace": "", + "resource_version": "", + "uid": "" + } + ], + "spec": [ + { + "active_deadline_seconds": 0, + "affinity": [], + "automount_service_account_token": true, + "container": [ + { + "args": [], + "command": [ + "python", + "explorer/server.py" + ], + "env": [ + { + "name": "DATA_DIR", + "value": "/warehouse", + "value_from": [] + } + ], + "env_from": [], + "image": "swarm-house/simulator:dev", + "image_pull_policy": "Never", + "lifecycle": [], + "liveness_probe": [], + "name": "explorer", + "port": [ + { + "container_port": 8088, + "host_ip": "", + "host_port": 0, + "name": "", + "protocol": "TCP" + } + ], + "readiness_probe": [], + "resources": [ + { + "limits": {}, + "requests": {} + } + ], + "security_context": [], + "startup_probe": [], + "stdin": false, + "stdin_once": false, + "termination_message_path": "/dev/termination-log", + "termination_message_policy": "File", + "tty": false, + "volume_device": [], + "volume_mount": [ + { + "mount_path": "/warehouse", + "mount_propagation": "None", + "name": "warehouse", + "read_only": true, + "sub_path": "", + "sub_path_expr": "" + } + ], + "working_dir": "" + } + ], + "dns_config": [], + "dns_policy": "ClusterFirst", + "enable_service_links": true, + "host_aliases": [], + "host_ipc": false, + "host_network": false, + "host_pid": false, + "hostname": "", + "image_pull_secrets": [], + "init_container": [], + "node_name": "", + "node_selector": {}, + "os": [], + "priority_class_name": "", + "readiness_gate": [], + "restart_policy": "Always", + "runtime_class_name": "", + "scheduler_name": "default-scheduler", + "security_context": [], + "service_account_name": "", + "share_process_namespace": false, + "subdomain": "", + "termination_grace_period_seconds": 30, + "toleration": [], + "topology_spread_constraint": [], + "volume": [ + { + "aws_elastic_block_store": [], + "azure_disk": [], + "azure_file": [], + "ceph_fs": [], + "cinder": [], + "config_map": [], + "csi": [], + "downward_api": [], + "empty_dir": [], + "ephemeral": [], + "fc": [], + "flex_volume": [], + "flocker": [], + "gce_persistent_disk": [], + "git_repo": [], + "glusterfs": [], + "host_path": [ + { + "path": "/data/.warehouse", + "type": "DirectoryOrCreate" + } + ], + "iscsi": [], + "local": [], + "name": "warehouse", + "nfs": [], + "persistent_volume_claim": [], + "photon_persistent_disk": [], + "projected": [], + "quobyte": [], + "rbd": [], + "secret": [], + "vsphere_volume": [] + } + ] + } + ] + } + ] + } + ], + "timeouts": null, + "wait_for_rollout": true + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "apps/v1", + "kind": "Deployment", + "name": "warehouse-explorer", + "namespace": "ground" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "kubernetes_namespace.ground" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_namespace", + "name": "ground", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ground", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": { + "app.kubernetes.io/part-of": "swarm-ground" + }, + "name": "ground", + "resource_version": "5926", + "uid": "e5be5fa0-11b9-4659-9bc1-f6b376f388bf" + } + ], + "timeouts": null, + "wait_for_default_service_account": false + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "v1", + "kind": "Namespace", + "name": "ground" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjozMDAwMDAwMDAwMDB9fQ==" + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_persistent_volume_claim", + "name": "minio", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ground/minio-data", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": {}, + "name": "minio-data", + "namespace": "ground", + "resource_version": "5947", + "uid": "480bcab4-d670-4f27-8909-9b1272d0005f" + } + ], + "spec": [ + { + "access_modes": [ + "ReadWriteOnce" + ], + "resources": [ + { + "limits": {}, + "requests": { + "storage": "2Gi" + } + } + ], + "selector": [], + "storage_class_name": "local-path", + "volume_mode": "Filesystem", + "volume_name": "" + } + ], + "timeouts": null, + "wait_until_bound": true + }, + "sensitive_attributes": [], + "identity_schema_version": 0, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjAifQ==" + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_secret", + "name": "minio", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "binary_data": null, + "binary_data_wo": null, + "binary_data_wo_revision": null, + "data": { + "MINIO_ROOT_PASSWORD": "warehouse-sim-only", + "MINIO_ROOT_USER": "warehouse" + }, + "data_wo": null, + "data_wo_revision": null, + "id": "ground/minio-credentials", + "immutable": false, + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": {}, + "name": "minio-credentials", + "namespace": "ground", + "resource_version": "5929", + "uid": "94e85160-0dba-4570-8e61-92e299760ce3" + } + ], + "timeouts": null, + "type": "Opaque", + "wait_for_service_account_token": true + }, + "sensitive_attributes": [ + [ + { + "type": "get_attr", + "value": "binary_data" + } + ], + [ + { + "type": "get_attr", + "value": "data" + } + ], + [ + { + "type": "get_attr", + "value": "data" + }, + { + "type": "index", + "value": { + "value": "MINIO_ROOT_PASSWORD", + "type": "string" + } + } + ] + ], + "identity_schema_version": 1, + "identity": { + "api_version": "v1", + "kind": "Secret", + "name": "minio-credentials", + "namespace": "ground" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMH19", + "dependencies": [ + "kubernetes_namespace.ground" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_service", + "name": "minio", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "id": "ground/minio", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": {}, + "name": "minio", + "namespace": "ground", + "resource_version": "5937", + "uid": "f3412ffd-efb1-4ac6-a8f1-a4a5be7e7b26" + } + ], + "spec": [ + { + "allocate_load_balancer_node_ports": true, + "cluster_ip": "10.43.22.201", + "cluster_ips": [ + "10.43.22.201" + ], + "external_ips": [], + "external_name": "", + "external_traffic_policy": "Cluster", + "health_check_node_port": 0, + "internal_traffic_policy": "Cluster", + "ip_families": [ + "IPv4" + ], + "ip_family_policy": "SingleStack", + "load_balancer_class": "", + "load_balancer_ip": "", + "load_balancer_source_ranges": [], + "port": [ + { + "app_protocol": "", + "name": "api", + "node_port": 32746, + "port": 9000, + "protocol": "TCP", + "target_port": "9000" + }, + { + "app_protocol": "", + "name": "console", + "node_port": 30901, + "port": 9001, + "protocol": "TCP", + "target_port": "9001" + } + ], + "publish_not_ready_addresses": false, + "selector": { + "app": "minio" + }, + "session_affinity": "None", + "session_affinity_config": [], + "type": "NodePort" + } + ], + "status": [ + { + "load_balancer": [ + { + "ingress": [] + } + ] + } + ], + "timeouts": null, + "wait_for_load_balancer": true + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "v1", + "kind": "Service", + "name": "minio", + "namespace": "ground" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "kubernetes_namespace.ground" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_service", + "name": "warehouse_explorer", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "id": "ground/warehouse-explorer", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": {}, + "name": "warehouse-explorer", + "namespace": "ground", + "resource_version": "5933", + "uid": "55457495-82fb-4def-ac18-730652f75507" + } + ], + "spec": [ + { + "allocate_load_balancer_node_ports": true, + "cluster_ip": "10.43.212.60", + "cluster_ips": [ + "10.43.212.60" + ], + "external_ips": [], + "external_name": "", + "external_traffic_policy": "Cluster", + "health_check_node_port": 0, + "internal_traffic_policy": "Cluster", + "ip_families": [ + "IPv4" + ], + "ip_family_policy": "SingleStack", + "load_balancer_class": "", + "load_balancer_ip": "", + "load_balancer_source_ranges": [], + "port": [ + { + "app_protocol": "", + "name": "", + "node_port": 30089, + "port": 8088, + "protocol": "TCP", + "target_port": "8088" + } + ], + "publish_not_ready_addresses": false, + "selector": { + "app": "warehouse-explorer" + }, + "session_affinity": "None", + "session_affinity_config": [], + "type": "NodePort" + } + ], + "status": [ + { + "load_balancer": [ + { + "ingress": [] + } + ] + } + ], + "timeouts": null, + "wait_for_load_balancer": true + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "v1", + "kind": "Service", + "name": "warehouse-explorer", + "namespace": "ground" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "kubernetes_namespace.ground" + ] + } + ] + } + ], + "check_results": null +} diff --git a/infra/terraform/ground/variables.tf b/infra/terraform/ground/variables.tf new file mode 100644 index 0000000..7a2dba4 --- /dev/null +++ b/infra/terraform/ground/variables.tf @@ -0,0 +1,72 @@ +variable "kubeconfig" { + description = "Path to the kubeconfig for the simulation cluster" + type = string + default = "~/.kube/config" +} + +variable "kube_context" { + description = "Kubeconfig context of the k3d simulation cluster" + type = string + default = "k3d-swarm-sim" +} + +variable "namespace" { + description = "Namespace for the ground warehouse segment" + type = string + default = "ground" +} + +variable "simulator_image" { + description = "Simulator image (provides the explorer; imported by Ansible)" + type = string + default = "swarm-house/simulator:dev" +} + +variable "lake_host_path" { + description = "Host path of the fleet's live lake (T1) inside the k3d node" + type = string + default = "/data" +} + +variable "warehouse_host_path" { + description = "Host path of the historical warehouse (T3) inside the k3d node; dot-prefixed so lake globs never match it" + type = string + default = "/data/.warehouse" +} + +variable "offload_schedule" { + description = "Cron schedule of the post-flight offload job (T1 -> T3)" + type = string + default = "*/2 * * * *" +} + +variable "minio_root_user" { + description = "MinIO root user (simulation only; production uses sealed credentials)" + type = string + default = "warehouse" +} + +variable "minio_root_password" { + description = "MinIO root password (simulation only)" + type = string + default = "warehouse-sim-only" + sensitive = true +} + +variable "minio_storage" { + description = "PVC size for the MinIO object store" + type = string + default = "2Gi" +} + +variable "node_ports" { + description = "Host-reachable NodePorts (must match the ports opened by sim-cluster.yml)" + type = object({ + warehouse_explorer = number + minio_console = number + }) + default = { + warehouse_explorer = 30089 + minio_console = 30901 + } +} diff --git a/infra/terraform/ground/versions.tf b/infra/terraform/ground/versions.tf new file mode 100644 index 0000000..18c36cc --- /dev/null +++ b/infra/terraform/ground/versions.tf @@ -0,0 +1,15 @@ +terraform { + required_version = ">= 1.5" + + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.33" + } + } +} + +provider "kubernetes" { + config_path = var.kubeconfig + config_context = var.kube_context +} diff --git a/infra/terraform/sim-env/.terraform.lock.hcl b/infra/terraform/sim-env/.terraform.lock.hcl new file mode 100644 index 0000000..a7e5860 --- /dev/null +++ b/infra/terraform/sim-env/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.38.0" + constraints = "~> 2.33" + hashes = [ + "h1:5CkveFo5ynsLdzKk+Kv+r7+U9rMrNjfZPT3a0N/fhgE=", + "zh:0af928d776eb269b192dc0ea0f8a3f0f5ec117224cd644bdacdc682300f84ba0", + "zh:1be998e67206f7cfc4ffe77c01a09ac91ce725de0abaec9030b22c0a832af44f", + "zh:326803fe5946023687d603f6f1bab24de7af3d426b01d20e51d4e6fbe4e7ec1b", + "zh:4a99ec8d91193af961de1abb1f824be73df07489301d62e6141a656b3ebfff12", + "zh:5136e51765d6a0b9e4dbcc3b38821e9736bd2136cf15e9aac11668f22db117d2", + "zh:63fab47349852d7802fb032e4f2b6a101ee1ce34b62557a9ad0f0f0f5b6ecfdc", + "zh:924fb0257e2d03e03e2bfe9c7b99aa73c195b1f19412ca09960001bee3c50d15", + "zh:b63a0be5e233f8f6727c56bed3b61eb9456ca7a8bb29539fba0837f1badf1396", + "zh:d39861aa21077f1bc899bc53e7233262e530ba8a3a2d737449b100daeb303e4d", + "zh:de0805e10ebe4c83ce3b728a67f6b0f9d18be32b25146aa89116634df5145ad4", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:faf23e45f0090eef8ba28a8aac7ec5d4fdf11a36c40a8d286304567d71c1e7db", + ] +} diff --git a/infra/terraform/sim-env/.terraform/providers/registry.terraform.io/hashicorp/kubernetes/2.38.0/linux_amd64/LICENSE.txt b/infra/terraform/sim-env/.terraform/providers/registry.terraform.io/hashicorp/kubernetes/2.38.0/linux_amd64/LICENSE.txt new file mode 100644 index 0000000..b9ac071 --- /dev/null +++ b/infra/terraform/sim-env/.terraform/providers/registry.terraform.io/hashicorp/kubernetes/2.38.0/linux_amd64/LICENSE.txt @@ -0,0 +1,375 @@ +Copyright (c) 2017 HashiCorp, Inc. + +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/infra/terraform/sim-env/.terraform/providers/registry.terraform.io/hashicorp/kubernetes/2.38.0/linux_amd64/terraform-provider-kubernetes_v2.38.0_x5 b/infra/terraform/sim-env/.terraform/providers/registry.terraform.io/hashicorp/kubernetes/2.38.0/linux_amd64/terraform-provider-kubernetes_v2.38.0_x5 new file mode 100755 index 0000000..a02e3e5 Binary files /dev/null and b/infra/terraform/sim-env/.terraform/providers/registry.terraform.io/hashicorp/kubernetes/2.38.0/linux_amd64/terraform-provider-kubernetes_v2.38.0_x5 differ diff --git a/infra/terraform/sim-env/README.md b/infra/terraform/sim-env/README.md new file mode 100644 index 0000000..b305735 --- /dev/null +++ b/infra/terraform/sim-env/README.md @@ -0,0 +1,40 @@ +# Terraform — simulation environment (T4) + +Declares the simulated fleet and its observability stack on the k3d cluster +created by [`../../ansible/sim-cluster.yml`](../../ansible/sim-cluster.yml). +This module is the executable miniature of the dev/sim environment from +[06 — Environments](../../../docs/06-environments.md). + +| Resource | Purpose | +| --- | --- | +| StatefulSet `drone` × `drone_count` | Virtual drones; stable pod names become `DRONE_ID`s; all write Hive-partitioned Parquet into the shared `/data` lake | +| Deployment `exporter` | Prometheus exporter reading the lake with DuckDB | +| Deployment `explorer` + NodePort 30088 | Partition tree + read-only SQL console (also feeds the prototype's live mode) | +| Deployment `prometheus` + NodePort 30990 | Scrapes the exporter | +| Deployment `grafana` + NodePort 30300 | Same dashboard JSON as the Compose profile, provisioned from a ConfigMap | + +## Usage + +```bash +ansible-playbook ../../ansible/sim-cluster.yml # once: cluster + image +terraform init +terraform apply +terraform output # URLs +``` + +Scale the fleet without touching YAML: + +```bash +terraform apply -var drone_count=9 -var speedup=4 +``` + +## Notes + +- `image_pull_policy = "Never"` — the simulator image is imported by the + Ansible playbook (`k3d image import`); the cluster never pulls from a + registry, mirroring the air-gap doctrine. +- Pods discover their identity from the pod name (StatefulSet ordinal), so + a flight survives `kubectl delete pod` the same way a drone survives a + power cycle: new process, same identity, new flight ID. +- State is local (`terraform.tfstate` in this directory) — the simulation + cluster is disposable; nothing here is shared infrastructure. diff --git a/infra/terraform/sim-env/main.tf b/infra/terraform/sim-env/main.tf new file mode 100644 index 0000000..9215ab3 --- /dev/null +++ b/infra/terraform/sim-env/main.tf @@ -0,0 +1,439 @@ +# Simulated fleet (T4 dev environment, docs/06) on the k3d cluster created +# by infra/ansible/sim-cluster.yml. Drones are StatefulSet replicas writing +# to a shared hostPath lake — the same /data contract as the on-board NVMe. + +locals { + labels = { "app.kubernetes.io/part-of" = "swarm-sim" } +} + +resource "kubernetes_namespace" "swarm" { + metadata { + name = var.namespace + labels = local.labels + } +} + +# --- Fleet ------------------------------------------------------------------ + +resource "kubernetes_stateful_set" "drone" { + metadata { + name = "drone" + namespace = kubernetes_namespace.swarm.metadata[0].name + labels = local.labels + } + + spec { + service_name = "drone" + replicas = var.drone_count + # A landed drone powers off; the next start is the next flight + pod_management_policy = "Parallel" + + selector { + match_labels = { app = "drone" } + } + + template { + metadata { + labels = merge(local.labels, { app = "drone" }) + } + + spec { + container { + name = "drone" + image = var.simulator_image + image_pull_policy = "Never" # imported via k3d image import + + env { + name = "DRONE_ID" + value_from { + field_ref { + field_path = "metadata.name" + } + } + } + env { + name = "DURATION_S" + value = tostring(var.flight_duration_s) + } + env { + name = "SPEEDUP" + value = tostring(var.speedup) + } + env { + name = "DATA_DIR" + value = "/data" + } + + volume_mount { + name = "lake" + mount_path = "/data" + } + + resources { + requests = { cpu = "50m", memory = "128Mi" } + limits = { cpu = "500m", memory = "512Mi" } + } + } + + volume { + name = "lake" + host_path { + path = var.data_host_path + } + } + } + } + } +} + +# --- Exporter + Prometheus + Grafana (observability, docs/07) ---------------- + +resource "kubernetes_deployment" "exporter" { + metadata { + name = "exporter" + namespace = kubernetes_namespace.swarm.metadata[0].name + labels = local.labels + } + + spec { + replicas = 1 + selector { + match_labels = { app = "exporter" } + } + template { + metadata { + labels = merge(local.labels, { app = "exporter" }) + } + spec { + container { + name = "exporter" + image = var.simulator_image + image_pull_policy = "Never" + command = ["python", "monitoring/exporter.py"] + env { + name = "DATA_DIR" + value = "/data" + } + env { + name = "EXPORTER_PORT" + value = "9105" + } + port { + container_port = 9105 + } + volume_mount { + name = "lake" + mount_path = "/data" + read_only = true + } + } + volume { + name = "lake" + host_path { + path = var.data_host_path + } + } + } + } + } +} + +resource "kubernetes_service" "exporter" { + metadata { + name = "exporter" + namespace = kubernetes_namespace.swarm.metadata[0].name + } + spec { + selector = { app = "exporter" } + port { + port = 9105 + target_port = 9105 + } + } +} + +resource "kubernetes_deployment" "explorer" { + metadata { + name = "explorer" + namespace = kubernetes_namespace.swarm.metadata[0].name + labels = local.labels + } + + spec { + replicas = 1 + selector { + match_labels = { app = "explorer" } + } + template { + metadata { + labels = merge(local.labels, { app = "explorer" }) + } + spec { + container { + name = "explorer" + image = var.simulator_image + image_pull_policy = "Never" + command = ["python", "explorer/server.py"] + env { + name = "DATA_DIR" + value = "/data" + } + env { + name = "EXPLORER_PORT" + value = "8088" + } + port { + container_port = 8088 + } + volume_mount { + name = "lake" + mount_path = "/data" + read_only = true + } + } + volume { + name = "lake" + host_path { + path = var.data_host_path + } + } + } + } + } +} + +resource "kubernetes_service" "explorer" { + metadata { + name = "explorer" + namespace = kubernetes_namespace.swarm.metadata[0].name + } + spec { + type = "NodePort" + selector = { app = "explorer" } + port { + port = 8088 + target_port = 8088 + node_port = var.node_ports.explorer + } + } +} + +resource "kubernetes_config_map" "prometheus" { + metadata { + name = "prometheus-config" + namespace = kubernetes_namespace.swarm.metadata[0].name + } + data = { + "prometheus.yml" = <<-EOT + global: + scrape_interval: 5s + scrape_configs: + - job_name: swarm + static_configs: + - targets: ["exporter:9105"] + EOT + } +} + +resource "kubernetes_deployment" "prometheus" { + metadata { + name = "prometheus" + namespace = kubernetes_namespace.swarm.metadata[0].name + labels = local.labels + } + + spec { + replicas = 1 + selector { + match_labels = { app = "prometheus" } + } + template { + metadata { + labels = merge(local.labels, { app = "prometheus" }) + } + spec { + container { + name = "prometheus" + image = "prom/prometheus:v2.53.0" + port { + container_port = 9090 + } + volume_mount { + name = "config" + mount_path = "/etc/prometheus" + } + } + volume { + name = "config" + config_map { + name = kubernetes_config_map.prometheus.metadata[0].name + } + } + } + } + } +} + +resource "kubernetes_service" "prometheus" { + metadata { + name = "prometheus" + namespace = kubernetes_namespace.swarm.metadata[0].name + } + spec { + type = "NodePort" + selector = { app = "prometheus" } + port { + port = 9090 + target_port = 9090 + node_port = var.node_ports.prometheus + } + } +} + +resource "kubernetes_config_map" "grafana_provisioning" { + metadata { + name = "grafana-provisioning" + namespace = kubernetes_namespace.swarm.metadata[0].name + } + data = { + "datasource.yml" = <<-EOT + apiVersion: 1 + datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + EOT + "dashboards.yml" = <<-EOT + apiVersion: 1 + providers: + - name: swarm + folder: "" + type: file + options: + path: /var/lib/grafana/dashboards + EOT + } +} + +resource "kubernetes_config_map" "grafana_dashboard" { + metadata { + name = "grafana-dashboard" + namespace = kubernetes_namespace.swarm.metadata[0].name + } + data = { + "swarm.json" = file("${path.module}/../../../simulator/monitoring/grafana/dashboards/swarm.json") + } +} + +resource "kubernetes_deployment" "grafana" { + metadata { + name = "grafana" + namespace = kubernetes_namespace.swarm.metadata[0].name + labels = local.labels + } + + spec { + replicas = 1 + selector { + match_labels = { app = "grafana" } + } + template { + metadata { + labels = merge(local.labels, { app = "grafana" }) + } + spec { + container { + name = "grafana" + image = "grafana/grafana:11.1.0" + port { + container_port = 3000 + } + + env { + name = "GF_AUTH_ANONYMOUS_ENABLED" + value = "true" + } + env { + name = "GF_AUTH_ANONYMOUS_ORG_ROLE" + value = "Admin" + } + env { + name = "GF_AUTH_DISABLE_LOGIN_FORM" + value = "true" + } + # Air-gap hygiene — same flags as the Compose profile + env { + name = "GF_ANALYTICS_REPORTING_ENABLED" + value = "false" + } + env { + name = "GF_ANALYTICS_CHECK_FOR_UPDATES" + value = "false" + } + env { + name = "GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES" + value = "false" + } + + volume_mount { + name = "provisioning-datasources" + mount_path = "/etc/grafana/provisioning/datasources" + } + volume_mount { + name = "provisioning-dashboards" + mount_path = "/etc/grafana/provisioning/dashboards" + } + volume_mount { + name = "dashboards" + mount_path = "/var/lib/grafana/dashboards" + } + } + + volume { + name = "provisioning-datasources" + config_map { + name = kubernetes_config_map.grafana_provisioning.metadata[0].name + items { + key = "datasource.yml" + path = "datasource.yml" + } + } + } + volume { + name = "provisioning-dashboards" + config_map { + name = kubernetes_config_map.grafana_provisioning.metadata[0].name + items { + key = "dashboards.yml" + path = "dashboards.yml" + } + } + } + volume { + name = "dashboards" + config_map { + name = kubernetes_config_map.grafana_dashboard.metadata[0].name + } + } + } + } + } +} + +resource "kubernetes_service" "grafana" { + metadata { + name = "grafana" + namespace = kubernetes_namespace.swarm.metadata[0].name + } + spec { + type = "NodePort" + selector = { app = "grafana" } + port { + port = 3000 + target_port = 3000 + node_port = var.node_ports.grafana + } + } +} diff --git a/infra/terraform/sim-env/outputs.tf b/infra/terraform/sim-env/outputs.tf new file mode 100644 index 0000000..47e41bf --- /dev/null +++ b/infra/terraform/sim-env/outputs.tf @@ -0,0 +1,23 @@ +output "explorer_url" { + description = "Data-plane explorer (partition tree + read-only SQL)" + value = "http://localhost:${var.node_ports.explorer}" +} + +output "grafana_url" { + description = "Grafana dashboards" + value = "http://localhost:${var.node_ports.grafana}" +} + +output "prometheus_url" { + description = "Prometheus" + value = "http://localhost:${var.node_ports.prometheus}" +} + +output "fleet" { + description = "Deployed fleet shape" + value = { + drones = var.drone_count + flight_duration_s = var.flight_duration_s + speedup = var.speedup + } +} diff --git a/infra/terraform/sim-env/terraform.tfstate b/infra/terraform/sim-env/terraform.tfstate new file mode 100644 index 0000000..eac6d7c --- /dev/null +++ b/infra/terraform/sim-env/terraform.tfstate @@ -0,0 +1,1791 @@ +{ + "version": 4, + "terraform_version": "1.15.6", + "serial": 19, + "lineage": "e4ea037b-4cc0-6b11-c7c6-7c3c1f968e04", + "outputs": { + "explorer_url": { + "value": "http://localhost:30088", + "type": "string" + }, + "fleet": { + "value": { + "drones": 5, + "flight_duration_s": 300, + "speedup": 2 + }, + "type": [ + "object", + { + "drones": "number", + "flight_duration_s": "number", + "speedup": "number" + } + ] + }, + "grafana_url": { + "value": "http://localhost:30300", + "type": "string" + }, + "prometheus_url": { + "value": "http://localhost:30990", + "type": "string" + } + }, + "resources": [ + { + "mode": "managed", + "type": "kubernetes_config_map", + "name": "grafana_dashboard", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "binary_data": {}, + "data": { + "swarm.json": "{\n \"uid\": \"swarm-fleet\",\n \"title\": \"Swarm Fleet — generated data\",\n \"tags\": [\"swarm\"],\n \"timezone\": \"utc\",\n \"schemaVersion\": 39,\n \"version\": 1,\n \"refresh\": \"5s\",\n \"time\": { \"from\": \"now-15m\", \"to\": \"now\" },\n \"panels\": [\n {\n \"id\": 1, \"type\": \"stat\", \"title\": \"Telemetry rows\",\n \"gridPos\": { \"h\": 5, \"w\": 4, \"x\": 0, \"y\": 0 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"swarm-prom\" },\n \"targets\": [{ \"expr\": \"sum(swarm_rows_total)\", \"instant\": true, \"refId\": \"A\" }],\n \"options\": { \"reduceOptions\": { \"calcs\": [\"lastNotNull\"] }, \"colorMode\": \"value\" },\n \"fieldConfig\": { \"defaults\": { \"unit\": \"short\", \"color\": { \"mode\": \"fixed\", \"fixedColor\": \"blue\" } }, \"overrides\": [] }\n },\n {\n \"id\": 2, \"type\": \"stat\", \"title\": \"Detections\",\n \"gridPos\": { \"h\": 5, \"w\": 4, \"x\": 4, \"y\": 0 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"swarm-prom\" },\n \"targets\": [{ \"expr\": \"sum(swarm_detections_total)\", \"instant\": true, \"refId\": \"A\" }],\n \"options\": { \"reduceOptions\": { \"calcs\": [\"lastNotNull\"] }, \"colorMode\": \"value\" },\n \"fieldConfig\": { \"defaults\": { \"unit\": \"short\", \"color\": { \"mode\": \"fixed\", \"fixedColor\": \"green\" } }, \"overrides\": [] }\n },\n {\n \"id\": 3, \"type\": \"stat\", \"title\": \"Pose frames (sent+received)\",\n \"gridPos\": { \"h\": 5, \"w\": 4, \"x\": 8, \"y\": 0 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"swarm-prom\" },\n \"targets\": [{ \"expr\": \"sum(swarm_state_frames_total)\", \"instant\": true, \"refId\": \"A\" }],\n \"options\": { \"reduceOptions\": { \"calcs\": [\"lastNotNull\"] }, \"colorMode\": \"value\" },\n \"fieldConfig\": { \"defaults\": { \"unit\": \"short\" }, \"overrides\": [] }\n },\n {\n \"id\": 4, \"type\": \"stat\", \"title\": \"Parquet on disk\",\n \"gridPos\": { \"h\": 5, \"w\": 4, \"x\": 12, \"y\": 0 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"swarm-prom\" },\n \"targets\": [{ \"expr\": \"sum(swarm_parquet_bytes)\", \"instant\": true, \"refId\": \"A\" }],\n \"options\": { \"reduceOptions\": { \"calcs\": [\"lastNotNull\"] }, \"colorMode\": \"value\" },\n \"fieldConfig\": { \"defaults\": { \"unit\": \"bytes\", \"color\": { \"mode\": \"fixed\", \"fixedColor\": \"purple\" } }, \"overrides\": [] }\n },\n {\n \"id\": 5, \"type\": \"stat\", \"title\": \"Parquet files\",\n \"gridPos\": { \"h\": 5, \"w\": 4, \"x\": 16, \"y\": 0 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"swarm-prom\" },\n \"targets\": [{ \"expr\": \"swarm_parquet_files\", \"instant\": true, \"refId\": \"A\" }],\n \"options\": { \"reduceOptions\": { \"calcs\": [\"lastNotNull\"] } },\n \"fieldConfig\": { \"defaults\": { \"unit\": \"short\" }, \"overrides\": [] }\n },\n {\n \"id\": 6, \"type\": \"gauge\", \"title\": \"Fleet battery (min)\",\n \"gridPos\": { \"h\": 5, \"w\": 4, \"x\": 20, \"y\": 0 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"swarm-prom\" },\n \"targets\": [{ \"expr\": \"min(swarm_battery_pct)\", \"instant\": true, \"refId\": \"A\" }],\n \"fieldConfig\": {\n \"defaults\": {\n \"unit\": \"percent\", \"min\": 0, \"max\": 100,\n \"thresholds\": { \"mode\": \"absolute\", \"steps\": [\n { \"color\": \"red\", \"value\": null }, { \"color\": \"yellow\", \"value\": 30 }, { \"color\": \"green\", \"value\": 60 }\n ] }\n }, \"overrides\": []\n }\n },\n {\n \"id\": 7, \"type\": \"timeseries\", \"title\": \"Telemetry rows by drone\",\n \"gridPos\": { \"h\": 8, \"w\": 12, \"x\": 0, \"y\": 5 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"swarm-prom\" },\n \"targets\": [{ \"expr\": \"sum by (drone) (swarm_rows_total)\", \"legendFormat\": \"{{drone}}\", \"refId\": \"A\" }],\n \"fieldConfig\": { \"defaults\": { \"unit\": \"short\", \"custom\": { \"fillOpacity\": 12 } }, \"overrides\": [] }\n },\n {\n \"id\": 8, \"type\": \"timeseries\", \"title\": \"Rows by sensor (fleet)\",\n \"gridPos\": { \"h\": 8, \"w\": 12, \"x\": 12, \"y\": 5 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"swarm-prom\" },\n \"targets\": [{ \"expr\": \"sum by (sensor) (swarm_rows_total)\", \"legendFormat\": \"{{sensor}}\", \"refId\": \"A\" }],\n \"fieldConfig\": { \"defaults\": { \"unit\": \"short\", \"custom\": { \"fillOpacity\": 12 } }, \"overrides\": [] }\n },\n {\n \"id\": 9, \"type\": \"timeseries\", \"title\": \"Battery per drone\",\n \"gridPos\": { \"h\": 8, \"w\": 8, \"x\": 0, \"y\": 13 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"swarm-prom\" },\n \"targets\": [{ \"expr\": \"swarm_battery_pct\", \"legendFormat\": \"{{drone}}\", \"refId\": \"A\" }],\n \"fieldConfig\": { \"defaults\": { \"unit\": \"percent\", \"min\": 0, \"max\": 100 }, \"overrides\": [] }\n },\n {\n \"id\": 10, \"type\": \"timeseries\", \"title\": \"RSSI per link\",\n \"gridPos\": { \"h\": 8, \"w\": 8, \"x\": 8, \"y\": 13 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"swarm-prom\" },\n \"targets\": [{ \"expr\": \"swarm_rssi_dbm\", \"legendFormat\": \"{{drone}} ← {{peer}}\", \"refId\": \"A\" }],\n \"fieldConfig\": { \"defaults\": { \"unit\": \"dBm\" }, \"overrides\": [] }\n },\n {\n \"id\": 11, \"type\": \"timeseries\", \"title\": \"Inter-drone distance\",\n \"gridPos\": { \"h\": 8, \"w\": 8, \"x\": 16, \"y\": 13 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"swarm-prom\" },\n \"targets\": [{ \"expr\": \"swarm_peer_distance_m\", \"legendFormat\": \"{{drone}} ↔ {{peer}}\", \"refId\": \"A\" }],\n \"fieldConfig\": { \"defaults\": { \"unit\": \"lengthm\" }, \"overrides\": [] }\n },\n {\n \"id\": 12, \"type\": \"bargauge\", \"title\": \"Detections by class\",\n \"gridPos\": { \"h\": 7, \"w\": 12, \"x\": 0, \"y\": 21 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"swarm-prom\" },\n \"targets\": [{ \"expr\": \"sum by (cls) (swarm_detections_total)\", \"legendFormat\": \"{{cls}}\", \"instant\": true, \"refId\": \"A\" }],\n \"options\": { \"displayMode\": \"gradient\", \"orientation\": \"horizontal\", \"reduceOptions\": { \"calcs\": [\"lastNotNull\"] } },\n \"fieldConfig\": { \"defaults\": { \"unit\": \"short\", \"color\": { \"mode\": \"continuous-GrYlRd\" } }, \"overrides\": [] }\n },\n {\n \"id\": 13, \"type\": \"timeseries\", \"title\": \"Pose frames by direction\",\n \"gridPos\": { \"h\": 7, \"w\": 12, \"x\": 12, \"y\": 21 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"swarm-prom\" },\n \"targets\": [{ \"expr\": \"sum by (drone, direction) (swarm_state_frames_total)\", \"legendFormat\": \"{{drone}} {{direction}}\", \"refId\": \"A\" }],\n \"fieldConfig\": { \"defaults\": { \"unit\": \"short\", \"custom\": { \"fillOpacity\": 8 } }, \"overrides\": [] }\n }\n ]\n}\n" + }, + "id": "swarm/grafana-dashboard", + "immutable": false, + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": {}, + "name": "grafana-dashboard", + "namespace": "swarm", + "resource_version": "476", + "uid": "31523ebd-1116-4e3e-a9a0-cdb01d4b8e37" + } + ] + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "v1", + "kind": "ConfigMap", + "name": "grafana-dashboard", + "namespace": "swarm" + }, + "private": "bnVsbA==", + "dependencies": [ + "kubernetes_namespace.swarm" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_config_map", + "name": "grafana_provisioning", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "binary_data": {}, + "data": { + "dashboards.yml": "apiVersion: 1\nproviders:\n - name: swarm\n folder: \"\"\n type: file\n options:\n path: /var/lib/grafana/dashboards\n", + "datasource.yml": "apiVersion: 1\ndatasources:\n - name: Prometheus\n type: prometheus\n access: proxy\n url: http://prometheus:9090\n isDefault: true\n" + }, + "id": "swarm/grafana-provisioning", + "immutable": false, + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": {}, + "name": "grafana-provisioning", + "namespace": "swarm", + "resource_version": "475", + "uid": "a26bceb4-8ae8-42c5-ba0b-99f2e66f8765" + } + ] + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "v1", + "kind": "ConfigMap", + "name": "grafana-provisioning", + "namespace": "swarm" + }, + "private": "bnVsbA==", + "dependencies": [ + "kubernetes_namespace.swarm" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_config_map", + "name": "prometheus", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "binary_data": {}, + "data": { + "prometheus.yml": "global:\n scrape_interval: 5s\nscrape_configs:\n - job_name: swarm\n static_configs:\n - targets: [\"exporter:9105\"]\n" + }, + "id": "swarm/prometheus-config", + "immutable": false, + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": {}, + "name": "prometheus-config", + "namespace": "swarm", + "resource_version": "477", + "uid": "07c719f5-19df-4761-af3e-6c847ea1fbc7" + } + ] + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "v1", + "kind": "ConfigMap", + "name": "prometheus-config", + "namespace": "swarm" + }, + "private": "bnVsbA==", + "dependencies": [ + "kubernetes_namespace.swarm" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_deployment", + "name": "explorer", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "id": "swarm/explorer", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 2, + "labels": { + "app.kubernetes.io/part-of": "swarm-sim" + }, + "name": "explorer", + "namespace": "swarm", + "resource_version": "6307", + "uid": "b20f626f-fe4d-4052-9d65-ed938b359ad5" + } + ], + "spec": [ + { + "min_ready_seconds": 0, + "paused": false, + "progress_deadline_seconds": 600, + "replicas": "1", + "revision_history_limit": 10, + "selector": [ + { + "match_expressions": [], + "match_labels": { + "app": "explorer" + } + } + ], + "strategy": [ + { + "rolling_update": [ + { + "max_surge": "25%", + "max_unavailable": "25%" + } + ], + "type": "RollingUpdate" + } + ], + "template": [ + { + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": { + "app": "explorer", + "app.kubernetes.io/part-of": "swarm-sim" + }, + "name": "", + "namespace": "", + "resource_version": "", + "uid": "" + } + ], + "spec": [ + { + "active_deadline_seconds": 0, + "affinity": [], + "automount_service_account_token": true, + "container": [ + { + "args": [], + "command": [ + "python", + "explorer/server.py" + ], + "env": [ + { + "name": "DATA_DIR", + "value": "/data", + "value_from": [] + }, + { + "name": "EXPLORER_PORT", + "value": "8088", + "value_from": [] + } + ], + "env_from": [], + "image": "swarm-house/simulator:dev", + "image_pull_policy": "Never", + "lifecycle": [], + "liveness_probe": [], + "name": "explorer", + "port": [ + { + "container_port": 8088, + "host_ip": "", + "host_port": 0, + "name": "", + "protocol": "TCP" + } + ], + "readiness_probe": [], + "resources": [ + { + "limits": {}, + "requests": {} + } + ], + "security_context": [], + "startup_probe": [], + "stdin": false, + "stdin_once": false, + "termination_message_path": "/dev/termination-log", + "termination_message_policy": "File", + "tty": false, + "volume_device": [], + "volume_mount": [ + { + "mount_path": "/data", + "mount_propagation": "None", + "name": "lake", + "read_only": true, + "sub_path": "", + "sub_path_expr": "" + } + ], + "working_dir": "" + } + ], + "dns_config": [], + "dns_policy": "ClusterFirst", + "enable_service_links": true, + "host_aliases": [], + "host_ipc": false, + "host_network": false, + "host_pid": false, + "hostname": "", + "image_pull_secrets": [], + "init_container": [], + "node_name": "", + "node_selector": {}, + "os": [], + "priority_class_name": "", + "readiness_gate": [], + "restart_policy": "Always", + "runtime_class_name": "", + "scheduler_name": "default-scheduler", + "security_context": [], + "service_account_name": "", + "share_process_namespace": false, + "subdomain": "", + "termination_grace_period_seconds": 30, + "toleration": [], + "topology_spread_constraint": [], + "volume": [ + { + "aws_elastic_block_store": [], + "azure_disk": [], + "azure_file": [], + "ceph_fs": [], + "cinder": [], + "config_map": [], + "csi": [], + "downward_api": [], + "empty_dir": [], + "ephemeral": [], + "fc": [], + "flex_volume": [], + "flocker": [], + "gce_persistent_disk": [], + "git_repo": [], + "glusterfs": [], + "host_path": [ + { + "path": "/data", + "type": "" + } + ], + "iscsi": [], + "local": [], + "name": "lake", + "nfs": [], + "persistent_volume_claim": [], + "photon_persistent_disk": [], + "projected": [], + "quobyte": [], + "rbd": [], + "secret": [], + "vsphere_volume": [] + } + ] + } + ] + } + ] + } + ], + "timeouts": null, + "wait_for_rollout": true + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "apps/v1", + "kind": "Deployment", + "name": "explorer", + "namespace": "swarm" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "kubernetes_namespace.swarm" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_deployment", + "name": "exporter", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "id": "swarm/exporter", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 2, + "labels": { + "app.kubernetes.io/part-of": "swarm-sim" + }, + "name": "exporter", + "namespace": "swarm", + "resource_version": "6311", + "uid": "60740b0c-e61d-4577-878e-929ee27ce6d7" + } + ], + "spec": [ + { + "min_ready_seconds": 0, + "paused": false, + "progress_deadline_seconds": 600, + "replicas": "1", + "revision_history_limit": 10, + "selector": [ + { + "match_expressions": [], + "match_labels": { + "app": "exporter" + } + } + ], + "strategy": [ + { + "rolling_update": [ + { + "max_surge": "25%", + "max_unavailable": "25%" + } + ], + "type": "RollingUpdate" + } + ], + "template": [ + { + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": { + "app": "exporter", + "app.kubernetes.io/part-of": "swarm-sim" + }, + "name": "", + "namespace": "", + "resource_version": "", + "uid": "" + } + ], + "spec": [ + { + "active_deadline_seconds": 0, + "affinity": [], + "automount_service_account_token": true, + "container": [ + { + "args": [], + "command": [ + "python", + "monitoring/exporter.py" + ], + "env": [ + { + "name": "DATA_DIR", + "value": "/data", + "value_from": [] + }, + { + "name": "EXPORTER_PORT", + "value": "9105", + "value_from": [] + } + ], + "env_from": [], + "image": "swarm-house/simulator:dev", + "image_pull_policy": "Never", + "lifecycle": [], + "liveness_probe": [], + "name": "exporter", + "port": [ + { + "container_port": 9105, + "host_ip": "", + "host_port": 0, + "name": "", + "protocol": "TCP" + } + ], + "readiness_probe": [], + "resources": [ + { + "limits": {}, + "requests": {} + } + ], + "security_context": [], + "startup_probe": [], + "stdin": false, + "stdin_once": false, + "termination_message_path": "/dev/termination-log", + "termination_message_policy": "File", + "tty": false, + "volume_device": [], + "volume_mount": [ + { + "mount_path": "/data", + "mount_propagation": "None", + "name": "lake", + "read_only": true, + "sub_path": "", + "sub_path_expr": "" + } + ], + "working_dir": "" + } + ], + "dns_config": [], + "dns_policy": "ClusterFirst", + "enable_service_links": true, + "host_aliases": [], + "host_ipc": false, + "host_network": false, + "host_pid": false, + "hostname": "", + "image_pull_secrets": [], + "init_container": [], + "node_name": "", + "node_selector": {}, + "os": [], + "priority_class_name": "", + "readiness_gate": [], + "restart_policy": "Always", + "runtime_class_name": "", + "scheduler_name": "default-scheduler", + "security_context": [], + "service_account_name": "", + "share_process_namespace": false, + "subdomain": "", + "termination_grace_period_seconds": 30, + "toleration": [], + "topology_spread_constraint": [], + "volume": [ + { + "aws_elastic_block_store": [], + "azure_disk": [], + "azure_file": [], + "ceph_fs": [], + "cinder": [], + "config_map": [], + "csi": [], + "downward_api": [], + "empty_dir": [], + "ephemeral": [], + "fc": [], + "flex_volume": [], + "flocker": [], + "gce_persistent_disk": [], + "git_repo": [], + "glusterfs": [], + "host_path": [ + { + "path": "/data", + "type": "" + } + ], + "iscsi": [], + "local": [], + "name": "lake", + "nfs": [], + "persistent_volume_claim": [], + "photon_persistent_disk": [], + "projected": [], + "quobyte": [], + "rbd": [], + "secret": [], + "vsphere_volume": [] + } + ] + } + ] + } + ] + } + ], + "timeouts": null, + "wait_for_rollout": true + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "apps/v1", + "kind": "Deployment", + "name": "exporter", + "namespace": "swarm" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "kubernetes_namespace.swarm" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_deployment", + "name": "grafana", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "id": "swarm/grafana", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 1, + "labels": { + "app.kubernetes.io/part-of": "swarm-sim" + }, + "name": "grafana", + "namespace": "swarm", + "resource_version": "665", + "uid": "32db5903-64e3-45aa-a6a2-432004b59cd9" + } + ], + "spec": [ + { + "min_ready_seconds": 0, + "paused": false, + "progress_deadline_seconds": 600, + "replicas": "1", + "revision_history_limit": 10, + "selector": [ + { + "match_expressions": [], + "match_labels": { + "app": "grafana" + } + } + ], + "strategy": [ + { + "rolling_update": [ + { + "max_surge": "25%", + "max_unavailable": "25%" + } + ], + "type": "RollingUpdate" + } + ], + "template": [ + { + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": { + "app": "grafana", + "app.kubernetes.io/part-of": "swarm-sim" + }, + "name": "", + "namespace": "", + "resource_version": "", + "uid": "" + } + ], + "spec": [ + { + "active_deadline_seconds": 0, + "affinity": [], + "automount_service_account_token": true, + "container": [ + { + "args": [], + "command": [], + "env": [ + { + "name": "GF_AUTH_ANONYMOUS_ENABLED", + "value": "true", + "value_from": [] + }, + { + "name": "GF_AUTH_ANONYMOUS_ORG_ROLE", + "value": "Admin", + "value_from": [] + }, + { + "name": "GF_AUTH_DISABLE_LOGIN_FORM", + "value": "true", + "value_from": [] + }, + { + "name": "GF_ANALYTICS_REPORTING_ENABLED", + "value": "false", + "value_from": [] + }, + { + "name": "GF_ANALYTICS_CHECK_FOR_UPDATES", + "value": "false", + "value_from": [] + }, + { + "name": "GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES", + "value": "false", + "value_from": [] + } + ], + "env_from": [], + "image": "grafana/grafana:11.1.0", + "image_pull_policy": "IfNotPresent", + "lifecycle": [], + "liveness_probe": [], + "name": "grafana", + "port": [ + { + "container_port": 3000, + "host_ip": "", + "host_port": 0, + "name": "", + "protocol": "TCP" + } + ], + "readiness_probe": [], + "resources": [ + { + "limits": {}, + "requests": {} + } + ], + "security_context": [], + "startup_probe": [], + "stdin": false, + "stdin_once": false, + "termination_message_path": "/dev/termination-log", + "termination_message_policy": "File", + "tty": false, + "volume_device": [], + "volume_mount": [ + { + "mount_path": "/etc/grafana/provisioning/datasources", + "mount_propagation": "None", + "name": "provisioning-datasources", + "read_only": false, + "sub_path": "", + "sub_path_expr": "" + }, + { + "mount_path": "/etc/grafana/provisioning/dashboards", + "mount_propagation": "None", + "name": "provisioning-dashboards", + "read_only": false, + "sub_path": "", + "sub_path_expr": "" + }, + { + "mount_path": "/var/lib/grafana/dashboards", + "mount_propagation": "None", + "name": "dashboards", + "read_only": false, + "sub_path": "", + "sub_path_expr": "" + } + ], + "working_dir": "" + } + ], + "dns_config": [], + "dns_policy": "ClusterFirst", + "enable_service_links": true, + "host_aliases": [], + "host_ipc": false, + "host_network": false, + "host_pid": false, + "hostname": "", + "image_pull_secrets": [], + "init_container": [], + "node_name": "", + "node_selector": {}, + "os": [], + "priority_class_name": "", + "readiness_gate": [], + "restart_policy": "Always", + "runtime_class_name": "", + "scheduler_name": "default-scheduler", + "security_context": [], + "service_account_name": "", + "share_process_namespace": false, + "subdomain": "", + "termination_grace_period_seconds": 30, + "toleration": [], + "topology_spread_constraint": [], + "volume": [ + { + "aws_elastic_block_store": [], + "azure_disk": [], + "azure_file": [], + "ceph_fs": [], + "cinder": [], + "config_map": [ + { + "default_mode": "0644", + "items": [ + { + "key": "datasource.yml", + "mode": "", + "path": "datasource.yml" + } + ], + "name": "grafana-provisioning", + "optional": false + } + ], + "csi": [], + "downward_api": [], + "empty_dir": [], + "ephemeral": [], + "fc": [], + "flex_volume": [], + "flocker": [], + "gce_persistent_disk": [], + "git_repo": [], + "glusterfs": [], + "host_path": [], + "iscsi": [], + "local": [], + "name": "provisioning-datasources", + "nfs": [], + "persistent_volume_claim": [], + "photon_persistent_disk": [], + "projected": [], + "quobyte": [], + "rbd": [], + "secret": [], + "vsphere_volume": [] + }, + { + "aws_elastic_block_store": [], + "azure_disk": [], + "azure_file": [], + "ceph_fs": [], + "cinder": [], + "config_map": [ + { + "default_mode": "0644", + "items": [ + { + "key": "dashboards.yml", + "mode": "", + "path": "dashboards.yml" + } + ], + "name": "grafana-provisioning", + "optional": false + } + ], + "csi": [], + "downward_api": [], + "empty_dir": [], + "ephemeral": [], + "fc": [], + "flex_volume": [], + "flocker": [], + "gce_persistent_disk": [], + "git_repo": [], + "glusterfs": [], + "host_path": [], + "iscsi": [], + "local": [], + "name": "provisioning-dashboards", + "nfs": [], + "persistent_volume_claim": [], + "photon_persistent_disk": [], + "projected": [], + "quobyte": [], + "rbd": [], + "secret": [], + "vsphere_volume": [] + }, + { + "aws_elastic_block_store": [], + "azure_disk": [], + "azure_file": [], + "ceph_fs": [], + "cinder": [], + "config_map": [ + { + "default_mode": "0644", + "items": [], + "name": "grafana-dashboard", + "optional": false + } + ], + "csi": [], + "downward_api": [], + "empty_dir": [], + "ephemeral": [], + "fc": [], + "flex_volume": [], + "flocker": [], + "gce_persistent_disk": [], + "git_repo": [], + "glusterfs": [], + "host_path": [], + "iscsi": [], + "local": [], + "name": "dashboards", + "nfs": [], + "persistent_volume_claim": [], + "photon_persistent_disk": [], + "projected": [], + "quobyte": [], + "rbd": [], + "secret": [], + "vsphere_volume": [] + } + ] + } + ] + } + ] + } + ], + "timeouts": null, + "wait_for_rollout": true + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "apps/v1", + "kind": "Deployment", + "name": "grafana", + "namespace": "swarm" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "kubernetes_config_map.grafana_dashboard", + "kubernetes_config_map.grafana_provisioning", + "kubernetes_namespace.swarm" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_deployment", + "name": "prometheus", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "id": "swarm/prometheus", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 1, + "labels": { + "app.kubernetes.io/part-of": "swarm-sim" + }, + "name": "prometheus", + "namespace": "swarm", + "resource_version": "653", + "uid": "2b48ee10-c5fa-422a-9843-2ed54be51015" + } + ], + "spec": [ + { + "min_ready_seconds": 0, + "paused": false, + "progress_deadline_seconds": 600, + "replicas": "1", + "revision_history_limit": 10, + "selector": [ + { + "match_expressions": [], + "match_labels": { + "app": "prometheus" + } + } + ], + "strategy": [ + { + "rolling_update": [ + { + "max_surge": "25%", + "max_unavailable": "25%" + } + ], + "type": "RollingUpdate" + } + ], + "template": [ + { + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": { + "app": "prometheus", + "app.kubernetes.io/part-of": "swarm-sim" + }, + "name": "", + "namespace": "", + "resource_version": "", + "uid": "" + } + ], + "spec": [ + { + "active_deadline_seconds": 0, + "affinity": [], + "automount_service_account_token": true, + "container": [ + { + "args": [], + "command": [], + "env": [], + "env_from": [], + "image": "prom/prometheus:v2.53.0", + "image_pull_policy": "IfNotPresent", + "lifecycle": [], + "liveness_probe": [], + "name": "prometheus", + "port": [ + { + "container_port": 9090, + "host_ip": "", + "host_port": 0, + "name": "", + "protocol": "TCP" + } + ], + "readiness_probe": [], + "resources": [ + { + "limits": {}, + "requests": {} + } + ], + "security_context": [], + "startup_probe": [], + "stdin": false, + "stdin_once": false, + "termination_message_path": "/dev/termination-log", + "termination_message_policy": "File", + "tty": false, + "volume_device": [], + "volume_mount": [ + { + "mount_path": "/etc/prometheus", + "mount_propagation": "None", + "name": "config", + "read_only": false, + "sub_path": "", + "sub_path_expr": "" + } + ], + "working_dir": "" + } + ], + "dns_config": [], + "dns_policy": "ClusterFirst", + "enable_service_links": true, + "host_aliases": [], + "host_ipc": false, + "host_network": false, + "host_pid": false, + "hostname": "", + "image_pull_secrets": [], + "init_container": [], + "node_name": "", + "node_selector": {}, + "os": [], + "priority_class_name": "", + "readiness_gate": [], + "restart_policy": "Always", + "runtime_class_name": "", + "scheduler_name": "default-scheduler", + "security_context": [], + "service_account_name": "", + "share_process_namespace": false, + "subdomain": "", + "termination_grace_period_seconds": 30, + "toleration": [], + "topology_spread_constraint": [], + "volume": [ + { + "aws_elastic_block_store": [], + "azure_disk": [], + "azure_file": [], + "ceph_fs": [], + "cinder": [], + "config_map": [ + { + "default_mode": "0644", + "items": [], + "name": "prometheus-config", + "optional": false + } + ], + "csi": [], + "downward_api": [], + "empty_dir": [], + "ephemeral": [], + "fc": [], + "flex_volume": [], + "flocker": [], + "gce_persistent_disk": [], + "git_repo": [], + "glusterfs": [], + "host_path": [], + "iscsi": [], + "local": [], + "name": "config", + "nfs": [], + "persistent_volume_claim": [], + "photon_persistent_disk": [], + "projected": [], + "quobyte": [], + "rbd": [], + "secret": [], + "vsphere_volume": [] + } + ] + } + ] + } + ] + } + ], + "timeouts": null, + "wait_for_rollout": true + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "apps/v1", + "kind": "Deployment", + "name": "prometheus", + "namespace": "swarm" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "kubernetes_config_map.prometheus", + "kubernetes_namespace.swarm" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_namespace", + "name": "swarm", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "swarm", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": { + "app.kubernetes.io/part-of": "swarm-sim" + }, + "name": "swarm", + "resource_version": "472", + "uid": "9d9c3037-f53f-496a-afea-9b3673909789" + } + ], + "timeouts": null, + "wait_for_default_service_account": false + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "v1", + "kind": "Namespace", + "name": "swarm" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjozMDAwMDAwMDAwMDB9fQ==" + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_service", + "name": "explorer", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "id": "swarm/explorer", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": {}, + "name": "explorer", + "namespace": "swarm", + "resource_version": "490", + "uid": "ffd1b5de-cb46-4f2d-9efb-557e3cdbcf2c" + } + ], + "spec": [ + { + "allocate_load_balancer_node_ports": true, + "cluster_ip": "10.43.195.143", + "cluster_ips": [ + "10.43.195.143" + ], + "external_ips": [], + "external_name": "", + "external_traffic_policy": "Cluster", + "health_check_node_port": 0, + "internal_traffic_policy": "Cluster", + "ip_families": [ + "IPv4" + ], + "ip_family_policy": "SingleStack", + "load_balancer_class": "", + "load_balancer_ip": "", + "load_balancer_source_ranges": [], + "port": [ + { + "app_protocol": "", + "name": "", + "node_port": 30088, + "port": 8088, + "protocol": "TCP", + "target_port": "8088" + } + ], + "publish_not_ready_addresses": false, + "selector": { + "app": "explorer" + }, + "session_affinity": "None", + "session_affinity_config": [], + "type": "NodePort" + } + ], + "status": [ + { + "load_balancer": [ + { + "ingress": [] + } + ] + } + ], + "timeouts": null, + "wait_for_load_balancer": true + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "v1", + "kind": "Service", + "name": "explorer", + "namespace": "swarm" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "kubernetes_namespace.swarm" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_service", + "name": "exporter", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "id": "swarm/exporter", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": {}, + "name": "exporter", + "namespace": "swarm", + "resource_version": "482", + "uid": "3d541a97-c8db-4d3f-9713-8f000eefbc99" + } + ], + "spec": [ + { + "allocate_load_balancer_node_ports": true, + "cluster_ip": "10.43.102.73", + "cluster_ips": [ + "10.43.102.73" + ], + "external_ips": [], + "external_name": "", + "external_traffic_policy": "", + "health_check_node_port": 0, + "internal_traffic_policy": "Cluster", + "ip_families": [ + "IPv4" + ], + "ip_family_policy": "SingleStack", + "load_balancer_class": "", + "load_balancer_ip": "", + "load_balancer_source_ranges": [], + "port": [ + { + "app_protocol": "", + "name": "", + "node_port": 0, + "port": 9105, + "protocol": "TCP", + "target_port": "9105" + } + ], + "publish_not_ready_addresses": false, + "selector": { + "app": "exporter" + }, + "session_affinity": "None", + "session_affinity_config": [], + "type": "ClusterIP" + } + ], + "status": [ + { + "load_balancer": [ + { + "ingress": [] + } + ] + } + ], + "timeouts": null, + "wait_for_load_balancer": true + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "v1", + "kind": "Service", + "name": "exporter", + "namespace": "swarm" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "kubernetes_namespace.swarm" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_service", + "name": "grafana", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "id": "swarm/grafana", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": {}, + "name": "grafana", + "namespace": "swarm", + "resource_version": "492", + "uid": "e97550f8-5a8b-4cd1-aef0-1b3455b463d6" + } + ], + "spec": [ + { + "allocate_load_balancer_node_ports": true, + "cluster_ip": "10.43.215.97", + "cluster_ips": [ + "10.43.215.97" + ], + "external_ips": [], + "external_name": "", + "external_traffic_policy": "Cluster", + "health_check_node_port": 0, + "internal_traffic_policy": "Cluster", + "ip_families": [ + "IPv4" + ], + "ip_family_policy": "SingleStack", + "load_balancer_class": "", + "load_balancer_ip": "", + "load_balancer_source_ranges": [], + "port": [ + { + "app_protocol": "", + "name": "", + "node_port": 30300, + "port": 3000, + "protocol": "TCP", + "target_port": "3000" + } + ], + "publish_not_ready_addresses": false, + "selector": { + "app": "grafana" + }, + "session_affinity": "None", + "session_affinity_config": [], + "type": "NodePort" + } + ], + "status": [ + { + "load_balancer": [ + { + "ingress": [] + } + ] + } + ], + "timeouts": null, + "wait_for_load_balancer": true + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "v1", + "kind": "Service", + "name": "grafana", + "namespace": "swarm" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "kubernetes_namespace.swarm" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_service", + "name": "prometheus", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "id": "swarm/prometheus", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": {}, + "name": "prometheus", + "namespace": "swarm", + "resource_version": "481", + "uid": "52e5a8d4-deb4-4054-a335-7a84d41a620a" + } + ], + "spec": [ + { + "allocate_load_balancer_node_ports": true, + "cluster_ip": "10.43.224.73", + "cluster_ips": [ + "10.43.224.73" + ], + "external_ips": [], + "external_name": "", + "external_traffic_policy": "Cluster", + "health_check_node_port": 0, + "internal_traffic_policy": "Cluster", + "ip_families": [ + "IPv4" + ], + "ip_family_policy": "SingleStack", + "load_balancer_class": "", + "load_balancer_ip": "", + "load_balancer_source_ranges": [], + "port": [ + { + "app_protocol": "", + "name": "", + "node_port": 30990, + "port": 9090, + "protocol": "TCP", + "target_port": "9090" + } + ], + "publish_not_ready_addresses": false, + "selector": { + "app": "prometheus" + }, + "session_affinity": "None", + "session_affinity_config": [], + "type": "NodePort" + } + ], + "status": [ + { + "load_balancer": [ + { + "ingress": [] + } + ] + } + ], + "timeouts": null, + "wait_for_load_balancer": true + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "v1", + "kind": "Service", + "name": "prometheus", + "namespace": "swarm" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "kubernetes_namespace.swarm" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_stateful_set", + "name": "drone", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "id": "swarm/drone", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 1, + "labels": { + "app.kubernetes.io/part-of": "swarm-sim" + }, + "name": "drone", + "namespace": "swarm", + "resource_version": "6209", + "uid": "ba8b7adf-7d13-4682-bce1-b77739f4e573" + } + ], + "spec": [ + { + "min_ready_seconds": 0, + "persistent_volume_claim_retention_policy": [ + { + "when_deleted": "Retain", + "when_scaled": "Retain" + } + ], + "pod_management_policy": "Parallel", + "replicas": "5", + "revision_history_limit": 0, + "selector": [ + { + "match_expressions": [], + "match_labels": { + "app": "drone" + } + } + ], + "service_name": "drone", + "template": [ + { + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": { + "app": "drone", + "app.kubernetes.io/part-of": "swarm-sim" + }, + "name": "", + "namespace": "", + "resource_version": "", + "uid": "" + } + ], + "spec": [ + { + "active_deadline_seconds": 0, + "affinity": [], + "automount_service_account_token": true, + "container": [ + { + "args": [], + "command": [], + "env": [ + { + "name": "DRONE_ID", + "value": "", + "value_from": [ + { + "config_map_key_ref": [], + "field_ref": [ + { + "api_version": "v1", + "field_path": "metadata.name" + } + ], + "resource_field_ref": [], + "secret_key_ref": [] + } + ] + }, + { + "name": "DURATION_S", + "value": "300", + "value_from": [] + }, + { + "name": "SPEEDUP", + "value": "2", + "value_from": [] + }, + { + "name": "DATA_DIR", + "value": "/data", + "value_from": [] + } + ], + "env_from": [], + "image": "swarm-house/simulator:dev", + "image_pull_policy": "Never", + "lifecycle": [], + "liveness_probe": [], + "name": "drone", + "port": [], + "readiness_probe": [], + "resources": [ + { + "limits": { + "cpu": "500m", + "memory": "512Mi" + }, + "requests": { + "cpu": "50m", + "memory": "128Mi" + } + } + ], + "security_context": [], + "startup_probe": [], + "stdin": false, + "stdin_once": false, + "termination_message_path": "/dev/termination-log", + "termination_message_policy": "File", + "tty": false, + "volume_device": [], + "volume_mount": [ + { + "mount_path": "/data", + "mount_propagation": "None", + "name": "lake", + "read_only": false, + "sub_path": "", + "sub_path_expr": "" + } + ], + "working_dir": "" + } + ], + "dns_config": [], + "dns_policy": "ClusterFirst", + "enable_service_links": true, + "host_aliases": [], + "host_ipc": false, + "host_network": false, + "host_pid": false, + "hostname": "", + "image_pull_secrets": [], + "init_container": [], + "node_name": "", + "node_selector": {}, + "os": [], + "priority_class_name": "", + "readiness_gate": [], + "restart_policy": "Always", + "runtime_class_name": "", + "scheduler_name": "default-scheduler", + "security_context": [], + "service_account_name": "", + "share_process_namespace": false, + "subdomain": "", + "termination_grace_period_seconds": 30, + "toleration": [], + "topology_spread_constraint": [], + "volume": [ + { + "aws_elastic_block_store": [], + "azure_disk": [], + "azure_file": [], + "ceph_fs": [], + "cinder": [], + "config_map": [], + "csi": [], + "downward_api": [], + "empty_dir": [], + "ephemeral": [], + "fc": [], + "flex_volume": [], + "flocker": [], + "gce_persistent_disk": [], + "git_repo": [], + "glusterfs": [], + "host_path": [ + { + "path": "/data", + "type": "" + } + ], + "iscsi": [], + "local": [], + "name": "lake", + "nfs": [], + "persistent_volume_claim": [], + "photon_persistent_disk": [], + "projected": [], + "quobyte": [], + "rbd": [], + "secret": [], + "vsphere_volume": [] + } + ] + } + ] + } + ], + "update_strategy": [], + "volume_claim_template": [] + } + ], + "timeouts": null, + "wait_for_rollout": true + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "apps/v1", + "kind": "StatefulSet", + "name": "drone", + "namespace": "swarm" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJyZWFkIjo2MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=", + "dependencies": [ + "kubernetes_namespace.swarm" + ] + } + ] + } + ], + "check_results": [ + { + "object_kind": "var", + "config_addr": "var.drone_count", + "status": "pass", + "objects": [ + { + "object_addr": "var.drone_count", + "status": "pass" + } + ] + } + ] +} diff --git a/infra/terraform/sim-env/terraform.tfstate.backup b/infra/terraform/sim-env/terraform.tfstate.backup new file mode 100644 index 0000000..0042e01 --- /dev/null +++ b/infra/terraform/sim-env/terraform.tfstate.backup @@ -0,0 +1,1763 @@ +{ + "version": 4, + "terraform_version": "1.15.6", + "serial": 16, + "lineage": "e4ea037b-4cc0-6b11-c7c6-7c3c1f968e04", + "outputs": { + "explorer_url": { + "value": "http://localhost:30088", + "type": "string" + }, + "fleet": { + "value": { + "drones": 5, + "flight_duration_s": 300, + "speedup": 2 + }, + "type": [ + "object", + { + "drones": "number", + "flight_duration_s": "number", + "speedup": "number" + } + ] + }, + "grafana_url": { + "value": "http://localhost:30300", + "type": "string" + }, + "prometheus_url": { + "value": "http://localhost:30990", + "type": "string" + } + }, + "resources": [ + { + "mode": "managed", + "type": "kubernetes_config_map", + "name": "grafana_dashboard", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "binary_data": {}, + "data": { + "swarm.json": "{\n \"uid\": \"swarm-fleet\",\n \"title\": \"Swarm Fleet — generated data\",\n \"tags\": [\"swarm\"],\n \"timezone\": \"utc\",\n \"schemaVersion\": 39,\n \"version\": 1,\n \"refresh\": \"5s\",\n \"time\": { \"from\": \"now-15m\", \"to\": \"now\" },\n \"panels\": [\n {\n \"id\": 1, \"type\": \"stat\", \"title\": \"Telemetry rows\",\n \"gridPos\": { \"h\": 5, \"w\": 4, \"x\": 0, \"y\": 0 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"swarm-prom\" },\n \"targets\": [{ \"expr\": \"sum(swarm_rows_total)\", \"instant\": true, \"refId\": \"A\" }],\n \"options\": { \"reduceOptions\": { \"calcs\": [\"lastNotNull\"] }, \"colorMode\": \"value\" },\n \"fieldConfig\": { \"defaults\": { \"unit\": \"short\", \"color\": { \"mode\": \"fixed\", \"fixedColor\": \"blue\" } }, \"overrides\": [] }\n },\n {\n \"id\": 2, \"type\": \"stat\", \"title\": \"Detections\",\n \"gridPos\": { \"h\": 5, \"w\": 4, \"x\": 4, \"y\": 0 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"swarm-prom\" },\n \"targets\": [{ \"expr\": \"sum(swarm_detections_total)\", \"instant\": true, \"refId\": \"A\" }],\n \"options\": { \"reduceOptions\": { \"calcs\": [\"lastNotNull\"] }, \"colorMode\": \"value\" },\n \"fieldConfig\": { \"defaults\": { \"unit\": \"short\", \"color\": { \"mode\": \"fixed\", \"fixedColor\": \"green\" } }, \"overrides\": [] }\n },\n {\n \"id\": 3, \"type\": \"stat\", \"title\": \"Pose frames (sent+received)\",\n \"gridPos\": { \"h\": 5, \"w\": 4, \"x\": 8, \"y\": 0 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"swarm-prom\" },\n \"targets\": [{ \"expr\": \"sum(swarm_state_frames_total)\", \"instant\": true, \"refId\": \"A\" }],\n \"options\": { \"reduceOptions\": { \"calcs\": [\"lastNotNull\"] }, \"colorMode\": \"value\" },\n \"fieldConfig\": { \"defaults\": { \"unit\": \"short\" }, \"overrides\": [] }\n },\n {\n \"id\": 4, \"type\": \"stat\", \"title\": \"Parquet on disk\",\n \"gridPos\": { \"h\": 5, \"w\": 4, \"x\": 12, \"y\": 0 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"swarm-prom\" },\n \"targets\": [{ \"expr\": \"sum(swarm_parquet_bytes)\", \"instant\": true, \"refId\": \"A\" }],\n \"options\": { \"reduceOptions\": { \"calcs\": [\"lastNotNull\"] }, \"colorMode\": \"value\" },\n \"fieldConfig\": { \"defaults\": { \"unit\": \"bytes\", \"color\": { \"mode\": \"fixed\", \"fixedColor\": \"purple\" } }, \"overrides\": [] }\n },\n {\n \"id\": 5, \"type\": \"stat\", \"title\": \"Parquet files\",\n \"gridPos\": { \"h\": 5, \"w\": 4, \"x\": 16, \"y\": 0 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"swarm-prom\" },\n \"targets\": [{ \"expr\": \"swarm_parquet_files\", \"instant\": true, \"refId\": \"A\" }],\n \"options\": { \"reduceOptions\": { \"calcs\": [\"lastNotNull\"] } },\n \"fieldConfig\": { \"defaults\": { \"unit\": \"short\" }, \"overrides\": [] }\n },\n {\n \"id\": 6, \"type\": \"gauge\", \"title\": \"Fleet battery (min)\",\n \"gridPos\": { \"h\": 5, \"w\": 4, \"x\": 20, \"y\": 0 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"swarm-prom\" },\n \"targets\": [{ \"expr\": \"min(swarm_battery_pct)\", \"instant\": true, \"refId\": \"A\" }],\n \"fieldConfig\": {\n \"defaults\": {\n \"unit\": \"percent\", \"min\": 0, \"max\": 100,\n \"thresholds\": { \"mode\": \"absolute\", \"steps\": [\n { \"color\": \"red\", \"value\": null }, { \"color\": \"yellow\", \"value\": 30 }, { \"color\": \"green\", \"value\": 60 }\n ] }\n }, \"overrides\": []\n }\n },\n {\n \"id\": 7, \"type\": \"timeseries\", \"title\": \"Telemetry rows by drone\",\n \"gridPos\": { \"h\": 8, \"w\": 12, \"x\": 0, \"y\": 5 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"swarm-prom\" },\n \"targets\": [{ \"expr\": \"sum by (drone) (swarm_rows_total)\", \"legendFormat\": \"{{drone}}\", \"refId\": \"A\" }],\n \"fieldConfig\": { \"defaults\": { \"unit\": \"short\", \"custom\": { \"fillOpacity\": 12 } }, \"overrides\": [] }\n },\n {\n \"id\": 8, \"type\": \"timeseries\", \"title\": \"Rows by sensor (fleet)\",\n \"gridPos\": { \"h\": 8, \"w\": 12, \"x\": 12, \"y\": 5 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"swarm-prom\" },\n \"targets\": [{ \"expr\": \"sum by (sensor) (swarm_rows_total)\", \"legendFormat\": \"{{sensor}}\", \"refId\": \"A\" }],\n \"fieldConfig\": { \"defaults\": { \"unit\": \"short\", \"custom\": { \"fillOpacity\": 12 } }, \"overrides\": [] }\n },\n {\n \"id\": 9, \"type\": \"timeseries\", \"title\": \"Battery per drone\",\n \"gridPos\": { \"h\": 8, \"w\": 8, \"x\": 0, \"y\": 13 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"swarm-prom\" },\n \"targets\": [{ \"expr\": \"swarm_battery_pct\", \"legendFormat\": \"{{drone}}\", \"refId\": \"A\" }],\n \"fieldConfig\": { \"defaults\": { \"unit\": \"percent\", \"min\": 0, \"max\": 100 }, \"overrides\": [] }\n },\n {\n \"id\": 10, \"type\": \"timeseries\", \"title\": \"RSSI per link\",\n \"gridPos\": { \"h\": 8, \"w\": 8, \"x\": 8, \"y\": 13 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"swarm-prom\" },\n \"targets\": [{ \"expr\": \"swarm_rssi_dbm\", \"legendFormat\": \"{{drone}} ← {{peer}}\", \"refId\": \"A\" }],\n \"fieldConfig\": { \"defaults\": { \"unit\": \"dBm\" }, \"overrides\": [] }\n },\n {\n \"id\": 11, \"type\": \"timeseries\", \"title\": \"Inter-drone distance\",\n \"gridPos\": { \"h\": 8, \"w\": 8, \"x\": 16, \"y\": 13 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"swarm-prom\" },\n \"targets\": [{ \"expr\": \"swarm_peer_distance_m\", \"legendFormat\": \"{{drone}} ↔ {{peer}}\", \"refId\": \"A\" }],\n \"fieldConfig\": { \"defaults\": { \"unit\": \"lengthm\" }, \"overrides\": [] }\n },\n {\n \"id\": 12, \"type\": \"bargauge\", \"title\": \"Detections by class\",\n \"gridPos\": { \"h\": 7, \"w\": 12, \"x\": 0, \"y\": 21 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"swarm-prom\" },\n \"targets\": [{ \"expr\": \"sum by (cls) (swarm_detections_total)\", \"legendFormat\": \"{{cls}}\", \"instant\": true, \"refId\": \"A\" }],\n \"options\": { \"displayMode\": \"gradient\", \"orientation\": \"horizontal\", \"reduceOptions\": { \"calcs\": [\"lastNotNull\"] } },\n \"fieldConfig\": { \"defaults\": { \"unit\": \"short\", \"color\": { \"mode\": \"continuous-GrYlRd\" } }, \"overrides\": [] }\n },\n {\n \"id\": 13, \"type\": \"timeseries\", \"title\": \"Pose frames by direction\",\n \"gridPos\": { \"h\": 7, \"w\": 12, \"x\": 12, \"y\": 21 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"swarm-prom\" },\n \"targets\": [{ \"expr\": \"sum by (drone, direction) (swarm_state_frames_total)\", \"legendFormat\": \"{{drone}} {{direction}}\", \"refId\": \"A\" }],\n \"fieldConfig\": { \"defaults\": { \"unit\": \"short\", \"custom\": { \"fillOpacity\": 8 } }, \"overrides\": [] }\n }\n ]\n}\n" + }, + "id": "swarm/grafana-dashboard", + "immutable": false, + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": {}, + "name": "grafana-dashboard", + "namespace": "swarm", + "resource_version": "476", + "uid": "31523ebd-1116-4e3e-a9a0-cdb01d4b8e37" + } + ] + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "v1", + "kind": "ConfigMap", + "name": "grafana-dashboard", + "namespace": "swarm" + }, + "private": "bnVsbA==", + "dependencies": [ + "kubernetes_namespace.swarm" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_config_map", + "name": "grafana_provisioning", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "binary_data": {}, + "data": { + "dashboards.yml": "apiVersion: 1\nproviders:\n - name: swarm\n folder: \"\"\n type: file\n options:\n path: /var/lib/grafana/dashboards\n", + "datasource.yml": "apiVersion: 1\ndatasources:\n - name: Prometheus\n type: prometheus\n access: proxy\n url: http://prometheus:9090\n isDefault: true\n" + }, + "id": "swarm/grafana-provisioning", + "immutable": false, + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": {}, + "name": "grafana-provisioning", + "namespace": "swarm", + "resource_version": "475", + "uid": "a26bceb4-8ae8-42c5-ba0b-99f2e66f8765" + } + ] + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "v1", + "kind": "ConfigMap", + "name": "grafana-provisioning", + "namespace": "swarm" + }, + "private": "bnVsbA==", + "dependencies": [ + "kubernetes_namespace.swarm" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_config_map", + "name": "prometheus", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "binary_data": {}, + "data": { + "prometheus.yml": "global:\n scrape_interval: 5s\nscrape_configs:\n - job_name: swarm\n static_configs:\n - targets: [\"exporter:9105\"]\n" + }, + "id": "swarm/prometheus-config", + "immutable": false, + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": {}, + "name": "prometheus-config", + "namespace": "swarm", + "resource_version": "477", + "uid": "07c719f5-19df-4761-af3e-6c847ea1fbc7" + } + ] + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "v1", + "kind": "ConfigMap", + "name": "prometheus-config", + "namespace": "swarm" + }, + "private": "bnVsbA==", + "dependencies": [ + "kubernetes_namespace.swarm" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_deployment", + "name": "explorer", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "id": "swarm/explorer", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 1, + "labels": { + "app.kubernetes.io/part-of": "swarm-sim" + }, + "name": "explorer", + "namespace": "swarm", + "resource_version": "5676", + "uid": "b20f626f-fe4d-4052-9d65-ed938b359ad5" + } + ], + "spec": [ + { + "min_ready_seconds": 0, + "paused": false, + "progress_deadline_seconds": 600, + "replicas": "1", + "revision_history_limit": 10, + "selector": [ + { + "match_expressions": [], + "match_labels": { + "app": "explorer" + } + } + ], + "strategy": [ + { + "rolling_update": [ + { + "max_surge": "25%", + "max_unavailable": "25%" + } + ], + "type": "RollingUpdate" + } + ], + "template": [ + { + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": { + "app": "explorer", + "app.kubernetes.io/part-of": "swarm-sim" + }, + "name": "", + "namespace": "", + "resource_version": "", + "uid": "" + } + ], + "spec": [ + { + "active_deadline_seconds": 0, + "affinity": [], + "automount_service_account_token": true, + "container": [ + { + "args": [], + "command": [ + "python", + "explorer/server.py" + ], + "env": [ + { + "name": "DATA_DIR", + "value": "/data", + "value_from": [] + } + ], + "env_from": [], + "image": "swarm-house/simulator:dev", + "image_pull_policy": "Never", + "lifecycle": [], + "liveness_probe": [], + "name": "explorer", + "port": [ + { + "container_port": 8088, + "host_ip": "", + "host_port": 0, + "name": "", + "protocol": "TCP" + } + ], + "readiness_probe": [], + "resources": [ + { + "limits": {}, + "requests": {} + } + ], + "security_context": [], + "startup_probe": [], + "stdin": false, + "stdin_once": false, + "termination_message_path": "/dev/termination-log", + "termination_message_policy": "File", + "tty": false, + "volume_device": [], + "volume_mount": [ + { + "mount_path": "/data", + "mount_propagation": "None", + "name": "lake", + "read_only": true, + "sub_path": "", + "sub_path_expr": "" + } + ], + "working_dir": "" + } + ], + "dns_config": [], + "dns_policy": "ClusterFirst", + "enable_service_links": true, + "host_aliases": [], + "host_ipc": false, + "host_network": false, + "host_pid": false, + "hostname": "", + "image_pull_secrets": [], + "init_container": [], + "node_name": "", + "node_selector": {}, + "os": [], + "priority_class_name": "", + "readiness_gate": [], + "restart_policy": "Always", + "runtime_class_name": "", + "scheduler_name": "default-scheduler", + "security_context": [], + "service_account_name": "", + "share_process_namespace": false, + "subdomain": "", + "termination_grace_period_seconds": 30, + "toleration": [], + "topology_spread_constraint": [], + "volume": [ + { + "aws_elastic_block_store": [], + "azure_disk": [], + "azure_file": [], + "ceph_fs": [], + "cinder": [], + "config_map": [], + "csi": [], + "downward_api": [], + "empty_dir": [], + "ephemeral": [], + "fc": [], + "flex_volume": [], + "flocker": [], + "gce_persistent_disk": [], + "git_repo": [], + "glusterfs": [], + "host_path": [ + { + "path": "/data", + "type": "" + } + ], + "iscsi": [], + "local": [], + "name": "lake", + "nfs": [], + "persistent_volume_claim": [], + "photon_persistent_disk": [], + "projected": [], + "quobyte": [], + "rbd": [], + "secret": [], + "vsphere_volume": [] + } + ] + } + ] + } + ] + } + ], + "timeouts": null, + "wait_for_rollout": null + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "apps/v1", + "kind": "Deployment", + "name": "explorer", + "namespace": "swarm" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9" + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_deployment", + "name": "exporter", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "id": "swarm/exporter", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 1, + "labels": { + "app.kubernetes.io/part-of": "swarm-sim" + }, + "name": "exporter", + "namespace": "swarm", + "resource_version": "5700", + "uid": "60740b0c-e61d-4577-878e-929ee27ce6d7" + } + ], + "spec": [ + { + "min_ready_seconds": 0, + "paused": false, + "progress_deadline_seconds": 600, + "replicas": "1", + "revision_history_limit": 10, + "selector": [ + { + "match_expressions": [], + "match_labels": { + "app": "exporter" + } + } + ], + "strategy": [ + { + "rolling_update": [ + { + "max_surge": "25%", + "max_unavailable": "25%" + } + ], + "type": "RollingUpdate" + } + ], + "template": [ + { + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": { + "app": "exporter", + "app.kubernetes.io/part-of": "swarm-sim" + }, + "name": "", + "namespace": "", + "resource_version": "", + "uid": "" + } + ], + "spec": [ + { + "active_deadline_seconds": 0, + "affinity": [], + "automount_service_account_token": true, + "container": [ + { + "args": [], + "command": [ + "python", + "monitoring/exporter.py" + ], + "env": [ + { + "name": "DATA_DIR", + "value": "/data", + "value_from": [] + } + ], + "env_from": [], + "image": "swarm-house/simulator:dev", + "image_pull_policy": "Never", + "lifecycle": [], + "liveness_probe": [], + "name": "exporter", + "port": [ + { + "container_port": 9105, + "host_ip": "", + "host_port": 0, + "name": "", + "protocol": "TCP" + } + ], + "readiness_probe": [], + "resources": [ + { + "limits": {}, + "requests": {} + } + ], + "security_context": [], + "startup_probe": [], + "stdin": false, + "stdin_once": false, + "termination_message_path": "/dev/termination-log", + "termination_message_policy": "File", + "tty": false, + "volume_device": [], + "volume_mount": [ + { + "mount_path": "/data", + "mount_propagation": "None", + "name": "lake", + "read_only": true, + "sub_path": "", + "sub_path_expr": "" + } + ], + "working_dir": "" + } + ], + "dns_config": [], + "dns_policy": "ClusterFirst", + "enable_service_links": true, + "host_aliases": [], + "host_ipc": false, + "host_network": false, + "host_pid": false, + "hostname": "", + "image_pull_secrets": [], + "init_container": [], + "node_name": "", + "node_selector": {}, + "os": [], + "priority_class_name": "", + "readiness_gate": [], + "restart_policy": "Always", + "runtime_class_name": "", + "scheduler_name": "default-scheduler", + "security_context": [], + "service_account_name": "", + "share_process_namespace": false, + "subdomain": "", + "termination_grace_period_seconds": 30, + "toleration": [], + "topology_spread_constraint": [], + "volume": [ + { + "aws_elastic_block_store": [], + "azure_disk": [], + "azure_file": [], + "ceph_fs": [], + "cinder": [], + "config_map": [], + "csi": [], + "downward_api": [], + "empty_dir": [], + "ephemeral": [], + "fc": [], + "flex_volume": [], + "flocker": [], + "gce_persistent_disk": [], + "git_repo": [], + "glusterfs": [], + "host_path": [ + { + "path": "/data", + "type": "" + } + ], + "iscsi": [], + "local": [], + "name": "lake", + "nfs": [], + "persistent_volume_claim": [], + "photon_persistent_disk": [], + "projected": [], + "quobyte": [], + "rbd": [], + "secret": [], + "vsphere_volume": [] + } + ] + } + ] + } + ] + } + ], + "timeouts": null, + "wait_for_rollout": null + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "apps/v1", + "kind": "Deployment", + "name": "exporter", + "namespace": "swarm" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9" + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_deployment", + "name": "grafana", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "id": "swarm/grafana", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 1, + "labels": { + "app.kubernetes.io/part-of": "swarm-sim" + }, + "name": "grafana", + "namespace": "swarm", + "resource_version": "665", + "uid": "32db5903-64e3-45aa-a6a2-432004b59cd9" + } + ], + "spec": [ + { + "min_ready_seconds": 0, + "paused": false, + "progress_deadline_seconds": 600, + "replicas": "1", + "revision_history_limit": 10, + "selector": [ + { + "match_expressions": [], + "match_labels": { + "app": "grafana" + } + } + ], + "strategy": [ + { + "rolling_update": [ + { + "max_surge": "25%", + "max_unavailable": "25%" + } + ], + "type": "RollingUpdate" + } + ], + "template": [ + { + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": { + "app": "grafana", + "app.kubernetes.io/part-of": "swarm-sim" + }, + "name": "", + "namespace": "", + "resource_version": "", + "uid": "" + } + ], + "spec": [ + { + "active_deadline_seconds": 0, + "affinity": [], + "automount_service_account_token": true, + "container": [ + { + "args": [], + "command": [], + "env": [ + { + "name": "GF_AUTH_ANONYMOUS_ENABLED", + "value": "true", + "value_from": [] + }, + { + "name": "GF_AUTH_ANONYMOUS_ORG_ROLE", + "value": "Admin", + "value_from": [] + }, + { + "name": "GF_AUTH_DISABLE_LOGIN_FORM", + "value": "true", + "value_from": [] + }, + { + "name": "GF_ANALYTICS_REPORTING_ENABLED", + "value": "false", + "value_from": [] + }, + { + "name": "GF_ANALYTICS_CHECK_FOR_UPDATES", + "value": "false", + "value_from": [] + }, + { + "name": "GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES", + "value": "false", + "value_from": [] + } + ], + "env_from": [], + "image": "grafana/grafana:11.1.0", + "image_pull_policy": "IfNotPresent", + "lifecycle": [], + "liveness_probe": [], + "name": "grafana", + "port": [ + { + "container_port": 3000, + "host_ip": "", + "host_port": 0, + "name": "", + "protocol": "TCP" + } + ], + "readiness_probe": [], + "resources": [ + { + "limits": {}, + "requests": {} + } + ], + "security_context": [], + "startup_probe": [], + "stdin": false, + "stdin_once": false, + "termination_message_path": "/dev/termination-log", + "termination_message_policy": "File", + "tty": false, + "volume_device": [], + "volume_mount": [ + { + "mount_path": "/etc/grafana/provisioning/datasources", + "mount_propagation": "None", + "name": "provisioning-datasources", + "read_only": false, + "sub_path": "", + "sub_path_expr": "" + }, + { + "mount_path": "/etc/grafana/provisioning/dashboards", + "mount_propagation": "None", + "name": "provisioning-dashboards", + "read_only": false, + "sub_path": "", + "sub_path_expr": "" + }, + { + "mount_path": "/var/lib/grafana/dashboards", + "mount_propagation": "None", + "name": "dashboards", + "read_only": false, + "sub_path": "", + "sub_path_expr": "" + } + ], + "working_dir": "" + } + ], + "dns_config": [], + "dns_policy": "ClusterFirst", + "enable_service_links": true, + "host_aliases": [], + "host_ipc": false, + "host_network": false, + "host_pid": false, + "hostname": "", + "image_pull_secrets": [], + "init_container": [], + "node_name": "", + "node_selector": {}, + "os": [], + "priority_class_name": "", + "readiness_gate": [], + "restart_policy": "Always", + "runtime_class_name": "", + "scheduler_name": "default-scheduler", + "security_context": [], + "service_account_name": "", + "share_process_namespace": false, + "subdomain": "", + "termination_grace_period_seconds": 30, + "toleration": [], + "topology_spread_constraint": [], + "volume": [ + { + "aws_elastic_block_store": [], + "azure_disk": [], + "azure_file": [], + "ceph_fs": [], + "cinder": [], + "config_map": [ + { + "default_mode": "0644", + "items": [ + { + "key": "datasource.yml", + "mode": "", + "path": "datasource.yml" + } + ], + "name": "grafana-provisioning", + "optional": false + } + ], + "csi": [], + "downward_api": [], + "empty_dir": [], + "ephemeral": [], + "fc": [], + "flex_volume": [], + "flocker": [], + "gce_persistent_disk": [], + "git_repo": [], + "glusterfs": [], + "host_path": [], + "iscsi": [], + "local": [], + "name": "provisioning-datasources", + "nfs": [], + "persistent_volume_claim": [], + "photon_persistent_disk": [], + "projected": [], + "quobyte": [], + "rbd": [], + "secret": [], + "vsphere_volume": [] + }, + { + "aws_elastic_block_store": [], + "azure_disk": [], + "azure_file": [], + "ceph_fs": [], + "cinder": [], + "config_map": [ + { + "default_mode": "0644", + "items": [ + { + "key": "dashboards.yml", + "mode": "", + "path": "dashboards.yml" + } + ], + "name": "grafana-provisioning", + "optional": false + } + ], + "csi": [], + "downward_api": [], + "empty_dir": [], + "ephemeral": [], + "fc": [], + "flex_volume": [], + "flocker": [], + "gce_persistent_disk": [], + "git_repo": [], + "glusterfs": [], + "host_path": [], + "iscsi": [], + "local": [], + "name": "provisioning-dashboards", + "nfs": [], + "persistent_volume_claim": [], + "photon_persistent_disk": [], + "projected": [], + "quobyte": [], + "rbd": [], + "secret": [], + "vsphere_volume": [] + }, + { + "aws_elastic_block_store": [], + "azure_disk": [], + "azure_file": [], + "ceph_fs": [], + "cinder": [], + "config_map": [ + { + "default_mode": "0644", + "items": [], + "name": "grafana-dashboard", + "optional": false + } + ], + "csi": [], + "downward_api": [], + "empty_dir": [], + "ephemeral": [], + "fc": [], + "flex_volume": [], + "flocker": [], + "gce_persistent_disk": [], + "git_repo": [], + "glusterfs": [], + "host_path": [], + "iscsi": [], + "local": [], + "name": "dashboards", + "nfs": [], + "persistent_volume_claim": [], + "photon_persistent_disk": [], + "projected": [], + "quobyte": [], + "rbd": [], + "secret": [], + "vsphere_volume": [] + } + ] + } + ] + } + ] + } + ], + "timeouts": null, + "wait_for_rollout": true + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "apps/v1", + "kind": "Deployment", + "name": "grafana", + "namespace": "swarm" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "kubernetes_config_map.grafana_dashboard", + "kubernetes_config_map.grafana_provisioning", + "kubernetes_namespace.swarm" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_deployment", + "name": "prometheus", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "id": "swarm/prometheus", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 1, + "labels": { + "app.kubernetes.io/part-of": "swarm-sim" + }, + "name": "prometheus", + "namespace": "swarm", + "resource_version": "653", + "uid": "2b48ee10-c5fa-422a-9843-2ed54be51015" + } + ], + "spec": [ + { + "min_ready_seconds": 0, + "paused": false, + "progress_deadline_seconds": 600, + "replicas": "1", + "revision_history_limit": 10, + "selector": [ + { + "match_expressions": [], + "match_labels": { + "app": "prometheus" + } + } + ], + "strategy": [ + { + "rolling_update": [ + { + "max_surge": "25%", + "max_unavailable": "25%" + } + ], + "type": "RollingUpdate" + } + ], + "template": [ + { + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": { + "app": "prometheus", + "app.kubernetes.io/part-of": "swarm-sim" + }, + "name": "", + "namespace": "", + "resource_version": "", + "uid": "" + } + ], + "spec": [ + { + "active_deadline_seconds": 0, + "affinity": [], + "automount_service_account_token": true, + "container": [ + { + "args": [], + "command": [], + "env": [], + "env_from": [], + "image": "prom/prometheus:v2.53.0", + "image_pull_policy": "IfNotPresent", + "lifecycle": [], + "liveness_probe": [], + "name": "prometheus", + "port": [ + { + "container_port": 9090, + "host_ip": "", + "host_port": 0, + "name": "", + "protocol": "TCP" + } + ], + "readiness_probe": [], + "resources": [ + { + "limits": {}, + "requests": {} + } + ], + "security_context": [], + "startup_probe": [], + "stdin": false, + "stdin_once": false, + "termination_message_path": "/dev/termination-log", + "termination_message_policy": "File", + "tty": false, + "volume_device": [], + "volume_mount": [ + { + "mount_path": "/etc/prometheus", + "mount_propagation": "None", + "name": "config", + "read_only": false, + "sub_path": "", + "sub_path_expr": "" + } + ], + "working_dir": "" + } + ], + "dns_config": [], + "dns_policy": "ClusterFirst", + "enable_service_links": true, + "host_aliases": [], + "host_ipc": false, + "host_network": false, + "host_pid": false, + "hostname": "", + "image_pull_secrets": [], + "init_container": [], + "node_name": "", + "node_selector": {}, + "os": [], + "priority_class_name": "", + "readiness_gate": [], + "restart_policy": "Always", + "runtime_class_name": "", + "scheduler_name": "default-scheduler", + "security_context": [], + "service_account_name": "", + "share_process_namespace": false, + "subdomain": "", + "termination_grace_period_seconds": 30, + "toleration": [], + "topology_spread_constraint": [], + "volume": [ + { + "aws_elastic_block_store": [], + "azure_disk": [], + "azure_file": [], + "ceph_fs": [], + "cinder": [], + "config_map": [ + { + "default_mode": "0644", + "items": [], + "name": "prometheus-config", + "optional": false + } + ], + "csi": [], + "downward_api": [], + "empty_dir": [], + "ephemeral": [], + "fc": [], + "flex_volume": [], + "flocker": [], + "gce_persistent_disk": [], + "git_repo": [], + "glusterfs": [], + "host_path": [], + "iscsi": [], + "local": [], + "name": "config", + "nfs": [], + "persistent_volume_claim": [], + "photon_persistent_disk": [], + "projected": [], + "quobyte": [], + "rbd": [], + "secret": [], + "vsphere_volume": [] + } + ] + } + ] + } + ] + } + ], + "timeouts": null, + "wait_for_rollout": true + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "apps/v1", + "kind": "Deployment", + "name": "prometheus", + "namespace": "swarm" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "kubernetes_config_map.prometheus", + "kubernetes_namespace.swarm" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_namespace", + "name": "swarm", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "swarm", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": { + "app.kubernetes.io/part-of": "swarm-sim" + }, + "name": "swarm", + "resource_version": "472", + "uid": "9d9c3037-f53f-496a-afea-9b3673909789" + } + ], + "timeouts": null, + "wait_for_default_service_account": false + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "v1", + "kind": "Namespace", + "name": "swarm" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjozMDAwMDAwMDAwMDB9fQ==" + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_service", + "name": "explorer", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "id": "swarm/explorer", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": {}, + "name": "explorer", + "namespace": "swarm", + "resource_version": "490", + "uid": "ffd1b5de-cb46-4f2d-9efb-557e3cdbcf2c" + } + ], + "spec": [ + { + "allocate_load_balancer_node_ports": true, + "cluster_ip": "10.43.195.143", + "cluster_ips": [ + "10.43.195.143" + ], + "external_ips": [], + "external_name": "", + "external_traffic_policy": "Cluster", + "health_check_node_port": 0, + "internal_traffic_policy": "Cluster", + "ip_families": [ + "IPv4" + ], + "ip_family_policy": "SingleStack", + "load_balancer_class": "", + "load_balancer_ip": "", + "load_balancer_source_ranges": [], + "port": [ + { + "app_protocol": "", + "name": "", + "node_port": 30088, + "port": 8088, + "protocol": "TCP", + "target_port": "8088" + } + ], + "publish_not_ready_addresses": false, + "selector": { + "app": "explorer" + }, + "session_affinity": "None", + "session_affinity_config": [], + "type": "NodePort" + } + ], + "status": [ + { + "load_balancer": [ + { + "ingress": [] + } + ] + } + ], + "timeouts": null, + "wait_for_load_balancer": true + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "v1", + "kind": "Service", + "name": "explorer", + "namespace": "swarm" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "kubernetes_namespace.swarm" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_service", + "name": "exporter", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "id": "swarm/exporter", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": {}, + "name": "exporter", + "namespace": "swarm", + "resource_version": "482", + "uid": "3d541a97-c8db-4d3f-9713-8f000eefbc99" + } + ], + "spec": [ + { + "allocate_load_balancer_node_ports": true, + "cluster_ip": "10.43.102.73", + "cluster_ips": [ + "10.43.102.73" + ], + "external_ips": [], + "external_name": "", + "external_traffic_policy": "", + "health_check_node_port": 0, + "internal_traffic_policy": "Cluster", + "ip_families": [ + "IPv4" + ], + "ip_family_policy": "SingleStack", + "load_balancer_class": "", + "load_balancer_ip": "", + "load_balancer_source_ranges": [], + "port": [ + { + "app_protocol": "", + "name": "", + "node_port": 0, + "port": 9105, + "protocol": "TCP", + "target_port": "9105" + } + ], + "publish_not_ready_addresses": false, + "selector": { + "app": "exporter" + }, + "session_affinity": "None", + "session_affinity_config": [], + "type": "ClusterIP" + } + ], + "status": [ + { + "load_balancer": [ + { + "ingress": [] + } + ] + } + ], + "timeouts": null, + "wait_for_load_balancer": true + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "v1", + "kind": "Service", + "name": "exporter", + "namespace": "swarm" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "kubernetes_namespace.swarm" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_service", + "name": "grafana", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "id": "swarm/grafana", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": {}, + "name": "grafana", + "namespace": "swarm", + "resource_version": "492", + "uid": "e97550f8-5a8b-4cd1-aef0-1b3455b463d6" + } + ], + "spec": [ + { + "allocate_load_balancer_node_ports": true, + "cluster_ip": "10.43.215.97", + "cluster_ips": [ + "10.43.215.97" + ], + "external_ips": [], + "external_name": "", + "external_traffic_policy": "Cluster", + "health_check_node_port": 0, + "internal_traffic_policy": "Cluster", + "ip_families": [ + "IPv4" + ], + "ip_family_policy": "SingleStack", + "load_balancer_class": "", + "load_balancer_ip": "", + "load_balancer_source_ranges": [], + "port": [ + { + "app_protocol": "", + "name": "", + "node_port": 30300, + "port": 3000, + "protocol": "TCP", + "target_port": "3000" + } + ], + "publish_not_ready_addresses": false, + "selector": { + "app": "grafana" + }, + "session_affinity": "None", + "session_affinity_config": [], + "type": "NodePort" + } + ], + "status": [ + { + "load_balancer": [ + { + "ingress": [] + } + ] + } + ], + "timeouts": null, + "wait_for_load_balancer": true + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "v1", + "kind": "Service", + "name": "grafana", + "namespace": "swarm" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "kubernetes_namespace.swarm" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_service", + "name": "prometheus", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "id": "swarm/prometheus", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": {}, + "name": "prometheus", + "namespace": "swarm", + "resource_version": "481", + "uid": "52e5a8d4-deb4-4054-a335-7a84d41a620a" + } + ], + "spec": [ + { + "allocate_load_balancer_node_ports": true, + "cluster_ip": "10.43.224.73", + "cluster_ips": [ + "10.43.224.73" + ], + "external_ips": [], + "external_name": "", + "external_traffic_policy": "Cluster", + "health_check_node_port": 0, + "internal_traffic_policy": "Cluster", + "ip_families": [ + "IPv4" + ], + "ip_family_policy": "SingleStack", + "load_balancer_class": "", + "load_balancer_ip": "", + "load_balancer_source_ranges": [], + "port": [ + { + "app_protocol": "", + "name": "", + "node_port": 30990, + "port": 9090, + "protocol": "TCP", + "target_port": "9090" + } + ], + "publish_not_ready_addresses": false, + "selector": { + "app": "prometheus" + }, + "session_affinity": "None", + "session_affinity_config": [], + "type": "NodePort" + } + ], + "status": [ + { + "load_balancer": [ + { + "ingress": [] + } + ] + } + ], + "timeouts": null, + "wait_for_load_balancer": true + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "v1", + "kind": "Service", + "name": "prometheus", + "namespace": "swarm" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "kubernetes_namespace.swarm" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_stateful_set", + "name": "drone", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "id": "swarm/drone", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 1, + "labels": { + "app.kubernetes.io/part-of": "swarm-sim" + }, + "name": "drone", + "namespace": "swarm", + "resource_version": "5923", + "uid": "ba8b7adf-7d13-4682-bce1-b77739f4e573" + } + ], + "spec": [ + { + "min_ready_seconds": 0, + "persistent_volume_claim_retention_policy": [ + { + "when_deleted": "Retain", + "when_scaled": "Retain" + } + ], + "pod_management_policy": "Parallel", + "replicas": "5", + "revision_history_limit": 0, + "selector": [ + { + "match_expressions": [], + "match_labels": { + "app": "drone" + } + } + ], + "service_name": "drone", + "template": [ + { + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": { + "app": "drone", + "app.kubernetes.io/part-of": "swarm-sim" + }, + "name": "", + "namespace": "", + "resource_version": "", + "uid": "" + } + ], + "spec": [ + { + "active_deadline_seconds": 0, + "affinity": [], + "automount_service_account_token": true, + "container": [ + { + "args": [], + "command": [], + "env": [ + { + "name": "DRONE_ID", + "value": "", + "value_from": [ + { + "config_map_key_ref": [], + "field_ref": [ + { + "api_version": "v1", + "field_path": "metadata.name" + } + ], + "resource_field_ref": [], + "secret_key_ref": [] + } + ] + }, + { + "name": "DURATION_S", + "value": "300", + "value_from": [] + }, + { + "name": "SPEEDUP", + "value": "2", + "value_from": [] + }, + { + "name": "DATA_DIR", + "value": "/data", + "value_from": [] + } + ], + "env_from": [], + "image": "swarm-house/simulator:dev", + "image_pull_policy": "Never", + "lifecycle": [], + "liveness_probe": [], + "name": "drone", + "port": [], + "readiness_probe": [], + "resources": [ + { + "limits": { + "cpu": "500m", + "memory": "512Mi" + }, + "requests": { + "cpu": "50m", + "memory": "128Mi" + } + } + ], + "security_context": [], + "startup_probe": [], + "stdin": false, + "stdin_once": false, + "termination_message_path": "/dev/termination-log", + "termination_message_policy": "File", + "tty": false, + "volume_device": [], + "volume_mount": [ + { + "mount_path": "/data", + "mount_propagation": "None", + "name": "lake", + "read_only": false, + "sub_path": "", + "sub_path_expr": "" + } + ], + "working_dir": "" + } + ], + "dns_config": [], + "dns_policy": "ClusterFirst", + "enable_service_links": true, + "host_aliases": [], + "host_ipc": false, + "host_network": false, + "host_pid": false, + "hostname": "", + "image_pull_secrets": [], + "init_container": [], + "node_name": "", + "node_selector": {}, + "os": [], + "priority_class_name": "", + "readiness_gate": [], + "restart_policy": "Always", + "runtime_class_name": "", + "scheduler_name": "default-scheduler", + "security_context": [], + "service_account_name": "", + "share_process_namespace": false, + "subdomain": "", + "termination_grace_period_seconds": 30, + "toleration": [], + "topology_spread_constraint": [], + "volume": [ + { + "aws_elastic_block_store": [], + "azure_disk": [], + "azure_file": [], + "ceph_fs": [], + "cinder": [], + "config_map": [], + "csi": [], + "downward_api": [], + "empty_dir": [], + "ephemeral": [], + "fc": [], + "flex_volume": [], + "flocker": [], + "gce_persistent_disk": [], + "git_repo": [], + "glusterfs": [], + "host_path": [ + { + "path": "/data", + "type": "" + } + ], + "iscsi": [], + "local": [], + "name": "lake", + "nfs": [], + "persistent_volume_claim": [], + "photon_persistent_disk": [], + "projected": [], + "quobyte": [], + "rbd": [], + "secret": [], + "vsphere_volume": [] + } + ] + } + ] + } + ], + "update_strategy": [], + "volume_claim_template": [] + } + ], + "timeouts": null, + "wait_for_rollout": true + }, + "sensitive_attributes": [], + "identity_schema_version": 1, + "identity": { + "api_version": "apps/v1", + "kind": "StatefulSet", + "name": "drone", + "namespace": "swarm" + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJyZWFkIjo2MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=", + "dependencies": [ + "kubernetes_namespace.swarm" + ] + } + ] + } + ], + "check_results": null +} diff --git a/infra/terraform/sim-env/variables.tf b/infra/terraform/sim-env/variables.tf new file mode 100644 index 0000000..a447d15 --- /dev/null +++ b/infra/terraform/sim-env/variables.tf @@ -0,0 +1,66 @@ +variable "kubeconfig" { + description = "Path to the kubeconfig for the simulation cluster" + type = string + default = "~/.kube/config" +} + +variable "kube_context" { + description = "Kubeconfig context of the k3d simulation cluster" + type = string + default = "k3d-swarm-sim" +} + +variable "namespace" { + description = "Namespace for the simulated fleet" + type = string + default = "swarm" +} + +variable "simulator_image" { + description = "Simulator image (imported into k3d by infra/ansible/sim-cluster.yml)" + type = string + default = "swarm-house/simulator:dev" +} + +variable "drone_count" { + description = "Number of virtual drones in the fleet" + type = number + default = 5 + + validation { + condition = var.drone_count >= 1 && var.drone_count <= 32 + error_message = "drone_count must be between 1 and 32." + } +} + +variable "flight_duration_s" { + description = "Simulated flight length in seconds; each pod restart begins a new flight" + type = number + default = 300 +} + +variable "speedup" { + description = "Wall-clock acceleration of the simulation" + type = number + default = 2 +} + +variable "data_host_path" { + description = "Host path inside the k3d node mounted as the shared Parquet lake" + type = string + default = "/data" +} + +variable "node_ports" { + description = "Host-reachable NodePorts (must match the ports opened by sim-cluster.yml)" + type = object({ + explorer = number + grafana = number + prometheus = number + }) + default = { + explorer = 30088 + grafana = 30300 + prometheus = 30990 + } +} diff --git a/infra/terraform/sim-env/versions.tf b/infra/terraform/sim-env/versions.tf new file mode 100644 index 0000000..18c36cc --- /dev/null +++ b/infra/terraform/sim-env/versions.tf @@ -0,0 +1,15 @@ +terraform { + required_version = ">= 1.5" + + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.33" + } + } +} + +provider "kubernetes" { + config_path = var.kubeconfig + config_context = var.kube_context +} diff --git a/prototype/src/App.tsx b/prototype/src/App.tsx index 24233fa..c6af2d0 100644 --- a/prototype/src/App.tsx +++ b/prototype/src/App.tsx @@ -1,4 +1,5 @@ import { useEffect, useMemo, useRef, useState } from "react"; +import { DEFAULT_EXPLORER_URL, fetchLivePoses, toLiveWorld } from "./live"; import { AREA, MAX_ASPECT, @@ -24,17 +25,22 @@ function displayAspect(): number { return Math.max(1, Math.min(MAX_ASPECT, q)); } +const LIVE_POLL_MS = 1000; + export default function App(): JSX.Element { const [droneCount, setDroneCount] = useState(8); const [speedup, setSpeedup] = useState(1); const [paused, setPaused] = useState(false); const [aspect, setAspect] = useState(displayAspect); + const [mode, setMode] = useState<"sim" | "live">("sim"); + const [liveError, setLiveError] = useState(null); const [world, setWorld] = useState(() => { seed(42); return makeWorld(8, displayAspect()); }); const raf = useRef(0); const last = useRef(performance.now()); + const liveWorld = useRef(null); useEffect(() => { seed(42); @@ -55,6 +61,7 @@ export default function App(): JSX.Element { }, []); useEffect(() => { + if (mode !== "sim") return; const loop = (now: number): void => { const dt = Math.min(0.05, (now - last.current) / 1000); last.current = now; @@ -65,7 +72,40 @@ export default function App(): JSX.Element { }; raf.current = requestAnimationFrame(loop); return () => cancelAnimationFrame(raf.current); - }, [paused, speedup]); + }, [paused, speedup, mode]); + + // Live mode: poll the explorer's read-only SQL API for real poses + useEffect(() => { + if (mode !== "live") return; + let cancelled = false; + const poll = async (): Promise => { + try { + const poses = await fetchLivePoses(DEFAULT_EXPLORER_URL); + if (cancelled) return; + setLiveError(poses.length === 0 ? "no drone data in the lake yet" : null); + liveWorld.current = toLiveWorld(poses, liveWorld.current, LIVE_POLL_MS / 1000); + setWorld(liveWorld.current); + } catch (err) { + if (!cancelled) setLiveError(err instanceof Error ? err.message : String(err)); + } + }; + void poll(); + const timer = window.setInterval(() => void poll(), LIVE_POLL_MS); + return () => { + cancelled = true; + window.clearInterval(timer); + }; + }, [mode]); + + useEffect(() => { + if (mode === "sim") { + liveWorld.current = null; + setLiveError(null); + seed(42); + setWorld(makeWorld(droneCount, aspect)); + } + // eslint-disable-next-line react-hooks/exhaustive-deps -- reset only on mode change + }, [mode]); const activeLinks = useMemo( () => [...world.links.values()].sort((a, b) => b.rate - a.rate), @@ -83,32 +123,48 @@ export default function App(): JSX.Element {
- - - + {mode === "live" && ( + + {liveError ?? `live · ${DEFAULT_EXPLORER_URL}`} + + )} + {mode === "sim" && ( + <> + + + + + )}
@@ -183,7 +239,7 @@ export default function App(): JSX.Element { > - dr-{String(d.id + 1).padStart(2, "0")} · ch{d.channel} · {d.battery.toFixed(0)}% + {d.name ?? `dr-${String(d.id + 1).padStart(2, "0")} · ch${d.channel}`} · {d.battery.toFixed(0)}% ))} @@ -201,7 +257,8 @@ export default function App(): JSX.Element { {busiest.map((l) => (
- dr-{String(l.a + 1).padStart(2, "0")} ↔ dr-{String(l.b + 1).padStart(2, "0")} + {world.drones[l.a]?.name ?? `dr-${String(l.a + 1).padStart(2, "0")}`} ↔{" "} + {world.drones[l.b]?.name ?? `dr-${String(l.b + 1).padStart(2, "0")}`} {fmtBytes(l.rate)}/s · ↑{fmtBytes(l.totalUp)} ↓{fmtBytes(l.totalDown)} @@ -297,6 +354,9 @@ const styles: Record = { fontSize: 12, cursor: "pointer", }, + buttonActive: { background: "#1a3a2a", color: "#39d98a", borderColor: "#2a5c3c" }, + liveOk: { fontSize: 11, color: "#39d98a" }, + liveError: { fontSize: 11, color: "#c05a6a" }, main: { display: "flex", flex: 1, gap: 0, minHeight: 0 }, canvas: { flex: 1, minWidth: 0, display: "block" }, panel: { diff --git a/prototype/src/live.ts b/prototype/src/live.ts new file mode 100644 index 0000000..152ad70 --- /dev/null +++ b/prototype/src/live.ts @@ -0,0 +1,129 @@ +// Live mode: real drone poses from the explorer's read-only SQL API. +// The same gated /api/query endpoint a peer drone (or an engineer on the +// bench) would use — the prototype is just another read-only consumer. + +import { AREA, LINK_RANGE, POSE_RATE_BYTES, type LinkStats, type World } from "./sim"; + +export const DEFAULT_EXPLORER_URL = + (import.meta.env.VITE_EXPLORER_URL as string | undefined) ?? "http://localhost:30088"; + +export interface LivePose { + id: string; + x: number; // meters, mission frame + y: number; + yawDeg: number; + battery: number | null; + tsNs: number; +} + +// Latest pose per drone from its own 'sent' state rows, joined with the +// latest battery reading from telemetry. arg_max keeps it a single scan. +const LIVE_SQL = ` +WITH pose AS ( + SELECT drone_id, + arg_max(pos_x, ts_ns) AS x, + arg_max(pos_y, ts_ns) AS y, + arg_max(yaw, ts_ns) AS yaw, + max(ts_ns) AS ts_ns + FROM state + WHERE direction = 'sent' + GROUP BY drone_id +), +batt AS ( + SELECT drone_id, arg_max(level_pct, ts_ns) AS battery + FROM telemetry + WHERE sensor = 'battery' + GROUP BY drone_id +) +SELECT p.drone_id, p.x, p.y, p.yaw, p.ts_ns, b.battery +FROM pose p LEFT JOIN batt b USING (drone_id) +ORDER BY p.drone_id`; + +interface QueryResponse { + columns?: string[]; + rows?: (string | number | null)[][]; + error?: string; +} + +export async function fetchLivePoses(baseUrl: string): Promise { + const res = await fetch(`${baseUrl.replace(/\/$/, "")}/api/query`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sql: LIVE_SQL }), + }); + if (!res.ok) throw new Error(`explorer responded ${res.status}`); + const data = (await res.json()) as QueryResponse; + if (data.error) throw new Error(data.error); + return (data.rows ?? []).map((r) => ({ + id: String(r[0]), + x: Number(r[1]), + y: Number(r[2]), + yawDeg: Number(r[3]), + tsNs: Number(r[4]), + battery: r[5] === null ? null : Number(r[5]), + })); +} + +// Scale mission-frame meters into the prototype's square view. The virtual +// drones patrol a smaller square than the sim world, so fit the observed +// extent (with padding) instead of assuming a size. +function fitScale(poses: LivePose[]): number { + const extent = Math.max(400, ...poses.map((p) => Math.max(p.x, p.y))) * 1.15; + return AREA / extent; +} + +function linkKey(a: number, b: number): string { + return a < b ? `${a}-${b}` : `${b}-${a}`; +} + +/** Convert live poses into the World shape the renderer already speaks. */ +export function toLiveWorld(poses: LivePose[], prev: World | null, dtS: number): World { + const k = fitScale(poses); + const drones = poses.map((p, i) => ({ + id: i, + name: p.id, + pos: { x: p.x * k, y: p.y * k }, + vel: { x: 0, y: 0 }, + heading: (p.yawDeg * Math.PI) / 180, + waypoint: 0, + battery: p.battery ?? 100, + channel: 0, + })); + + // Every in-range pair exchanges pose broadcasts; accumulate totals so the + // panel keeps its meaning between polls. + const links = new Map(); + let totalBytes = prev?.totalBytes ?? 0; + for (let i = 0; i < drones.length; i++) { + for (let j = i + 1; j < drones.length; j++) { + const dist = Math.hypot( + drones[i].pos.x - drones[j].pos.x, + drones[i].pos.y - drones[j].pos.y, + ); + if (dist > LINK_RANGE) continue; + const key = linkKey(i, j); + const old = prev?.links.get(key); + const bytes = POSE_RATE_BYTES * 2 * dtS; + totalBytes += bytes; + links.set(key, { + key, + a: i, + b: j, + totalUp: (old?.totalUp ?? 0) + bytes / 2, + totalDown: (old?.totalDown ?? 0) + bytes / 2, + rate: POSE_RATE_BYTES * 2, + flash: 0.6, + bulk: 0, + }); + } + } + + return { + t: (prev?.t ?? 0) + dtS, + drones, + obstacles: [], + links, + totalBytes, + broadcasts: (prev?.broadcasts ?? 0) + links.size * 2, + }; +} diff --git a/prototype/src/sim.ts b/prototype/src/sim.ts index 1970620..6e18131 100644 --- a/prototype/src/sim.ts +++ b/prototype/src/sim.ts @@ -9,6 +9,7 @@ export interface Vec { export interface Drone { id: number; + name?: string; // live mode: real drone_id from the data plane pos: Vec; vel: Vec; heading: number; // radians @@ -58,6 +59,8 @@ const CRUISE = 28; const SEPARATION = 55; const POSE_BYTES = 46; const POSE_HZ = 5; +// Steady per-link broadcast throughput (both directions), bytes/s +export const POSE_RATE_BYTES = POSE_BYTES * POSE_HZ; const CHANNELS = [1, 6, 11, 36, 40, 44, 149, 157]; let seedState = 1234; diff --git a/prototype/src/vite-env.d.ts b/prototype/src/vite-env.d.ts new file mode 100644 index 0000000..11f02fe --- /dev/null +++ b/prototype/src/vite-env.d.ts @@ -0,0 +1 @@ +/// diff --git a/simulator/README.md b/simulator/README.md index 99fa4ad..380754c 100644 --- a/simulator/README.md +++ b/simulator/README.md @@ -14,14 +14,16 @@ FLIGHT_ID=$(date -u +%Y%m%dT%H%MZ)-sim docker compose up --build --scale drone=5 DRONE_COUNT=10 DURATION_S=300 SPEEDUP=4 docker compose up --build --scale drone=10 ``` -Output lands in `./data/` with the standard layout: +Output lands in [`../var/t1/`](../var/t1/) (the shared T1 lake — same path the k3d fleet and warehouse offload use). Override with `SWARM_T1_DIR` if needed: ``` -data/dataset=telemetry/flight=…/drone=…/sensor=imu/year=…/…/hour=…/data.parquet -data/dataset=detections/flight=…/drone=…/… -data/dataset=state/flight=…/drone=…/… ← sent + received broadcasts +var/t1/dataset=telemetry/flight=…/drone=…/sensor=imu/year=…/…/hour=…/data.parquet +var/t1/dataset=detections/flight=…/drone=…/… +var/t1/dataset=state/flight=…/drone=…/… ← sent + received broadcasts ``` +Historical flights after offload live under [`../var/t3/`](../var/t3/) (T3 warehouse). + ## Run a single drone without Docker ```bash diff --git a/simulator/docker-compose.yml b/simulator/docker-compose.yml index e730eb8..38e6bb4 100644 --- a/simulator/docker-compose.yml +++ b/simulator/docker-compose.yml @@ -14,7 +14,7 @@ services: # Compose default bridge network: subnet-directed broadcast works BROADCAST_ADDR: ${BROADCAST_ADDR:-255.255.255.255} volumes: - - ./data:/data + - ${SWARM_T1_DIR:-../var/t1}:/data # --- Monitoring profile: docker compose --profile monitoring up --- @@ -26,7 +26,7 @@ services: DATA_DIR: /data SCAN_INTERVAL_S: "5" volumes: - - ./data:/data:ro + - ${SWARM_T1_DIR:-../var/t1}:/data:ro - ./monitoring:/app/monitoring:ro ports: - "${EXPORTER_PORT:-9105}:9105" @@ -38,7 +38,7 @@ services: environment: DATA_DIR: /data volumes: - - ./data:/data:ro + - ${SWARM_T1_DIR:-../var/t1}:/data:ro - ./explorer:/app/explorer:ro ports: - "${EXPLORER_UI_PORT:-8088}:8088" diff --git a/simulator/explorer/server.py b/simulator/explorer/server.py index 68a55ef..b649521 100644 --- a/simulator/explorer/server.py +++ b/simulator/explorer/server.py @@ -116,9 +116,19 @@ class Handler(BaseHTTPRequestHandler): self.send_response(code) self.send_header("Content-Type", ctype) self.send_header("Content-Length", str(len(body))) + # Dev CORS: lets the prototype's live mode poll the API from another + # origin. Everything behind this is read-only by construction. + self.send_header("Access-Control-Allow-Origin", "*") self.end_headers() self.wfile.write(body) + def do_OPTIONS(self) -> None: # noqa: N802 — http.server API + self.send_response(204) + self.send_header("Access-Control-Allow-Origin", "*") + self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS") + self.send_header("Access-Control-Allow-Headers", "Content-Type") + self.end_headers() + def _json(self, payload: dict, code: int = 200) -> None: self._send(code, json.dumps(payload, default=str).encode(), "application/json") diff --git a/var/t1/.gitkeep b/var/t1/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/var/t3/.gitkeep b/var/t3/.gitkeep new file mode 100644 index 0000000..e69de29