From 2a6b50929c80a911de99c0fea5b53deb7f43d58c Mon Sep 17 00:00:00 2001 From: Andriy Oblivantsev Date: Fri, 17 Jul 2026 12:16:39 +0100 Subject: [PATCH] docs: align wire-spec and mark implemented vs proposed Keep the 45-byte pose frame consistent across docs, simulator, and prototype, clarify MinIO is not peer sync, and state clearly that the repo is a from-scratch platform sketch with options rather than mandates. --- README.md | 19 ++++++++++++ docs/02-architecture.md | 19 +++++++----- docs/04-swarm-sync.md | 36 ++++++++++++++-------- docs/12-design-journey.md | 25 +++++++++------ docs/adr/ADR-0004-sql-over-ssh-contract.md | 15 ++++++--- prototype/src/App.tsx | 2 +- prototype/src/sim.ts | 2 +- simulator/tests/test_broadcast.py | 2 +- simulator/virtual_drone/broadcast.py | 7 +++-- 9 files changed, 85 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 7ca6989..11945e7 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,25 @@ This repository describes how to build, deliver, test, and operate that platform 7. **SQL is the contract.** Peer data access is read-only DuckDB SQL over SSH forced commands — the query language already lives on both ends, so no service, port, or protocol is invented for it. 8. **No debug API.** The bench and the flight use the same channel: an engineer debugging on the ground runs the identical query through the identical wrapper, permissions, and output format a peer drone would use. What you test is what flies. +None of the concrete tool picks above are mandates. This repo is a **from-scratch platform sketch**: enough structure to hire and build against, with every decision recorded so the team can replace a piece when a better fit appears. + +## Implemented now vs proposed next + +Honest map so a reader knows what runs today versus what is design intent. + +| Area | Implemented in this repo (runnable PoC) | Proposed for a production air-gapped fleet | +| --- | --- | --- | +| On-board layout | Hive-partitioned Parquet writer, seal step, DuckDB views | Same contract; Compose services under systemd | +| Pose path | Fixed **45-byte** UDP frame + 2D bandwidth visualisation | Zenoh pub/sub (UDP kept as degraded minimal profile) | +| Peer query | Read-only SQL **gate** over HTTP explorer (keyword allow-list) | Same gate idea via **SSH forced command** + OS/engine hardening | +| Bulk sync | Visualised opportunistic transfer volume | rsync/rclone over persistent SSH between peers | +| Mesh trust | Ansible templates for WireGuard + ed25519 forced commands | Provisioned per-device keys; nothing joins at runtime | +| Ground segment | k3d/Terraform sim: lake, Grafana, offload CronJob, optional MinIO | k3s warehouse, GitOps overlays, post-flight mirror | +| CI / delivery | GitHub Actions: pytest, smoke flight, Trivy, semver release + fleet manifest artifact | Self-hosted GitLab + registry inside the air gap (same stages) | +| Docs | Problem, architecture, ADRs, design journey, open questions | Living ADRs owned by the team | + +Start from the [design journey](docs/12-design-journey.md) for the story; use the table above when reviewing scope. + ## Documentation | Document | Contents | diff --git a/docs/02-architecture.md b/docs/02-architecture.md index c2aff81..29b5a2b 100644 --- a/docs/02-architecture.md +++ b/docs/02-architecture.md @@ -20,8 +20,9 @@ graph TB subgraph serving [Layer 3 — Serving and sync] HOOK["event hook
fires on new derived data"] PUB["state publisher
pub/sub broadcast"] - MINIO["MinIO
derived datasets bucket"] - QAPI["query API
peer data requests"] + BULK["bulk sync
rsync over SSH"] + MINIO["MinIO
optional derived bucket"] + QAPI["query API
SQL-over-SSH"] end end @@ -33,12 +34,14 @@ graph TB WRITER -->|derived rows| HOOK HOOK --> PUB HOOK --> MINIO + HOOK --> BULK QAPI --> DUCK - PUB -.->|mesh| PEERS["peer drones"] - MINIO -.->|replication| PEERS + PUB -.->|mesh pose| PEERS["peer drones"] + BULK -.->|sealed partitions| PEERS QAPI -.->|on demand| PEERS ``` +MinIO stays available where an S3 API helps (on-board derived datasets, ground warehouse). **In-flight peer bulk sync is SSH/rsync**, not object-store replication — see [04 — Swarm sync](04-swarm-sync.md). ### Layer 1 — Ingestion - `sensor-ingest` subscribes to sensor sources (ROS 2 topics where available, raw drivers otherwise) and normalizes them into typed streams: IMU, barometer, temperature, LiDAR, RSSI, power, and so on. @@ -53,10 +56,10 @@ graph TB ### Layer 3 — Serving and sync -- The **event hook** is the on-board "lambda": when the writer lands new *derived* rows (state, detections), it triggers registered actions — broadcast, MinIO upload, or a local mission-logic callback. Nothing polls. -- The **state publisher** broadcasts compact position/attitude/detection payloads over the mesh pub/sub (transport analysis in [04 — Swarm sync](04-swarm-sync.md)). -- **Bulk sync** pulls sealed derived partitions from peers over persistent SSH (rsync delta transfer); MinIO remains optional where an S3 API is wanted. -- **Peer queries** are read-only DuckDB SQL over SSH forced commands — SELECT-only gate, read-only OS user, columnar responses (the design decision and its reasoning are in [04](04-swarm-sync.md)). +- The **event hook** is the on-board "lambda": when the writer lands new *derived* rows (state, detections), it triggers registered actions — broadcast, optional MinIO put, bulk-sync hint, or a local mission-logic callback. Nothing polls. +- The **state publisher** broadcasts compact position/attitude/detection payloads over the mesh pub/sub (UDP in the PoC; Zenoh proposed — [04](04-swarm-sync.md)). +- **Bulk sync** pulls sealed derived partitions from peers over persistent SSH (rsync delta transfer). MinIO is optional where an S3 API is wanted; it is **not** the in-flight peer replication path. +- **Peer queries** are read-only DuckDB SQL — HTTP explorer gate in the PoC; **SSH forced commands** proposed for flight (SELECT-only gate, read-only OS user, columnar responses — [04](04-swarm-sync.md)). ## Communication planes diff --git a/docs/04-swarm-sync.md b/docs/04-swarm-sync.md index c7c4f38..b8a0730 100644 --- a/docs/04-swarm-sync.md +++ b/docs/04-swarm-sync.md @@ -16,19 +16,27 @@ Bandwidth is the scarcest resource in the system. Every message class gets an ex ## Broadcast payload: small on the wire, precise at rest -The `state` broadcast is a fixed compact frame: +The `state` broadcast is a fixed compact frame. The runnable PoC encodes it as +**45 bytes** little-endian (`simulator/virtual_drone/broadcast.py`); that size is +what the prototype uses for bandwidth estimates. -| Field | Type | Notes | +| Field | Wire type (PoC) | Notes | | --- | --- | --- | -| `drone_id` | uint16 | Fleet-scoped registry | +| `magic` + `version` | 2s + uint8 | `b"SH"`, version `1` | +| `drone_id` | 8s ascii | Zero-padded; a fleet `uint16` registry id is a natural production swap | | `ts_ns` | int64 | Epoch nanoseconds, same clock domain as storage | -| `pos_x/y/z` | int32 | **Millimeters** in the mission frame — quantized only here, storage keeps full float precision | -| `att_roll/pitch/yaw` | int16 | Centi-degrees | -| `vel_x/y/z` | int16 | cm/s | +| `pos_x/y/z` | 3 × int32 | **Millimeters** in the **mission frame** — quantized on the wire only; storage keeps full float precision | +| `att_roll/pitch/yaw` | 3 × int16 | Centi-degrees — attitude stays on the wire so peers need no local shape model | +| `vel_x/y/z` | 3 × int16 | cm/s | | `frame_ref` | uint8 | Frame of reference id (GPS-denied: local/visual-odometry frames must be explicit) | | `flags` | uint8 | Battery-low, returning, degraded-sensors, … | -~40 bytes per frame → a 50-drone swarm at 5 Hz is ~10 KB/s of pose traffic before transport overhead. Trivial even on a congested mesh. +45 bytes × 5 Hz × 50 drones ≈ 11 KB/s of pose traffic before transport overhead — still trivial on a congested mesh. + +An earlier sketch used relative coordinates and a bounding sphere (no attitude). +It was dropped: mission-frame pose + attitude is simpler to fuse post-flight and +costs almost nothing at this frame size. Relative localization remains a +consumer concern when `frame_ref` differs across peers ([09](09-open-questions.md)). `detections` events are slightly larger (class, confidence, bounding volume, ego-pose) but event-shaped and rare by comparison. @@ -51,25 +59,27 @@ graph LR SB -->|"store-and-forward relay"| SC ``` -### Recommended: Zenoh +### Recommended (proposal): Zenoh - Designed exactly for constrained, dynamic networks: built-in peer discovery, brokerless peer-to-peer mode, store-and-forward, and a query layer on top of pub/sub. - First-class robotics citizenship: an official ROS 2 RMW implementation exists, so the ingestion side and the sync side can share one middleware. - Tiny footprint, ARM64-native. +**PoC today:** the simulator and the 2D prototype exercise the **raw UDP** pose path only — the minimal degraded profile below. Zenoh is the proposed production pub/sub, not yet wired into the runnable stack. + ### Alternatives considered | Option | Verdict | | --- | --- | | **DDS multicast** (ROS 2 default) | Works, battle-tested; but discovery storms and tuning pain on lossy wireless meshes are well documented. Keep as fallback since ROS 2 speaks it natively | | **MQTT** | Needs a broker — a per-drone broker bridge is possible but adds moving parts for no gain over Zenoh | -| **Raw UDP multicast** | Perfect as a last-resort minimal profile for the pose broadcast alone (fixed frame, no discovery); no query layer, no reliability — documented as the degraded mode | -| **MinIO bucket replication** | Wrong tool for the 5 Hz pose path, right tool for bulk derived datasets — see below | +| **Raw UDP multicast** | **Implemented in the PoC** for pose broadcast (fixed frame, no discovery); no query layer, no reliability — also the documented degraded mode | +| **MinIO bucket replication** | Wrong tool for the 5 Hz pose path; optional on board for derived datasets and primary on the ground warehouse — not the in-flight bulk path | ## Two sync mechanisms, deliberately separate -1. **Fast path — pub/sub (Zenoh):** pose frames and detection events. Fire-and-forget with bounded staleness; consumers keep a peer-state cache. -2. **Bulk path — rsync over persistent SSH:** sealed `detections`/`state` Parquet partitions are pulled opportunistically between drones when links allow. This is how a drone that was out of range catches up on mission history without anyone re-sending events. +1. **Fast path — pub/sub (Zenoh proposed; UDP in the PoC):** pose frames and detection events. Fire-and-forget with bounded staleness; consumers keep a peer-state cache. +2. **Bulk path — rsync over persistent SSH (proposed):** sealed `detections`/`state` Parquet partitions are pulled opportunistically between drones when links allow. This is how a drone that was out of range catches up on mission history without anyone re-sending events. The PoC visualises bulk volume; it does not yet run real rsync between virtual drones. Why SSH-based bulk sync over object-store replication: @@ -105,7 +115,7 @@ SQL access must not become a write channel. A single "read-only connection" flag | Layer | Mechanism | What it stops | | --- | --- | --- | | 1. Key = operation | Forced command: the query key can only invoke the query wrapper, nothing else | Arbitrary exec, lateral movement | -| 2. Statement gate | Wrapper accepts a single statement, parses it, rejects anything but `SELECT` (no `COPY`, `ATTACH`, `INSTALL`, `SET`, multi-statements); parameters bound, not interpolated | SQL-as-a-write-channel, config tampering | +| 2. Statement gate | Wrapper accepts a single statement, rejects anything but `SELECT`/`WITH`/… (no `COPY`, `ATTACH`, `INSTALL`, `SET`, multi-statements); production should prefer a real parser + bound parameters, not keywords alone | SQL-as-a-write-channel, config tampering | | 3. OS permissions | Wrapper runs as a dedicated user with **read-only filesystem access** to the data root and write access to nothing | Any write that slips past layer 2 | | 4. Engine hardening | `:memory:` database, external access disabled except the data-root glob, extension loading off | Reaching outside the store | | 5. Resource caps | Timeout, memory cap, niced CPU (flight software always wins), response size budget | Denial of service via expensive queries | diff --git a/docs/12-design-journey.md b/docs/12-design-journey.md index 679eda1..2ce6da4 100644 --- a/docs/12-design-journey.md +++ b/docs/12-design-journey.md @@ -1,9 +1,12 @@ # 12 — Design journey How this design came together, told in the order the thinking actually happened. -It is a walk-through, not a report. The formal decisions, with options and -trade-offs, live as [Architecture Decision Records](adr/README.md). This is the -story behind them, and each chapter links straight into the code it produced. +It is a walk-through, not a report — and **not a mandate**. The goal is a +from-scratch platform sketch that shows how the pieces fit; every concrete +choice is an option with trade-offs recorded as +[Architecture Decision Records](adr/README.md). Replace any piece if a better +fit shows up. This is the story behind the decisions, and each chapter links +straight into the code it produced. --- @@ -70,15 +73,17 @@ stored and in which structure, not to step into that work. ## 5. What actually needs to sync The most time-critical item is where each peer is, so every unit has time to -react. The choice was to broadcast [relative pose](04-swarm-sync.md) (x, y, z and -time) instead of absolute coordinates. Relative is cheaper and enough for -coordination. To avoid sending orientation, a unit is modelled as a **sphere** -that bounds its extent. That trades a little compute for much less data, and no -per-shape encoding. If the units are identical, their 3D model can be provisioned -ahead instead of transmitted. +react. The wire frame carries **mission-frame position** (millimetres on the +wire, full float at rest), **attitude**, **velocity**, a `frame_ref`, and flags — +45 bytes at 5 Hz. That is cheap enough that shrinking further is not worth the +fusion pain. An earlier sketch used relative coordinates and a bounding sphere +with no orientation; it was dropped. When peers disagree on frames, +`frame_ref` makes the mismatch explicit for consumers +([09 — Open questions](09-open-questions.md)). > **Read the code** -> - [`simulator/virtual_drone/broadcast.py`](https://git.produktor.io/eSlider/swarm-house/src/branch/main/simulator/virtual_drone/broadcast.py#L1-L40) — the 46-byte pose frame, position quantized on the wire only +> - [`simulator/virtual_drone/broadcast.py`](https://git.produktor.io/eSlider/swarm-house/src/branch/main/simulator/virtual_drone/broadcast.py#L1-L45) — the 45-byte pose frame, position quantized on the wire only +> - [`prototype/src/sim.ts`](https://git.produktor.io/eSlider/swarm-house/src/branch/main/prototype/src/sim.ts#L60-L63) — `POSE_BYTES = 45` drives the volume estimate > - [`prototype/src/sim.ts`](https://git.produktor.io/eSlider/swarm-house/src/branch/main/prototype/src/sim.ts#L260-L277) — pose broadcasts at 5 Hz and opportunistic bulk sync, made visible ## 6. Security as nested layers, not one wall diff --git a/docs/adr/ADR-0004-sql-over-ssh-contract.md b/docs/adr/ADR-0004-sql-over-ssh-contract.md index d753df2..552166e 100644 --- a/docs/adr/ADR-0004-sql-over-ssh-contract.md +++ b/docs/adr/ADR-0004-sql-over-ssh-contract.md @@ -32,10 +32,15 @@ wrapper, permissions, and output format a peer drone would use. ## Consequences -- One access path is built, secured, and tested — "what you test is what +- One access path is designed for flight and bench — "what you test is what flies". -- The SQL gate is safety-critical and is covered by unit tests - (`simulator/tests/test_sql_gate.py`). -- The explorer and the prototype's live mode are *just another read-only - consumer* of this same contract ([`../04-swarm-sync.md`](../04-swarm-sync.md)). +- The statement gate in the PoC (`simulator/explorer/server.py`, covered by + `simulator/tests/test_sql_gate.py`) is a **first layer**: keyword allow/deny + over HTTP for the explorer. Production still needs the remaining layers in + [04 — Swarm sync](../04-swarm-sync.md) (forced-command key, read-only OS user, + engine hardening, resource caps) and a real SQL parser rather than keywords + alone. +- The explorer and the prototype's live mode are *stand-in consumers* of the + same read-only contract over HTTP today; the proposed flight path is + SQL-over-SSH ([`../04-swarm-sync.md`](../04-swarm-sync.md)). - Design principles 7 and 8 in [`../../README.md`](../../README.md) restate this. diff --git a/prototype/src/App.tsx b/prototype/src/App.tsx index 9e0c2a6..ba13013 100644 --- a/prototype/src/App.tsx +++ b/prototype/src/App.tsx @@ -268,7 +268,7 @@ export default function App(): JSX.Element { {busiest.length === 0 &&
no links in range
}
-
blue line — pose broadcast (5 Hz, ~46 B)
+
blue line — pose broadcast (5 Hz, 45 B)
green line — bulk sync (sealed partitions)
flash — broadcast event delivered
red circle — transit object crossing the area
diff --git a/prototype/src/sim.ts b/prototype/src/sim.ts index 6e18131..d9484ad 100644 --- a/prototype/src/sim.ts +++ b/prototype/src/sim.ts @@ -57,7 +57,7 @@ export function areaWidth(): number { export const LINK_RANGE = 320; const CRUISE = 28; const SEPARATION = 55; -const POSE_BYTES = 46; +const POSE_BYTES = 45; const POSE_HZ = 5; // Steady per-link broadcast throughput (both directions), bytes/s export const POSE_RATE_BYTES = POSE_BYTES * POSE_HZ; diff --git a/simulator/tests/test_broadcast.py b/simulator/tests/test_broadcast.py index ff560fe..d2bf327 100644 --- a/simulator/tests/test_broadcast.py +++ b/simulator/tests/test_broadcast.py @@ -9,7 +9,7 @@ from virtual_drone.flight import Pose def test_frame_size_matches_wire_spec() -> None: - # Documented as 46 B in docs/04; struct packs to FRAME.size (45 B on this layout). + # Keep docs/04, the journey, and prototype/src/sim.ts POSE_BYTES in lockstep. assert FRAME.size == 45 diff --git a/simulator/virtual_drone/broadcast.py b/simulator/virtual_drone/broadcast.py index c2e9e6b..f2a6124 100644 --- a/simulator/virtual_drone/broadcast.py +++ b/simulator/virtual_drone/broadcast.py @@ -1,12 +1,13 @@ """State broadcast over UDP: the compact pose frame from the sync design. -Frame layout (little-endian, 46 bytes): +Frame layout (little-endian, 45 bytes) — source of truth for docs and the +prototype bandwidth estimate: magic 2s b"SH" version B - drone_id 8s zero-padded ascii + drone_id 8s zero-padded ascii (PoC; production may switch to uint16 registry) ts_ns q epoch nanoseconds - pos_mm 3i position, millimeters (quantized on the wire only) + pos_mm 3i position in the mission frame, millimeters (wire quantization only) att_cdeg 3h roll/pitch/yaw, centi-degrees vel_cms 3h velocity, cm/s frame_ref B