diff --git a/.claude/rules/aether-api-payloads.md b/.claude/rules/aether-api-payloads.md new file mode 100644 index 0000000..e9ea42e --- /dev/null +++ b/.claude/rules/aether-api-payloads.md @@ -0,0 +1,49 @@ +# Aether API Payloads Context + +paths: + - incus-mitm + - api-interception + +## How Aether Communicates with Incus + +Aether (192.168.102.160) authenticates to the Incus cluster using a TLS +client certificate with subject `CN=root@oc-server`. Its fingerprint +prefix is `6cfd2a1949a7`. + +### Aether's Behavior Pattern + +1. Subscribes to Incus event stream (persistent websocket) +2. Reacts to lifecycle events by querying affected resources +3. When user triggers UI actions, makes Incus API calls server-side + +### Capturing Events + +```bash +# Live stream with Aether filter +incusos/helpers/incus-mitm --live --filter aether + +# Capture to file +incusos/helpers/incus-mitm --capture 120 +incusos/helpers/incus-mitm --analyze /tmp/incus-events-*.json --filter aether +``` + +### API Call Patterns + +All Incus operations follow: `request → operation (pending→running→success) → lifecycle event` + +Key patterns: +- Stop: `PUT /1.0/instances//state` +- Start: `PUT /1.0/instances//state` +- Snapshot: `POST /1.0/instances//snapshots` +- Delete: `DELETE /1.0/instances//snapshots/` +- Exec: `POST /1.0/instances//exec` (4 websockets) + +Aether reacts to lifecycle events with: `GET /1.0 → GET /1.0/instances/?recursion=1` + +### Aether's Own REST API + +Separate from Incus, JWT-authenticated: +- `POST /api/auth/token` +- `GET /api/clusters` +- `CRUD /api/clusters/{id}/rules` +- `CRUD /api/global/rules` diff --git a/.mcp.json b/.mcp.json index 259a959..e420c75 100644 --- a/.mcp.json +++ b/.mcp.json @@ -2,7 +2,7 @@ "mcpServers": { "playwright": { "command": "npx", - "args": ["@playwright/mcp@latest"] + "args": ["@playwright/mcp@latest", "--ignore-https-errors"] } } } diff --git a/CLAUDE.md b/CLAUDE.md index 0afd4f7..4aca880 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,7 +32,8 @@ incus-contrib/ │ │ ├── proxmox-screenshot # VMID -> PNG console screenshot │ │ ├── proxmox-api # Authenticated API calls (handles ! in token) │ │ ├── aether-browser # Playwright browser automation for Aether web UI -│ │ └── ovn-inspect # OVN/OVS topology inspection (--nb/--sb/--ovs/--trace) +│ │ ├── ovn-inspect # OVN/OVS topology inspection (--nb/--sb/--ovs/--trace) +│ │ └── incus-mitm # API traffic capture and analysis (--capture/--live/--analyze) │ ├── lab-test # Guided lab validation (12 test phases) │ ├── observe-deploy # Single-VM deploy with console screenshots │ ├── proxmox.yaml # Proxmox connection (gitignored) @@ -187,6 +188,7 @@ which files are being edited: | `awx-integration.md` | ansible/, deploy-awx, awx-manifests, AWX guide | | `haproxy-lb.md` | deploy-haproxy, HAProxy guide | | `ovn-internals.md` | ovn-inspect, ovn-deep-dive | +| `aether-api-payloads.md` | incus-mitm, api-interception | | `proxmox-ssh-rules.md` | **Always loaded** (no paths filter) | | `lab-infrastructure.md` | incusos-proxmox, lab-test, examples | diff --git a/incusos/helpers/incus-mitm b/incusos/helpers/incus-mitm new file mode 100755 index 0000000..5dcefe5 --- /dev/null +++ b/incusos/helpers/incus-mitm @@ -0,0 +1,397 @@ +#!/usr/bin/env bash +# incus-mitm -- Capture and analyze Incus API traffic via event stream +# +# Usage: incus-mitm [ACTION] [OPTIONS] +# +# Actions: --capture, --analyze, --live, --callers +# Requires: incus CLI with cluster remote configured, python3 +# +# The primary capture method uses `incus monitor` which captures all +# API operations without requiring TLS interception. + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Colors +# --------------------------------------------------------------------------- + +setup_colors() { + if [[ "${NO_COLOR:-}" == "1" || "${TERM:-}" == "dumb" ]]; then + BOLD="" DIM="" RESET="" CYAN="" GREEN="" YELLOW="" RED="" BLUE="" + else + BOLD=$'\033[1m' DIM=$'\033[2m' RESET=$'\033[0m' + CYAN=$'\033[36m' GREEN=$'\033[32m' YELLOW=$'\033[33m' + RED=$'\033[31m' BLUE=$'\033[34m' + fi +} +setup_colors + +# --------------------------------------------------------------------------- +# Usage +# --------------------------------------------------------------------------- + +usage() { + cat <<'EOF' +Usage: incus-mitm [ACTION] [OPTIONS] + +Capture and analyze Incus REST API traffic using the event stream. +No TLS interception needed — uses `incus monitor` to observe all +API calls, operations, and lifecycle events in real time. + +Actions: + --capture [SECONDS] Capture events for N seconds (default: 60) + --live Stream events in real time (Ctrl+C to stop) + --analyze FILE Analyze a previously captured JSON file + --callers FILE Show unique API callers by certificate + +Options: + --remote NAME Incus remote name (default: oc-node-01) + --output FILE Output file for capture (default: /tmp/incus-events-.json) + --filter CALLER Only show events from specific caller (aether, cli, node) + --dry-run Show commands that would be executed + --help, -h Show this help + +Examples: + incus-mitm --capture 120 # Capture 2 minutes of events + incus-mitm --capture 30 --output events.json + incus-mitm --live # Stream in real time + incus-mitm --analyze /tmp/incus-events.json + incus-mitm --callers /tmp/incus-events.json + incus-mitm --analyze events.json --filter aether + +Certificate Fingerprint Reference: + Capture events first, then use --callers to identify who is who. + Common patterns: + CN=maarten@* → CLI client + CN=root@oc-* → Aether SDN controller + CN=root@oc-node-* → Cluster inter-node traffic +EOF + exit 0 +} + +# --------------------------------------------------------------------------- +# Defaults and argument parsing +# --------------------------------------------------------------------------- + +ACTION="" +REMOTE="oc-node-01" +OUTPUT="" +CAPTURE_SECONDS=60 +FILTER="" +DRY_RUN=false +ANALYZE_FILE="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --capture) + ACTION="capture" + if [[ "${2:-}" =~ ^[0-9]+$ ]]; then + CAPTURE_SECONDS="$2" + shift + fi + shift + ;; + --live) ACTION="live"; shift ;; + --analyze) + ACTION="analyze" + ANALYZE_FILE="${2:?--analyze requires a file}" + shift 2 + ;; + --callers) + ACTION="callers" + ANALYZE_FILE="${2:?--callers requires a file}" + shift 2 + ;; + --remote) REMOTE="${2:?--remote requires a name}"; shift 2 ;; + --output) OUTPUT="${2:?--output requires a path}"; shift 2 ;; + --filter) FILTER="${2:?--filter requires a caller type}"; shift 2 ;; + --dry-run) DRY_RUN=true; shift ;; + -h|--help) usage ;; + *) + echo "Error: unknown option: $1" >&2 + echo "Run 'incus-mitm --help' for usage." >&2 + exit 1 + ;; + esac +done + +if [[ -z "$ACTION" ]]; then + usage +fi + +# --------------------------------------------------------------------------- +# Capture +# --------------------------------------------------------------------------- + +do_capture() { + local output="${OUTPUT:-/tmp/incus-events-$(date +%Y%m%d-%H%M%S).json}" + + if [[ "$DRY_RUN" == true ]]; then + echo "${DIM}[dry-run] timeout ${CAPTURE_SECONDS} incus monitor ${REMOTE}: --format=json > ${output}${RESET}" + return 0 + fi + + echo "${BOLD}Capturing Incus events for ${CAPTURE_SECONDS}s...${RESET}" + echo "${DIM}Output: ${output}${RESET}" + echo "${DIM}Trigger operations in Aether or via CLI to generate events.${RESET}" + echo + + timeout "$CAPTURE_SECONDS" incus monitor "${REMOTE}:" --format=json > "$output" 2>&1 || true + + local count + count=$(wc -l < "$output") + echo + echo "${GREEN}Captured ${count} events → ${output}${RESET}" + echo "Run: ${BOLD}incus-mitm --analyze ${output}${RESET}" +} + +# --------------------------------------------------------------------------- +# Live stream +# --------------------------------------------------------------------------- + +do_live() { + if [[ "$DRY_RUN" == true ]]; then + echo "${DIM}[dry-run] incus monitor ${REMOTE}: --format=json | python3 (live parser)${RESET}" + return 0 + fi + + echo "${BOLD}Live event stream (Ctrl+C to stop)${RESET}" + echo + + incus monitor "${REMOTE}:" --format=json 2>&1 | python3 -u -c " +import sys, json + +# Known fingerprint prefixes (populated during session) +known_certs = {} + +for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + e = json.loads(line) + except: + continue + + ts = e.get('timestamp', '')[11:19] + etype = e.get('type', '') + meta = e.get('metadata', {}) + + # Track cert fingerprints + if etype == 'logging' and 'Matched trusted cert' in meta.get('message', ''): + ctx = meta.get('context', {}) + fp = ctx.get('fingerprint', '') + subj = ctx.get('subject', '') + if fp and subj: + known_certs[fp[:20]] = subj + + # Show API calls + if etype == 'logging' and 'method' in meta.get('context', {}): + ctx = meta['context'] + fp = ctx.get('username', '')[:20] + caller = known_certs.get(fp, fp[:12] + '...') + if 'maarten' in caller: + caller = 'CLI' + elif 'oc-server' in caller: + caller = 'Aether' + elif 'oc-node' in caller: + caller = caller.split('root@')[1].split(',')[0] if 'root@' in caller else 'node' + + filter_val = '${FILTER}'.lower() + if filter_val: + if filter_val == 'aether' and 'Aether' not in caller: + continue + elif filter_val == 'cli' and 'CLI' not in caller: + continue + elif filter_val == 'node' and 'oc-node' not in caller: + continue + + print(f' \033[2m{ts}\033[0m {ctx[\"method\"]:6s} {ctx[\"url\"]:70s} \033[36m[{caller}]\033[0m', flush=True) + + # Show lifecycle events + elif etype == 'lifecycle': + action = meta.get('action', '') + source = (meta.get('source') or '').split('/')[-1] + print(f' \033[2m{ts}\033[0m \033[33m{action:30s}\033[0m {source}', flush=True) + + # Show operation status changes (only success/failure) + elif etype == 'operation': + status = meta.get('status', '') + if status in ('Success', 'Failure'): + desc = meta.get('description', '') + resources = [] + for k, v in (meta.get('resources') or {}).items(): + for item in v: + resources.append(item.split('/')[-1]) + res = ', '.join(resources) + color = '\033[32m' if status == 'Success' else '\033[31m' + print(f' \033[2m{ts}\033[0m {color}[{status:7s}]\033[0m {desc:30s} {res}', flush=True) +" +} + +# --------------------------------------------------------------------------- +# Analyze +# --------------------------------------------------------------------------- + +do_analyze() { + if [[ ! -f "$ANALYZE_FILE" ]]; then + echo "Error: file not found: $ANALYZE_FILE" >&2 + exit 1 + fi + + python3 -c " +import json, sys +from collections import defaultdict + +events = [] +with open('${ANALYZE_FILE}') as f: + for line in f: + line = line.strip() + if not line: continue + try: events.append(json.loads(line)) + except: pass + +# Build cert map +cert_map = {} +for e in events: + if e.get('type') == 'logging' and 'Matched trusted cert' in e.get('metadata', {}).get('message', ''): + ctx = e['metadata']['context'] + fp = ctx.get('fingerprint', '') + subj = ctx.get('subject', '') + if fp and subj: + cert_map[fp[:20]] = subj + +def caller_name(fp): + subj = cert_map.get(fp[:20], '') + if 'maarten' in subj: return 'CLI' + if 'oc-server' in subj: return 'Aether' + if 'oc-node' in subj: + return subj.split('root@')[1].split(',')[0] if 'root@' in subj else 'node' + return fp[:12] + '...' + +filter_val = '${FILTER}'.lower() + +print('\033[1m\033[36m=== API Calls ===\033[0m') +api_calls = [] +for e in events: + if e.get('type') == 'logging' and 'method' in e.get('metadata', {}).get('context', {}): + ctx = e['metadata']['context'] + caller = caller_name(ctx.get('username', '')) + if filter_val: + if filter_val == 'aether' and caller != 'Aether': continue + elif filter_val == 'cli' and caller != 'CLI': continue + ts = e['timestamp'][11:19] + api_calls.append((ts, ctx['method'], ctx['url'], caller)) + print(f\" {ts} {ctx['method']:6s} {ctx['url']:70s} [{caller}]\") + +print() +print('\033[1m\033[36m=== Operations ===\033[0m') +seen = set() +for e in events: + if e.get('type') == 'operation': + meta = e.get('metadata', {}) + key = f\"{meta.get('id')}_{meta.get('status')}\" + if key in seen: continue + seen.add(key) + status = meta.get('status', '') + if status not in ('Success', 'Failure', 'Cancelled'): continue + resources = [] + for k, v in (meta.get('resources') or {}).items(): + for item in v: resources.append(item.split('/')[-1]) + ts = e['timestamp'][11:19] + color = '\033[32m' if status == 'Success' else '\033[31m' + print(f\" {ts} {color}[{status:7s}]\033[0m {meta.get('description'):30s} {', '.join(resources)}\") + +print() +print('\033[1m\033[36m=== Lifecycle Events ===\033[0m') +for e in events: + if e.get('type') == 'lifecycle': + meta = e.get('metadata', {}) + ts = e['timestamp'][11:19] + source = (meta.get('source') or '').split('/')[-1] + print(f\" {ts} {meta.get('action'):30s} {source}\") + +print() +print('\033[1m\033[36m=== Summary ===\033[0m') +print(f' Total events: {len(events)}') +print(f' API calls: {len(api_calls)}') +callers = defaultdict(int) +methods = defaultdict(int) +for _, method, url, caller in api_calls: + callers[caller] += 1 + methods[method] += 1 +print(f' By caller:') +for c, n in sorted(callers.items(), key=lambda x: -x[1]): + print(f' {c}: {n}') +print(f' By method:') +for m, n in sorted(methods.items(), key=lambda x: -x[1]): + print(f' {m}: {n}') +" +} + +# --------------------------------------------------------------------------- +# Callers +# --------------------------------------------------------------------------- + +do_callers() { + if [[ ! -f "$ANALYZE_FILE" ]]; then + echo "Error: file not found: $ANALYZE_FILE" >&2 + exit 1 + fi + + python3 -c " +import json +from collections import defaultdict + +events = [] +with open('${ANALYZE_FILE}') as f: + for line in f: + line = line.strip() + if not line: continue + try: events.append(json.loads(line)) + except: pass + +cert_map = {} +call_counts = defaultdict(int) + +for e in events: + if e.get('type') == 'logging': + meta = e.get('metadata', {}) + ctx = meta.get('context', {}) + if 'Matched trusted cert' in meta.get('message', ''): + fp = ctx.get('fingerprint', '') + subj = ctx.get('subject', '') + if fp and subj: + cert_map[fp] = subj + if 'method' in ctx: + call_counts[ctx.get('username', '')] += 1 + +print('\033[1m\033[36m=== Certificate Fingerprints ===\033[0m') +for fp, subj in sorted(cert_map.items(), key=lambda x: x[1]): + calls = call_counts.get(fp, 0) + if 'maarten' in subj: + role = 'CLI client' + elif 'oc-server' in subj: + role = 'Aether SDN controller' + elif 'oc-node' in subj: + role = 'Cluster node' + else: + role = 'Unknown' + print(f' {fp[:40]}...') + print(f' Subject: {subj}') + print(f' Role: {role}') + print(f' Calls: {calls}') + print() +" +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +case "$ACTION" in + capture) do_capture ;; + live) do_live ;; + analyze) do_analyze ;; + callers) do_callers ;; +esac diff --git a/notes/api-interception-guide.md b/notes/api-interception-guide.md new file mode 100644 index 0000000..487fcca --- /dev/null +++ b/notes/api-interception-guide.md @@ -0,0 +1,249 @@ +# API Interception Guide: Observing Aether ↔ Incus Communication + +How to capture, analyze, and understand the REST API calls between +Aether (SDN controller) and the Incus cluster at the wire level. + +## Architecture + +``` +Aether (192.168.102.160) + │ Client cert: CN=root@oc-server + │ Subscribes to: /1.0/events (lifecycle stream) + │ + ├──→ Incus API (https://192.168.102.140-142:8443) + │ All instance/network/storage operations + │ + └──→ Aether's own REST API (/api/*) + Auth, clusters, ACLs only + JWT-authenticated, separate from Incus + +CLI (dev-vm-beelink) + │ Client cert: CN=maarten@dev-vm-beelink + │ + └──→ Incus API (same endpoints) + +Cluster nodes (oc-node-01/02/03) + │ Client certs: CN=root@oc-node-* + │ + └──→ Inter-node API (internal cluster operations) +``` + +## Capture Methods + +### Primary: Incus Event Stream + +The `incus monitor` command subscribes to the Incus event websocket and +captures every API call, operation lifecycle change, and state transition. + +```bash +# Quick capture (60 seconds) +incusos/helpers/incus-mitm --capture 60 + +# Long capture with custom output +incusos/helpers/incus-mitm --capture 300 --output /tmp/my-capture.json + +# Live streaming (Ctrl+C to stop) +incusos/helpers/incus-mitm --live + +# Filter for Aether only +incusos/helpers/incus-mitm --live --filter aether +``` + +**What gets captured:** +- Every REST API call (method, path, caller certificate) +- Operation lifecycle (Pending → Running → Success/Failure) +- Instance lifecycle events (created, started, stopped, deleted, etc.) +- Cluster-internal operations (node-to-node forwarding) + +**What doesn't get captured:** +- Request/response bodies (only the path and method) +- Aether-internal operations (JWT API calls don't go through Incus) +- Timing/latency data (only timestamps) + +### Secondary: Aether Go Log + +Aether has a live log viewable at `/logs/live` in the web UI (sidebar → +"View Live GO Log"). This shows Aether's perspective: what it decides to +do and what errors it encounters. + +Access via Playwright (session-authenticated route): +```bash +source env +NODE_PATH=~/node_modules node incusos/helpers/aether-browser screenshot /logs/live +``` + +### Tertiary: mitmproxy (TLS Interception) + +For capturing full request/response bodies, a mitmproxy reverse proxy +can be deployed. This is complex due to mutual TLS and is only needed +if the event stream doesn't provide enough detail. + +**Not implemented** — the event stream proved sufficient for understanding +the API protocol. + +## Analyzing Captures + +```bash +# Full analysis +incusos/helpers/incus-mitm --analyze /tmp/incus-events.json + +# Show only Aether's calls +incusos/helpers/incus-mitm --analyze /tmp/incus-events.json --filter aether + +# Identify callers by certificate +incusos/helpers/incus-mitm --callers /tmp/incus-events.json +``` + +## Certificate Fingerprints + +Each caller is identified by their TLS client certificate fingerprint. +The event stream logs certificate matching, allowing us to identify who +makes each API call. + +| Fingerprint Prefix | Subject | Role | +|---|---|---| +| `bb2dc9eac3d3...` | CN=maarten@dev-vm-beelink | CLI client | +| `6cfd2a1949a7...` | CN=root@oc-server | Aether SDN controller | +| `a68291ca3683...` | CN=root@oc-node-02 | Cluster inter-node | + +Aether's cert (`root@oc-server`) is the key one — it identifies all +operations that Aether initiates against the Incus cluster. + +## Captured API Patterns + +### Instance Stop (CLI-initiated) + +``` +CLI → PUT /1.0/instances//state (body: {"action":"stop"}) +CLI → GET /1.0/operations/ (poll for completion) +Incus → operation: Stopping instance [Pending → Running → Success] +Incus → lifecycle: instance-shutdown +``` + +### Instance Start (CLI-initiated) + +``` +CLI → PUT /1.0/instances//state (body: {"action":"start"}) +CLI → GET /1.0/operations/ +Incus → operation: Starting instance [Pending → Running → Success] +Incus → lifecycle: instance-started +``` + +### Snapshot Create + +``` +CLI → POST /1.0/instances//snapshots (body: {"name":"snap-name"}) +CLI → GET /1.0/operations/ +Incus → operation: Snapshotting instance [Pending → Running → Success] +Incus → lifecycle: instance-snapshot-created +Aether → GET /1.0 (reacts to lifecycle event) +Aether → GET /1.0/instances/?project=default&recursion=1 +``` + +### Snapshot Delete + +``` +CLI → DELETE /1.0/instances//snapshots/ +CLI → GET /1.0/operations/ +Incus → operation: Deleting snapshot [Pending → Running → Success] +Incus → lifecycle: instance-snapshot-deleted +Aether → GET /1.0 (reacts to lifecycle event) +Aether → GET /1.0/instances/?project=default&recursion=1 +``` + +### Exec Command + +``` +CLI → POST /1.0/instances//exec (body: {"command":["hostname"],...}) +Node → POST https://:8443/1.0/instances//exec (forwarded) +CLI → GET /1.0/operations//websocket?secret= +CLI → GET /1.0/operations//websocket?secret= +CLI → GET /1.0/operations//websocket?secret= +CLI → GET /1.0/operations//websocket?secret= +CLI → GET /1.0/operations/ (poll completion) +Incus → operation: Executing command [Pending → Running → Success] +Incus → lifecycle: instance-exec +``` + +Note: exec uses 4 websocket connections (stdin, stdout, stderr, control). +The operation is forwarded to the target node where the instance runs. + +### Network List + +``` +CLI → GET /1.0/networks?filter=&recursion=1 +``` + +### Instance Info + +``` +CLI → GET /1.0/instances/?recursion=1 +``` + +## Key Findings + +### 1. Aether Subscribes to Lifecycle Events + +Aether maintains a persistent connection to the Incus event stream. +When any lifecycle event occurs (instance created, snapshot deleted, etc.), +Aether immediately queries the affected resource. This is how Aether +keeps its UI in sync with the cluster state. + +Pattern: `lifecycle event → Aether GET /1.0 → Aether GET /1.0/instances/` + +### 2. Aether Uses Client Certificate Auth + +Aether authenticates to Incus using a TLS client certificate +(`CN=root@oc-server`), not API tokens. This certificate is trusted by +all cluster members. The certificate is stored in the Aether VM +(likely at `/root/.config/incus/client.crt`). + +### 3. Operations Are Forwarded + +When an operation targets an instance on a different node, the receiving +node forwards the request internally. This is visible in the event stream +as a `POST https://:8443/...` call from an "internal" caller. + +### 4. Every CLI Command Starts with GET /1.0 + +The Incus CLI always fetches `/1.0` first (server info/capabilities) +before making any other call. It also opens an `/1.0/events` websocket +for real-time operation status updates. + +### 5. Aether REST API Is Separate + +Aether's own API (`/api/auth/token`, `/api/clusters`, `/api/clusters/*/rules`) +is completely separate from the Incus API. These calls go to Aether's +HTTPS server (port 8443) and use JWT authentication. They are NOT visible +in the Incus event stream. + +Current Aether API coverage: +- `POST /api/auth/token` — JWT authentication +- `GET /api/clusters` — list managed clusters +- `CRUD /api/clusters/{id}/rules` — per-cluster ACL rules +- `CRUD /api/global/rules` — global ACL rules + +All other Aether features (deploys, HAProxy, blueprints, instance +management) are UI-only and use session-authenticated routes with CSRF +protection. These features call the Incus API directly via Aether's +server-side code, visible in the event stream as calls from the +`root@oc-server` certificate. + +## Reproducing the Capture + +```bash +# 1. Start capture in one terminal +incusos/helpers/incus-mitm --live + +# 2. In another terminal, trigger operations: +incus stop oc-node-01: +incus start oc-node-01: +incus snapshot create oc-node-01: test-snap +incus snapshot delete oc-node-01: test-snap + +# 3. Or trigger via Aether UI: +# - Deploy a container +# - Create a blueprint +# - Manage HAProxy +# (All will appear in the event stream as Aether API calls) +```