diff --git a/.gitignore b/.gitignore index 13aa91e..cb111d3 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,6 @@ sources/ # Playwright MCP logs .playwright-mcp/ + +# Claude Code local state +.claude/scheduled_tasks.lock diff --git a/ansible/playbooks/post-deploy.yml b/ansible/playbooks/post-deploy.yml index a5d6fd6..c558d09 100644 --- a/ansible/playbooks/post-deploy.yml +++ b/ansible/playbooks/post-deploy.yml @@ -29,7 +29,7 @@ vars: # Incus cluster API (any cluster member works) - incus_api: "https://192.168.102.140:8443" + incus_api: "https://10.10.0.111:8443" # AWX runner mounts project at /runner/project/ during job execution incus_cert: "/runner/project/incus-client.crt" incus_key: "/runner/project/incus-client.key" diff --git a/bin/lab-status b/bin/lab-status new file mode 100755 index 0000000..3ad4bef --- /dev/null +++ b/bin/lab-status @@ -0,0 +1,436 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly VERSION="0.1.0" +readonly SCRIPT_NAME="lab-status" +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly REPO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +readonly ENVS_DIR="${REPO_DIR}/envs" + +# --- Defaults ----------------------------------------------------------------- + +QUIET=false +VERBOSE=false +ENV_NAME="" +NO_CHECK=false +JSON_OUTPUT=false + +# Service arrays (parallel indexed) +declare -a SVC_NAMES=() +declare -a SVC_URLS=() +declare -a SVC_STATUSES=() + +# --- Colors & logging -------------------------------------------------------- + +setup_colors() { + if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then + RED='\033[0;31m' + GREEN='\033[0;32m' + YELLOW='\033[0;33m' + BLUE='\033[0;34m' + CYAN='\033[0;36m' + BOLD='\033[1m' + DIM='\033[2m' + RESET='\033[0m' + else + RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' RESET='' + fi +} + +info() { [[ "$QUIET" == true ]] && return; echo -e "${BLUE}[info]${RESET} $*"; } +success() { [[ "$QUIET" == true ]] && return; echo -e "${GREEN}[ok]${RESET} $*"; } +warn() { echo -e "${YELLOW}[warn]${RESET} $*" >&2; } +error() { echo -e "${RED}[error]${RESET} $*" >&2; } +detail() { [[ "$VERBOSE" != true ]] && return; echo -e "${DIM} $*${RESET}"; } + +# --- Usage -------------------------------------------------------------------- + +usage() { + cat </ and checks whether each service + is reachable. Any HTTP response (even 401/403) counts as UP; connection + timeout or refused counts as DOWN. + + Services detected: Proxmox, OC Server, Cluster Nodes, Aether, AWX, + HAProxy VIP, Grafana, Prometheus. + +${BOLD}EXAMPLES${RESET} + # Show hetzner lab with reachability checks + ${SCRIPT_NAME} --env hetzner + + # Show beelink lab without checks + ${SCRIPT_NAME} --env beelink --no-check + + # JSON output for scripting + ${SCRIPT_NAME} --env hetzner --json + +EOF +} + +# --- YAML helpers ------------------------------------------------------------- + +# Extract a value from a YAML section: yaml_val FILE SECTION KEY +yaml_val() { + local file="$1" section="$2" key="$3" + grep -A50 "^${section}:" "$file" \ + | grep "^ *${key}:" | head -1 \ + | sed "s/^ *${key}: *//" | tr -d '"' | sed 's/ *#.*//' || true +} + +# Extract a top-level YAML value: yaml_val_top FILE KEY +yaml_val_top() { + local file="$1" key="$2" + grep "^${key}:" "$file" | head -1 \ + | sed "s/^${key}: *//" | tr -d '"' | sed 's/ *#.*//' || true +} + +# --- Service collection ------------------------------------------------------- + +add_service() { + local name="$1" url="$2" + SVC_NAMES+=("$name") + SVC_URLS+=("$url") + SVC_STATUSES+=("") +} + +detect_env() { + if [[ -n "$ENV_NAME" ]]; then + if [[ ! -d "${ENVS_DIR}/${ENV_NAME}" ]]; then + error "Environment not found: ${ENV_NAME}" + local available + available=$(ls "$ENVS_DIR" 2>/dev/null | tr '\n' ' ') + [[ -n "$available" ]] && error "Available: ${available}" + exit 1 + fi + return + fi + + local envs=() + for d in "$ENVS_DIR"/*/; do + [[ -d "$d" ]] && envs+=("$(basename "$d")") + done + + if [[ ${#envs[@]} -eq 1 ]]; then + ENV_NAME="${envs[0]}" + info "Auto-detected environment: ${ENV_NAME}" + elif [[ ${#envs[@]} -eq 0 ]]; then + error "No environments found in ${ENVS_DIR}/" + exit 1 + else + error "Multiple environments found: ${envs[*]}" + error "Use --env NAME to select one" + exit 1 + fi +} + +collect_proxmox() { + local env_dir="${ENVS_DIR}/${ENV_NAME}" + local pve_file="" + + if [[ -f "${env_dir}/proxmox.yaml" ]]; then + pve_file="${env_dir}/proxmox.yaml" + elif [[ -f "${env_dir}/proxmox.yaml.example" ]]; then + pve_file="${env_dir}/proxmox.yaml.example" + warn "Using proxmox.yaml.example (proxmox.yaml not found)" + else + detail "No proxmox.yaml found, skipping Proxmox" + return + fi + + local host + host=$(yaml_val_top "$pve_file" "host") + if [[ -n "$host" ]]; then + host="${host%%/*}" + add_service "Proxmox" "https://${host}:8006" + detail "Proxmox from $(basename "$pve_file"): ${host}" + fi +} + +collect_lab_vms() { + local env_dir="${ENVS_DIR}/${ENV_NAME}" + local lab_file="" + + # Prefer lab-production.yaml over lab-cluster.yaml + if [[ -f "${env_dir}/lab-production.yaml" ]]; then + lab_file="${env_dir}/lab-production.yaml" + elif [[ -f "${env_dir}/lab-cluster.yaml" ]]; then + lab_file="${env_dir}/lab-cluster.yaml" + else + detail "No lab-*.yaml found, skipping cluster VMs" + return + fi + + detail "Lab VMs from $(basename "$lab_file")" + + local vm_data + vm_data=$(python3 -c " +import sys, re + +try: + import yaml + with open(sys.argv[1]) as f: + data = yaml.safe_load(f) +except ImportError: + data = {'vms': []} + with open(sys.argv[1]) as f: + content = f.read() + in_vms = False + current = {} + for line in content.split('\n'): + if line.startswith('vms:'): + in_vms = True + continue + if in_vms: + if line.startswith(' - '): + if current: + data['vms'].append(current) + current = {} + line = line[4:] + elif line.startswith(' '): + line = line[4:] + elif line and not line.startswith(' ') and not line.startswith('#'): + if current: + data['vms'].append(current) + break + else: + continue + m = re.match(r'(\w+):\s*(.*)', line.strip()) + if m: + current[m.group(1)] = m.group(2).strip('\"') + if current: + data['vms'].append(current) + +for vm in data.get('vms', []): + name = vm.get('name', '') + app = vm.get('app', '') + ip = vm.get('ip', '').split('/')[0] + print(f'{name}|{app}|{ip}') +" "$lab_file" 2>/dev/null) || return + + local node_idx=0 + while IFS='|' read -r name app ip; do + [[ -z "$ip" ]] && continue + case "$app" in + operations-center) + add_service "OC Server" "https://${ip}:8443" + ;; + incus) + node_idx=$((node_idx + 1)) + local label + label=$(printf "Cluster Node %02d" "$node_idx") + add_service "$label" "https://${ip}:8443" + ;; + esac + done <<< "$vm_data" +} + +collect_haproxy() { + local env_dir="${ENVS_DIR}/${ENV_NAME}" + local file="${env_dir}/haproxy.yaml" + [[ -f "$file" ]] || return + + local aether_url vip + aether_url=$(yaml_val "$file" "haproxy" "aether_url") + vip=$(yaml_val "$file" "haproxy" "vip") + + if [[ -n "$aether_url" ]]; then + add_service "Aether" "$aether_url" + detail "Aether from haproxy.yaml: ${aether_url}" + fi + if [[ -n "$vip" ]]; then + vip="${vip%%/*}" + add_service "HAProxy VIP" "http://${vip}:80" + detail "HAProxy VIP from haproxy.yaml: ${vip}" + fi +} + +collect_awx() { + local env_dir="${ENVS_DIR}/${ENV_NAME}" + local file="${env_dir}/awx.yaml" + [[ -f "$file" ]] || return + + local ip + ip=$(yaml_val "$file" "awx" "ip") + [[ -z "$ip" ]] && return + ip="${ip%%/*}" + add_service "AWX" "http://${ip}:30080" + detail "AWX from awx.yaml: ${ip}" +} + +collect_observability() { + local env_dir="${ENVS_DIR}/${ENV_NAME}" + local file="${env_dir}/observability.yaml" + [[ -f "$file" ]] || return + + local forward_vip + forward_vip=$(yaml_val "$file" "observability" "forward_vip") + [[ -z "$forward_vip" ]] && return + forward_vip="${forward_vip%%/*}" + + add_service "Grafana" "http://${forward_vip}:3000" + add_service "Prometheus" "http://${forward_vip}:9090" + detail "Observability from observability.yaml: forward_vip=${forward_vip}" +} + +# --- Reachability checks ----------------------------------------------------- + +check_reachability() { + local url="$1" result_file="$2" + local http_code + http_code=$(curl -sk --connect-timeout 2 --max-time 3 \ + -o /dev/null -w '%{http_code}' "$url" 2>/dev/null) || http_code="000" + if [[ "$http_code" != "000" ]]; then + echo "UP" > "$result_file" + else + echo "DOWN" > "$result_file" + fi +} + +run_checks() { + local tmpdir + tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/${SCRIPT_NAME}-XXXXXX") + # shellcheck disable=SC2064 + trap "rm -rf '$tmpdir'" EXIT + + local i + for i in "${!SVC_URLS[@]}"; do + check_reachability "${SVC_URLS[$i]}" "${tmpdir}/${i}" & + done + wait + + for i in "${!SVC_URLS[@]}"; do + if [[ -f "${tmpdir}/${i}" ]]; then + SVC_STATUSES[$i]=$(cat "${tmpdir}/${i}") + else + SVC_STATUSES[$i]="DOWN" + fi + done +} + +# --- Output ------------------------------------------------------------------- + +print_table() { + local max_name=7 max_url=3 # minimum: "Service" / "URL" + local i + for i in "${!SVC_NAMES[@]}"; do + [[ ${#SVC_NAMES[$i]} -gt $max_name ]] && max_name=${#SVC_NAMES[$i]} + [[ ${#SVC_URLS[$i]} -gt $max_url ]] && max_url=${#SVC_URLS[$i]} + done + + local total_width=$((max_name + max_url + 14)) + local sep + sep=$(printf '%*s' "$total_width" '' | tr ' ' '─') + + echo + echo -e "${BOLD}Lab Environment: ${ENV_NAME}${RESET}" + echo "$sep" + printf "%-${max_name}s %-${max_url}s %s\n" "Service" "URL" "Status" + echo "$sep" + + for i in "${!SVC_NAMES[@]}"; do + local status="${SVC_STATUSES[$i]:-—}" + local status_colored + case "$status" in + UP) status_colored="${GREEN}UP${RESET}" ;; + DOWN) status_colored="${RED}DOWN${RESET}" ;; + *) status_colored="$status" ;; + esac + printf "%-${max_name}s %-${max_url}s " "${SVC_NAMES[$i]}" "${SVC_URLS[$i]}" + echo -e "$status_colored" + done + + echo "$sep" + echo +} + +print_json() { + local last=$((${#SVC_NAMES[@]} - 1)) + echo "[" + local i + for i in "${!SVC_NAMES[@]}"; do + local comma="," + [[ $i -eq $last ]] && comma="" + local status_json + if [[ -n "${SVC_STATUSES[$i]:-}" ]]; then + status_json="\"${SVC_STATUSES[$i]}\"" + else + status_json="null" + fi + printf ' {"service": "%s", "url": "%s", "status": %s}%s\n' \ + "${SVC_NAMES[$i]}" "${SVC_URLS[$i]}" "$status_json" "$comma" + done + echo "]" +} + +# --- Argument parsing --------------------------------------------------------- + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + -e|--env) + [[ $# -lt 2 ]] && { error "--env requires a value"; exit 1; } + ENV_NAME="$2"; shift + ;; + --no-check) NO_CHECK=true ;; + --json) JSON_OUTPUT=true ;; + -v|--verbose) VERBOSE=true ;; + -q|--quiet) QUIET=true ;; + -h|--help) usage; exit 0 ;; + *) + error "Unknown option: $1" + echo "Run '${SCRIPT_NAME} --help' for usage." + exit 1 + ;; + esac + shift + done +} + +# --- Main --------------------------------------------------------------------- + +main() { + setup_colors + parse_args "$@" + detect_env + + # Collect services from all config files + collect_proxmox + collect_lab_vms + collect_haproxy + collect_awx + collect_observability + + if [[ ${#SVC_NAMES[@]} -eq 0 ]]; then + warn "No services found in envs/${ENV_NAME}/" + exit 0 + fi + + # Run reachability checks (parallel) + if [[ "$NO_CHECK" != true ]]; then + info "Checking ${#SVC_NAMES[@]} services..." + run_checks + fi + + # Output + if [[ "$JSON_OUTPUT" == true ]]; then + print_json + else + print_table + fi +} + +main "$@" diff --git a/envs/hetzner/awx.yaml b/envs/hetzner/awx.yaml index e66e330..e773b6a 100644 --- a/envs/hetzner/awx.yaml +++ b/envs/hetzner/awx.yaml @@ -22,4 +22,4 @@ awx: memory: 8GiB disk: 40GiB aether_url: https://10.10.0.120:8443 - aether_cluster_id: 52 + aether_cluster_id: 1 diff --git a/envs/hetzner/haproxy.yaml b/envs/hetzner/haproxy.yaml index c7755f5..1607074 100644 --- a/envs/hetzner/haproxy.yaml +++ b/envs/hetzner/haproxy.yaml @@ -4,8 +4,8 @@ # with a VIP on the UPLINK external range (10.10.0.200-210). # # Architecture: -# ffsdn-haproxy-52-01 (10.10.10.50) - HAProxy instance 1 on net-prod -# ffsdn-haproxy-52-02 (10.10.10.51) - HAProxy instance 2 on net-prod +# ffsdn-haproxy-1-01 (10.10.10.50) - HAProxy instance 1 on net-prod +# ffsdn-haproxy-1-02 (10.10.10.51) - HAProxy instance 2 on net-prod # nginx-lb-01/02/03 (10.10.10.60-62) - nginx test backends on net-prod # VIP: 10.10.0.200:80 - OVN external IP for external access # @@ -15,7 +15,7 @@ haproxy: cluster_remote: hz-cluster - cluster_id: 52 + cluster_id: 1 ovn_network: net-prod vip: 10.10.0.200 haproxy_01_ip: 10.10.10.50 diff --git a/envs/hetzner/observability.yaml b/envs/hetzner/observability.yaml index 32406d9..e579398 100644 --- a/envs/hetzner/observability.yaml +++ b/envs/hetzner/observability.yaml @@ -22,4 +22,4 @@ observability: node_exp_targets: hz-node-01,hz-node-02,hz-node-03 node_exp_ips: 10.10.10.71,10.10.10.72,10.10.10.73 haproxy_ips: 10.10.10.50,10.10.10.51 - haproxy_cluster_id: 52 + haproxy_cluster_id: 1 diff --git a/notes/aether-inventory-post-redeploy.md b/notes/aether-inventory-post-redeploy.md new file mode 100644 index 0000000..7a1904b --- /dev/null +++ b/notes/aether-inventory-post-redeploy.md @@ -0,0 +1,131 @@ +# Aether Inventory After Redeploy (Without DB Backup) + +**Date**: 2026-03-04 +**Target**: Hetzner lab, Aether at 10.10.0.120 +**Version**: 6.4.440 (up from 6.4.202 at time of redeploy) +**Activation code**: 443F4EEC6E088017 (new — DB reinstall generates a new one) + +## Summary + +Aether was redeployed from a golden image without restoring the previous +database. This inventory documents what survived (auto-discovered from +the live cluster) versus what was lost (stored only in the DB). + +### What survived (auto-discovered from cluster) + +These are things Aether reconstructs by connecting to the Incus cluster +and querying the API. They require no database state. + +| Feature | Status | Detail | +|---------|--------|--------| +| Cluster connection | **Present** | hz-cluster (ID 53, was 52 before redeploy) | +| Cluster members | **Present** | 3/3 nodes online, fully operational | +| Instance inventory | **Present** | 12 instances, all Running, correct IPs and locations | +| Instance tags | **Present** | HAProxy tags (ha-group, managed, role) visible | +| Storage pools | **Present** | local (zfs), 6% used (7GB/112.4GB) | +| Networking/OVN | **Present** | Visible via Manage INCUS Clusters > Networking | +| Cluster health | **Present** | Health dashboard shows CPU/mem/load per node | + +**Key insight**: Aether's core value — seeing your cluster and instances — +works immediately after a fresh deploy + cluster add. The Incus API is the +source of truth for all infrastructure state; Aether caches it but can +rebuild from scratch. + +### What was lost (DB-only state) + +These are things Aether stores in its PostgreSQL database that have no +external source of truth. They must be manually recreated. + +| Feature | Status | Detail | +|---------|--------|--------| +| Blueprints | **Empty** | "No blueprints created yet" — all designs gone | +| Deployed blueprints | **Empty** | No deployment history — tracking of which blueprints were deployed where is lost | +| Operations Center connections | **Empty** | No OC configured — previous OC link gone | +| Global firewall rules | **Empty** | 0 rules (API confirms: `[]`) | +| Cluster firewall rules | **Empty** | 0 rules (API confirms: `[]`) | +| RBAC users/roles | **Default only** | Only the built-in admin account exists | +| License | **Gone** | New activation code generated — previous license key invalid | +| Ledger / audit log | **Empty** | No historical log entries | +| Sync logs | **Empty** | No sync history | +| AWX job history | **Empty** | "No AWX job history yet" — past job runs not tracked | +| HAProxy image push state | **Not pushed** | Image v2.64 exists locally but shows "No Image" for hz-cluster — needs re-push | +| Auto-backup setting | **Off** | `auto_backup_enabled: False` (default) | + +### What was re-added manually (post-redeploy) + +These items exist in the current database because they were manually +reconfigured after the redeploy. + +| Feature | Status | Detail | +|---------|--------|--------| +| AWX endpoint | **Configured** | "lab-awx" at http://10.10.0.122:30080, created 2026-03-03 11:17:36 | +| AWX cluster config | **Configured** | hz-cluster linked to lab-awx, post-deploy template 9, decommission template 10, 600s timeout | +| AWX health | **Healthy** | Version 24.6.1, reachable | + +## Observations + +### Cluster ID changed (52 → 53) + +The previous Aether instance had the cluster registered as ID 52 (visible +in haproxy.yaml and awx.yaml configs). After redeploy, the new database +assigned it ID 53. This is a synthetic auto-increment ID internal to +Aether's DB. The cluster itself (certificate, nodes, instances) is +unchanged — only Aether's internal reference number differs. + +**Impact**: Scripts that hardcode `cluster_id: 52` (haproxy.yaml, awx.yaml) +would need updating if they interact with Aether's API using this ID. +The deploy scripts use the cluster_id for Aether API calls (HAProxy image +push, AWX cluster config). If these scripts are re-run against this Aether +instance, the ID mismatch would cause failures. + +### HAProxy image survived, push state didn't + +The HAProxy base image v2.64 (287 MB, built 2026-02-21) is stored on +Aether's local filesystem, not in the DB. It survived the redeploy. But +the DB record of which clusters it was pushed to was lost. The UI shows +"No Image" for hz-cluster, meaning the image needs to be re-pushed before +HAProxy LB management works through Aether. + +The HAProxy containers themselves (ffsdn-haproxy-52-01/02) are running +fine in the cluster — they don't depend on Aether for runtime operation. +Aether only manages their configuration and deployment lifecycle. + +### Settings are all defaults + +All settings reverted to defaults. Notable: +- `log_level: Debug` (default, verbose — consider changing to Info) +- `refresh_interval_minutes: 5` +- `auto_backup_enabled: False` — **should enable this time** +- `manage_instances: True` +- `manage_vcenters: False` +- `migration_manager: False` + +### What this tells us about Aether's architecture + +1. **Cluster state is ephemeral in Aether** — it's a cache of the Incus API. + Losing the DB doesn't lose infrastructure visibility. Re-add the cluster + and everything reappears within one sync cycle (5 min default). + +2. **Operational config is DB-only** — blueprints, ACL rules, RBAC, AWX + endpoints, OC connections, and deployment history live exclusively in + PostgreSQL. No export/import mechanism was observed. + +3. **The license is tied to the DB** — reinstalling generates a new + activation code, invalidating any existing license key. This is + explicitly noted on the licensing page. + +4. **HAProxy images are filesystem-based** — they survive DB loss but + lose their cluster push state. The containers in the cluster are + independent of Aether once deployed. + +5. **AWX integration is lightweight** — just an endpoint URL + template IDs. + Quick to re-add manually (as was done). Job history is lost but AWX + itself retains its own job history. + +## Recovery priority if this happens again + +1. Enable `auto_backup_enabled` immediately after setup +2. Periodically copy `/opt/ffsdn/backups/` off the Aether VM +3. After restore: re-add cluster, re-add AWX endpoint, re-push HAProxy image +4. Blueprints and ACL rules would need to be recreated from scratch + (consider documenting them externally)