#!/usr/bin/env bash # deploy-haproxy - Deploy and manage HAProxy load balancing on an Incus cluster # # Deploys test backends (nginx), builds a HAProxy image via Aether, # deploys HA infrastructure (2x HAProxy + OVN LB), and creates an # HTTP service with health checks. # # Uses Aether session-based authentication (not JWT) for HAProxy # management endpoints that are not in the public API. # # Usage: deploy-haproxy [OPTIONS] # Run 'deploy-haproxy --help' for full usage information. set -euo pipefail readonly VERSION="0.1.0" readonly SCRIPT_NAME="deploy-haproxy" readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # --------------------------------------------------------------------------- # Defaults (overridden by config file or CLI flags) # --------------------------------------------------------------------------- CLUSTER_REMOTE="" CLUSTER_ID="" OVN_NETWORK="net-prod" VIP="" HAPROXY_01_IP="" HAPROXY_02_IP="" HAPROXY_CPU="2" HAPROXY_MEMORY="1GB" BACKEND_IPS="" BACKEND_PORT="80" SERVICE_NAME="web-test" SERVICE_MODE="http" SERVICE_BALANCE="roundrobin" IMAGE_VERSION="1.0.0" AETHER_URL="" AETHER_USER="admin" AETHER_PASSWORD="" # Runtime flags DRY_RUN=false VERBOSE=false QUIET=false CONFIG_FILE="" ACTION="" SKIP_BACKENDS=false SKIP_IMAGE=false # Session state COOKIE_JAR="" CSRF_TOKEN="" # --------------------------------------------------------------------------- # Color and formatting # --------------------------------------------------------------------------- setup_colors() { if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[0;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' BOLD='\033[1m' DIM='\033[2m' RESET='\033[0m' else RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' RESET='' fi } info() { [[ "$QUIET" == true ]] && return; echo -e "${BLUE}[info]${RESET} $*"; } success() { [[ "$QUIET" == true ]] && return; echo -e "${GREEN}[ok]${RESET} $*"; } warn() { echo -e "${YELLOW}[warn]${RESET} $*" >&2; } error() { echo -e "${RED}[error]${RESET} $*" >&2; } step() { [[ "$QUIET" == true ]] && return; echo -e "${CYAN}[step]${RESET} ${BOLD}$*${RESET}"; } detail() { [[ "$VERBOSE" != true ]] && return; echo -e "${DIM} $*${RESET}"; } dry_run_guard() { if [[ "$DRY_RUN" == true ]]; then info "(dry-run) $*" return 1 fi return 0 } # --------------------------------------------------------------------------- # Help # --------------------------------------------------------------------------- usage() { cat </env or use --password" return 1 } # --------------------------------------------------------------------------- # Aether session authentication # --------------------------------------------------------------------------- aether_login() { step "Authenticating with Aether" COOKIE_JAR=$(mktemp /tmp/aether-haproxy-cookies.XXXXXX) detail "Cookie jar: ${COOKIE_JAR}" # Step 1: Get login page and extract CSRF token from hidden form field # Aether serves the login form at / (root), not /login # The form POSTs to /login local login_page login_page=$(curl -sSk -c "$COOKIE_JAR" "${AETHER_URL}/" 2>/dev/null) CSRF_TOKEN=$(echo "$login_page" \ | grep -o 'name="csrf_token" value="[^"]*"' \ | head -1 \ | sed 's/.*value="//;s/"//' || true) if [[ -z "$CSRF_TOKEN" ]]; then # Try alternate pattern (some versions use different attribute order) CSRF_TOKEN=$(echo "$login_page" \ | grep -o 'value="[^"]*" name="csrf_token"' \ | head -1 \ | sed 's/value="//;s/" name=.*//' || true) fi if [[ -z "$CSRF_TOKEN" ]]; then error "Failed to extract CSRF token from login page" error "Check that Aether is accessible at ${AETHER_URL}" return 1 fi detail "CSRF token: ${CSRF_TOKEN:0:20}..." # Step 2: POST login (form-urlencoded, NOT JSON) local login_code login_code=$(curl -sSk -b "$COOKIE_JAR" -c "$COOKIE_JAR" \ -X POST "${AETHER_URL}/login" \ -d "csrf_token=${CSRF_TOKEN}&username=${AETHER_USER}&password=${AETHER_PASSWORD}" \ -o /dev/null -w '%{http_code}' \ -L 2>/dev/null) detail "Login response: HTTP ${login_code}" # Step 3: Refresh CSRF token from an authenticated page local haproxy_page haproxy_page=$(curl -sSk -b "$COOKIE_JAR" "${AETHER_URL}/haproxy" 2>/dev/null) CSRF_TOKEN=$(echo "$haproxy_page" \ | grep -o 'name="csrf_token" value="[^"]*"' \ | head -1 \ | sed 's/.*value="//;s/"//' || true) if [[ -z "$CSRF_TOKEN" ]]; then error "Failed to get CSRF token after login — authentication may have failed" error "Verify password is correct (AETHER_ADMIN_PASSWORD in env file)" return 1 fi success "Authenticated with Aether (session + CSRF)" detail "CSRF token: ${CSRF_TOKEN:0:20}..." } aether_api() { local method="$1" path="$2" shift 2 curl -sSk -b "$COOKIE_JAR" \ -H "X-CSRF-Token: ${CSRF_TOKEN}" \ -H "Referer: ${AETHER_URL}/haproxy" \ -X "$method" "${AETHER_URL}${path}" \ "$@" } aether_cleanup() { if [[ -n "${COOKIE_JAR:-}" && -f "${COOKIE_JAR:-}" ]]; then rm -f "$COOKIE_JAR" fi } # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- # Split BACKEND_IPS into array get_backend_ips() { echo "$BACKEND_IPS" | tr ',' ' ' } get_backend_count() { local ips ips=$(get_backend_ips) echo "$ips" | wc -w } backend_name() { local index="$1" printf "nginx-lb-%02d" "$index" } haproxy_container_01() { echo "ffsdn-haproxy-${CLUSTER_ID}-01" } haproxy_container_02() { echo "ffsdn-haproxy-${CLUSTER_ID}-02" } container_exists() { local name="$1" incus info "${CLUSTER_REMOTE}:${name}" &>/dev/null 2>&1 } container_running() { local name="$1" local state state=$(incus info "${CLUSTER_REMOTE}:${name}" 2>/dev/null | grep -i '^status:' | awk '{print $2}') [[ "$state" == "RUNNING" || "$state" == "Running" ]] } # --------------------------------------------------------------------------- # Doctor — prerequisite checks # --------------------------------------------------------------------------- action_doctor() { step "Checking prerequisites" local ok=true # incus CLI if command -v incus &>/dev/null; then success "incus CLI: $(incus version 2>/dev/null || echo 'installed')" else error "incus CLI not found — install the Incus client" ok=false fi # curl if command -v curl &>/dev/null; then success "curl: $(curl --version | head -1 | awk '{print $2}')" else error "curl not found" ok=false fi # python3 if command -v python3 &>/dev/null; then success "python3: $(python3 --version 2>&1 | awk '{print $2}')" else error "python3 not found" ok=false fi # Aether password if load_password 2>/dev/null; then success "Aether password: configured" else error "Aether password: not found (set AETHER_ADMIN_PASSWORD in env file)" ok=false fi # Cluster remote if incus remote list -f csv 2>/dev/null | grep -q "^${CLUSTER_REMOTE},"; then success "Incus remote '${CLUSTER_REMOTE}': reachable" else warn "Incus remote '${CLUSTER_REMOTE}' not found (needed for deployment)" fi # Aether accessibility (login page is at root /) local aether_code aether_code=$(curl -sk -o /dev/null -w '%{http_code}' "${AETHER_URL}/" 2>/dev/null || echo "000") if [[ "$aether_code" == "200" || "$aether_code" == "302" ]]; then success "Aether: reachable at ${AETHER_URL} (HTTP ${aether_code})" else error "Aether: not reachable at ${AETHER_URL} (HTTP ${aether_code})" ok=false fi # Existing HAProxy containers if container_exists "$(haproxy_container_01)"; then info "HAProxy infrastructure exists on cluster ${CLUSTER_ID}" if container_running "$(haproxy_container_01)"; then success " $(haproxy_container_01): running" else warn " $(haproxy_container_01): not running" fi if container_running "$(haproxy_container_02)"; then success " $(haproxy_container_02): running" else warn " $(haproxy_container_02): not running" fi else info "No HAProxy infrastructure on cluster ${CLUSTER_ID} (--deploy to create)" fi # Existing backends local idx=1 for ip in $(get_backend_ips); do local name name=$(backend_name "$idx") if container_exists "$name"; then success " Backend ${name} (${ip}): exists" else info " Backend ${name} (${ip}): not deployed" fi idx=$((idx + 1)) done echo if [[ "$ok" == true ]]; then success "All prerequisites satisfied" else error "Some prerequisites missing — fix the issues above" return 1 fi } # --------------------------------------------------------------------------- # Deploy — full deployment pipeline # --------------------------------------------------------------------------- action_deploy() { step "Deploying HAProxy load balancing on cluster ${CLUSTER_ID}" echo info " Cluster: ${CLUSTER_REMOTE} (ID: ${CLUSTER_ID})" info " OVN network: ${OVN_NETWORK}" info " VIP: ${VIP}" info " HAProxy: ${HAPROXY_01_IP}, ${HAPROXY_02_IP}" info " Backends: ${BACKEND_IPS}" info " Service: ${SERVICE_NAME} (${SERVICE_MODE}, ${SERVICE_BALANCE})" echo load_password # Check existing infrastructure if container_exists "$(haproxy_container_01)"; then error "HAProxy infrastructure already exists ($(haproxy_container_01))" error "Use --status to check health, or --cleanup first." return 1 fi if [[ "$SKIP_BACKENDS" != true ]]; then phase_deploy_backends else info "Skipping backend deployment (--skip-backends)" fi # Authenticate with Aether for remaining phases if [[ "$DRY_RUN" != true ]]; then aether_login trap aether_cleanup EXIT fi if [[ "$SKIP_IMAGE" != true ]]; then phase_build_image else info "Skipping image build (--skip-image)" fi phase_deploy_infrastructure phase_create_service phase_verify echo success "HAProxy deployment complete!" info " VIP: http://${VIP}/" info " Service: ${SERVICE_NAME}" info " Backends: $(get_backend_count)" echo info "Next steps:" info " curl http://${VIP}/ # test load balancing" info " ${SCRIPT_NAME} --status # check health" } # --------------------------------------------------------------------------- # Phase 1: Deploy test backends # --------------------------------------------------------------------------- phase_deploy_backends() { step "Phase 1: Deploy test backends" local idx=1 for ip in $(get_backend_ips); do local name name=$(backend_name "$idx") if container_exists "$name"; then if container_running "$name"; then success "Backend ${name} already running" idx=$((idx + 1)) continue else warn "Backend ${name} exists but not running — starting" if dry_run_guard "incus start ${CLUSTER_REMOTE}:${name}"; then incus start "${CLUSTER_REMOTE}:${name}" fi idx=$((idx + 1)) continue fi fi if ! dry_run_guard "Launch ${name} on ${OVN_NETWORK} at ${ip}"; then idx=$((idx + 1)) continue fi info "Launching ${name} (${ip})..." incus launch images:debian/12 "${CLUSTER_REMOTE}:${name}" \ -n "$OVN_NETWORK" # Wait for container agent local elapsed=0 while ! incus exec "${CLUSTER_REMOTE}:${name}" -- true &>/dev/null 2>&1; do sleep 2 elapsed=$((elapsed + 2)) if [[ $elapsed -ge 60 ]]; then error "Container agent not available after 60s" return 1 fi done # Configure backend incus exec "${CLUSTER_REMOTE}:${name}" -- bash -c " # Wait for initial network sleep 3 # Install nginx + curl (curl needed for verification) export DEBIAN_FRONTEND=noninteractive apt-get update -qq apt-get install -y -qq nginx curl 2>&1 | tail -1 # Create unique index page cat > /var/www/html/index.html << HTML

Backend: ${name}

IP: ${ip}

HTML systemctl enable nginx systemctl start nginx " # Set static IP incus exec "${CLUSTER_REMOTE}:${name}" -- bash -c " cat > /etc/systemd/network/10-static.network << NETCFG [Match] Name=eth0 [Network] Address=${ip}/24 Gateway=10.10.10.1 DNS=10.10.10.1 NETCFG systemctl restart systemd-networkd " success "Backend ${name} deployed at ${ip}" idx=$((idx + 1)) done # Verify backends info "Waiting for backends to stabilize..." sleep 5 local all_ok=true idx=1 for ip in $(get_backend_ips); do local name name=$(backend_name "$idx") if container_running "$name"; then detail "Backend ${name}: running" else warn "Backend ${name}: not running" all_ok=false fi idx=$((idx + 1)) done if [[ "$all_ok" == true ]]; then success "All $(get_backend_count) backends deployed" else warn "Some backends may not be ready" fi } # --------------------------------------------------------------------------- # Phase 2: Build HAProxy image # --------------------------------------------------------------------------- phase_build_image() { step "Phase 2: Build HAProxy image" if ! dry_run_guard "Build HAProxy image v${IMAGE_VERSION} on cluster ${CLUSTER_ID}"; then return 0 fi # Check if image already exists local images_response images_response=$(aether_api GET "/haproxy/images" 2>/dev/null || echo "[]") local existing_version existing_version=$(echo "$images_response" \ | python3 -c " import sys, json try: imgs = json.load(sys.stdin) if isinstance(imgs, list): for img in imgs: if img.get('version') == '${IMAGE_VERSION}': print(img.get('id', '')) break except: pass " 2>/dev/null || true) if [[ -n "$existing_version" ]]; then success "Image v${IMAGE_VERSION} already exists (ID: ${existing_version})" info "Ensuring image is pushed to cluster and set as current..." aether_api POST "/haproxy/image/push" \ -H "Content-Type: application/json" \ -d "{\"image_id\": ${existing_version}, \"cluster_id\": ${CLUSTER_ID}}" \ 2>/dev/null || true aether_api POST "/haproxy/image/set-current" \ -H "Content-Type: application/json" \ -d "{\"image_id\": ${existing_version}}" \ 2>/dev/null || true success "Image v${IMAGE_VERSION} ready on cluster ${CLUSTER_ID}" return 0 fi # Build new image info "Building HAProxy image v${IMAGE_VERSION} on cluster ${CLUSTER_ID}..." info "This takes 5-10 minutes..." local build_response build_response=$(aether_api POST "/haproxy/image/build" \ -H "Content-Type: application/json" \ -d "{\"cluster_id\": ${CLUSTER_ID}, \"version\": \"${IMAGE_VERSION}\"}" \ 2>/dev/null) detail "Build response: ${build_response}" # Poll build status local elapsed=0 local timeout=600 local status="" while true; do status=$(aether_api GET "/haproxy/image/build-status" 2>/dev/null \ | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('status','unknown'))" \ 2>/dev/null || echo "unknown") detail "Build status: ${status} (${elapsed}s)" if [[ "$status" == "completed" || "$status" == "success" ]]; then success "Image build completed" break elif [[ "$status" == "failed" || "$status" == "error" ]]; then error "Image build failed" local build_error build_error=$(aether_api GET "/haproxy/image/build-status" 2>/dev/null \ | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('error','unknown'))" \ 2>/dev/null || true) [[ -n "$build_error" ]] && error "Error: ${build_error}" return 1 fi sleep 15 elapsed=$((elapsed + 15)) if [[ $elapsed -ge $timeout ]]; then error "Image build timed out after ${timeout}s" return 1 fi done # Get new image ID local image_id image_id=$(aether_api GET "/haproxy/images" 2>/dev/null \ | python3 -c " import sys, json try: imgs = json.load(sys.stdin) if isinstance(imgs, list): for img in imgs: if img.get('version') == '${IMAGE_VERSION}': print(img['id']) break except: pass " 2>/dev/null || true) if [[ -z "$image_id" ]]; then error "Cannot find built image — check Aether UI" return 1 fi # Push to cluster info "Pushing image to cluster ${CLUSTER_ID}..." aether_api POST "/haproxy/image/push" \ -H "Content-Type: application/json" \ -d "{\"image_id\": ${image_id}, \"cluster_id\": ${CLUSTER_ID}}" \ 2>/dev/null || true # Set as current info "Setting as current version..." aether_api POST "/haproxy/image/set-current" \ -H "Content-Type: application/json" \ -d "{\"image_id\": ${image_id}}" \ 2>/dev/null || true success "Image v${IMAGE_VERSION} built and pushed (ID: ${image_id})" } # --------------------------------------------------------------------------- # Phase 3: Deploy HAProxy infrastructure # --------------------------------------------------------------------------- phase_deploy_infrastructure() { step "Phase 3: Deploy HAProxy infrastructure" if ! dry_run_guard "Deploy infrastructure: VIP=${VIP}, HAProxy=${HAPROXY_01_IP}/${HAPROXY_02_IP}"; then return 0 fi # Check existing infrastructure local infra_response infra_response=$(aether_api GET "/haproxy/infrastructure/${CLUSTER_ID}" 2>/dev/null || echo "{}") local infra_exists infra_exists=$(echo "$infra_response" \ | python3 -c " import sys, json try: d = json.load(sys.stdin) if d.get('deployed') or d.get('haproxy_01_ip'): print('yes') else: print('no') except: print('no') " 2>/dev/null || echo "no") if [[ "$infra_exists" == "yes" ]]; then success "Infrastructure already deployed on cluster ${CLUSTER_ID}" return 0 fi info "Deploying infrastructure..." info " OVN network: ${OVN_NETWORK}" info " VIP: ${VIP}" info " HAProxy 01: ${HAPROXY_01_IP}" info " HAProxy 02: ${HAPROXY_02_IP}" info " CPU: ${HAPROXY_CPU}" info " Memory: ${HAPROXY_MEMORY}" local deploy_response deploy_response=$(aether_api POST "/haproxy/infrastructure/deploy" \ -H "Content-Type: application/json" \ -d "{ \"cluster_id\": ${CLUSTER_ID}, \"ovn_network\": \"${OVN_NETWORK}\", \"vip\": \"${VIP}\", \"haproxy_01_ip\": \"${HAPROXY_01_IP}\", \"haproxy_02_ip\": \"${HAPROXY_02_IP}\", \"cpu_limit\": \"${HAPROXY_CPU}\", \"memory_limit\": \"${HAPROXY_MEMORY}\" }" 2>/dev/null) detail "Deploy response: ${deploy_response}" # Verify containers appeared info "Waiting for HAProxy containers to start..." local elapsed=0 local timeout=120 while true; do if container_exists "$(haproxy_container_01)" && container_exists "$(haproxy_container_02)"; then break fi sleep 5 elapsed=$((elapsed + 5)) if [[ $elapsed -ge $timeout ]]; then error "HAProxy containers not found after ${timeout}s" error "Check Aether UI for deployment errors" return 1 fi detail "Waiting for containers... (${elapsed}s)" done # Wait for running state elapsed=0 while true; do if container_running "$(haproxy_container_01)" && container_running "$(haproxy_container_02)"; then break fi sleep 5 elapsed=$((elapsed + 5)) if [[ $elapsed -ge $timeout ]]; then warn "HAProxy containers exist but not all running after ${timeout}s" break fi done success "Infrastructure deployed" success " $(haproxy_container_01): $(container_running "$(haproxy_container_01)" && echo "running" || echo "not running")" success " $(haproxy_container_02): $(container_running "$(haproxy_container_02)" && echo "running" || echo "not running")" } # --------------------------------------------------------------------------- # Phase 4: Create HTTP service # --------------------------------------------------------------------------- phase_create_service() { step "Phase 4: Create HTTP service" if ! dry_run_guard "Create service: ${SERVICE_NAME} on ${VIP}:${BACKEND_PORT}"; then return 0 fi # Build backends JSON local backends_json="[" local idx=1 local first=true for ip in $(get_backend_ips); do local name name=$(backend_name "$idx") if [[ "$first" == true ]]; then first=false else backends_json+="," fi backends_json+="{\"name\":\"${name}\",\"ip\":\"${ip}\",\"port\":${BACKEND_PORT},\"weight\":100}" idx=$((idx + 1)) done backends_json+="]" detail "Backends JSON: ${backends_json}" info "Creating service '${SERVICE_NAME}'..." info " VIP: ${VIP}:${BACKEND_PORT}" info " Mode: ${SERVICE_MODE}" info " Balance: ${SERVICE_BALANCE}" info " Health: enabled" info " Backends: $(get_backend_count)" local service_response service_response=$(aether_api POST "/haproxy/service" \ -H "Content-Type: application/json" \ -d "{ \"cluster_id\": ${CLUSTER_ID}, \"name\": \"${SERVICE_NAME}\", \"description\": \"HTTP load balancer for nginx test backends\", \"vip\": \"${VIP}\", \"listen_port\": ${BACKEND_PORT}, \"mode\": \"${SERVICE_MODE}\", \"balance\": \"${SERVICE_BALANCE}\", \"health_check\": true, \"sticky_sessions\": false, \"backends\": ${backends_json} }" 2>/dev/null) detail "Service response: ${service_response}" success "Service '${SERVICE_NAME}' created" } # --------------------------------------------------------------------------- # Phase 5: Verify # --------------------------------------------------------------------------- phase_verify() { step "Phase 5: Verify deployment" info "Waiting for HAProxy to configure backends..." sleep 10 # Test VIP info "Testing VIP at http://${VIP}/..." local response response=$(curl -sk --connect-timeout 5 --max-time 10 "http://${VIP}/" 2>/dev/null || true) if [[ -n "$response" ]]; then local backend_hit backend_hit=$(echo "$response" | grep -o 'Backend: [^<]*' || echo "unknown") success "VIP responding: ${backend_hit}" else warn "VIP not responding yet — HAProxy may still be configuring" warn "Try: curl http://${VIP}/" fi # Test distribution if [[ -n "$response" ]]; then info "Testing traffic distribution (6 requests)..." local hits="" for i in $(seq 1 6); do local hit hit=$(curl -sk --connect-timeout 5 --max-time 10 "http://${VIP}/" 2>/dev/null \ | grep -o 'Backend: [^<]*' || echo "no response") detail " Request ${i}: ${hit}" hits="${hits}${hit}\n" done local unique_backends unique_backends=$(echo -e "$hits" | sort -u | grep -c 'Backend:' || echo 0) if [[ "$unique_backends" -gt 1 ]]; then success "Traffic distributed across ${unique_backends} backends" elif [[ "$unique_backends" -eq 1 ]]; then info "Traffic going to 1 backend (health checks may not have completed yet)" else warn "No successful responses — check HAProxy and backend health" fi fi } # --------------------------------------------------------------------------- # Status — health checks # --------------------------------------------------------------------------- action_status() { step "HAProxy Status for cluster ${CLUSTER_ID}" echo # HAProxy containers if container_exists "$(haproxy_container_01)"; then if container_running "$(haproxy_container_01)"; then success "$(haproxy_container_01): running" else error "$(haproxy_container_01): not running" fi else error "$(haproxy_container_01): not found (no infrastructure deployed)" return 1 fi if container_exists "$(haproxy_container_02)"; then if container_running "$(haproxy_container_02)"; then success "$(haproxy_container_02): running" else warn "$(haproxy_container_02): not running" fi else warn "$(haproxy_container_02): not found" fi # Backend containers echo info "Backends:" local idx=1 for ip in $(get_backend_ips); do local name name=$(backend_name "$idx") if container_exists "$name"; then if container_running "$name"; then success " ${name} (${ip}): running" else warn " ${name} (${ip}): not running" fi else info " ${name} (${ip}): not deployed" fi idx=$((idx + 1)) done # VIP connectivity echo info "VIP connectivity:" local vip_code vip_code=$(curl -sk -o /dev/null -w '%{http_code}' --connect-timeout 5 \ "http://${VIP}/" 2>/dev/null || echo "000") if [[ "$vip_code" == "200" ]]; then success "VIP http://${VIP}/ responding (HTTP ${vip_code})" else warn "VIP http://${VIP}/ not responding (HTTP ${vip_code})" fi # Aether health stats (requires session auth) echo if load_password 2>/dev/null; then aether_login 2>/dev/null trap aether_cleanup EXIT local health_response health_response=$(aether_api GET \ "/haproxy/service/health-stats?cluster_id=${CLUSTER_ID}&service_name=${SERVICE_NAME}" \ 2>/dev/null || echo "{}") local has_stats has_stats=$(echo "$health_response" \ | python3 -c " import sys, json try: d = json.load(sys.stdin) if d and not d.get('error'): print('yes') else: print('no') except: print('no') " 2>/dev/null || echo "no") if [[ "$has_stats" == "yes" ]]; then info "Service health (from Aether):" echo "$health_response" | python3 -c " import sys, json try: d = json.load(sys.stdin) # Print backend stats if available backends = d.get('backends', d.get('backend_stats', [])) if isinstance(backends, list): for b in backends: name = b.get('name', b.get('server', 'unknown')) status = b.get('status', 'unknown') icon = 'UP' if status.upper() == 'UP' else 'DOWN' print(f' {name}: {icon}') # Print frontend stats if available frontend = d.get('frontend', d.get('frontend_stats', {})) if isinstance(frontend, dict) and frontend: sessions = frontend.get('scur', frontend.get('current_sessions', '?')) print(f' Frontend sessions: {sessions}') except Exception as e: print(f' (could not parse stats: {e})') " 2>/dev/null || warn " Could not parse health stats" else info "No health stats available (service may not exist yet)" fi else info "Skipping Aether health stats (no password configured)" fi } # --------------------------------------------------------------------------- # Cleanup — remove everything # --------------------------------------------------------------------------- action_cleanup() { step "Cleaning up HAProxy deployment" echo local has_infra=false local has_backends=false if container_exists "$(haproxy_container_01)"; then has_infra=true fi local idx=1 for ip in $(get_backend_ips); do if container_exists "$(backend_name "$idx")"; then has_backends=true break fi idx=$((idx + 1)) done if [[ "$has_infra" == false && "$has_backends" == false ]]; then info "Nothing to clean up" return 0 fi echo if [[ "$has_infra" == true ]]; then warn "This will delete HAProxy infrastructure ($(haproxy_container_01), $(haproxy_container_02))" warn "All services and VIPs will be removed." fi if [[ "$has_backends" == true ]]; then warn "This will delete test backend containers." fi echo if [[ "$DRY_RUN" == true ]]; then info "(dry-run) Would delete infrastructure and backends" return 0 fi read -r -p "Type 'yes' to confirm: " confirm if [[ "$confirm" != "yes" ]]; then info "Aborted" return 0 fi # Delete infrastructure via Aether API (removes containers, VIPs, services) if [[ "$has_infra" == true ]]; then step "Removing HAProxy infrastructure via Aether" if load_password 2>/dev/null; then aether_login trap aether_cleanup EXIT local delete_response delete_response=$(aether_api DELETE "/haproxy/infrastructure/${CLUSTER_ID}" \ -H "Content-Type: application/json" \ 2>/dev/null || echo "") detail "Delete response: ${delete_response}" # Wait for containers to be removed local elapsed=0 while container_exists "$(haproxy_container_01)" && [[ $elapsed -lt 60 ]]; do sleep 5 elapsed=$((elapsed + 5)) detail "Waiting for cleanup... (${elapsed}s)" done if ! container_exists "$(haproxy_container_01)"; then success "HAProxy infrastructure removed" else warn "Infrastructure may not be fully removed — check Aether UI" fi else warn "Cannot authenticate with Aether — deleting containers directly" if container_running "$(haproxy_container_01)"; then incus stop "${CLUSTER_REMOTE}:$(haproxy_container_01)" --force 2>/dev/null || true fi incus delete "${CLUSTER_REMOTE}:$(haproxy_container_01)" --force 2>/dev/null || true if container_running "$(haproxy_container_02)"; then incus stop "${CLUSTER_REMOTE}:$(haproxy_container_02)" --force 2>/dev/null || true fi incus delete "${CLUSTER_REMOTE}:$(haproxy_container_02)" --force 2>/dev/null || true success "HAProxy containers deleted directly" fi fi # Delete backends if [[ "$has_backends" == true ]]; then step "Removing test backends" idx=1 for ip in $(get_backend_ips); do local name name=$(backend_name "$idx") if container_exists "$name"; then info "Deleting ${name}..." incus delete "${CLUSTER_REMOTE}:${name}" --force 2>/dev/null || true success "Deleted ${name}" fi idx=$((idx + 1)) done fi echo success "Cleanup complete" } # --------------------------------------------------------------------------- # Argument parsing # --------------------------------------------------------------------------- parse_args() { while [[ $# -gt 0 ]]; do case "$1" in --deploy) ACTION="deploy";; --status) ACTION="status";; --cleanup) ACTION="cleanup";; --doctor) ACTION="doctor";; --skip-backends) SKIP_BACKENDS=true;; --skip-image) SKIP_IMAGE=true;; -c|--config) CONFIG_FILE="$2"; shift;; -p|--password) AETHER_PASSWORD="$2"; shift;; -n|--dry-run) DRY_RUN=true;; -v|--verbose) VERBOSE=true;; -q|--quiet) QUIET=true;; -h|--help) usage; exit 0;; *) error "Unknown option: $1" echo "Run '${SCRIPT_NAME} --help' for usage." exit 1 ;; esac shift done if [[ -z "$ACTION" ]]; then error "No action specified" echo "Run '${SCRIPT_NAME} --help' for usage." exit 1 fi } # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- require_config() { local missing=() [[ -z "$CLUSTER_REMOTE" ]] && missing+=("cluster_remote") [[ -z "$CLUSTER_ID" ]] && missing+=("cluster_id") [[ -z "$VIP" ]] && missing+=("vip") [[ -z "$HAPROXY_01_IP" ]] && missing+=("haproxy_01_ip") [[ -z "$HAPROXY_02_IP" ]] && missing+=("haproxy_02_ip") [[ -z "$BACKEND_IPS" ]] && missing+=("backend_ips") [[ -z "$AETHER_URL" ]] && missing+=("aether_url") if [[ ${#missing[@]} -gt 0 ]]; then error "Required config missing: ${missing[*]}" error "Use --config FILE to load environment settings (see envs/)" exit 1 fi } main() { setup_colors parse_args "$@" # Load config file if provided if [[ -n "$CONFIG_FILE" ]]; then parse_config "$CONFIG_FILE" fi # Validate required fields (must come from --config or CLI flags) if [[ "$ACTION" != "doctor" ]]; then require_config fi echo case "$ACTION" in deploy) action_deploy;; status) action_status;; cleanup) action_cleanup;; doctor) action_doctor;; esac } main "$@"