maarten
/
incus-contrib
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
incus-contrib/incusos/deploy-haproxy

1212 lines
38 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
# deploy-haproxy - Deploy and manage HAProxy load balancing on an Incus cluster
#
# Deploys test backends (nginx), builds a HAProxy image via Aether,
# deploys HA infrastructure (2x HAProxy + OVN LB), and creates an
# HTTP service with health checks.
#
# Uses Aether session-based authentication (not JWT) for HAProxy
# management endpoints that are not in the public API.
#
# Usage: deploy-haproxy [OPTIONS]
# Run 'deploy-haproxy --help' for full usage information.
set -euo pipefail
readonly VERSION="0.1.0"
readonly SCRIPT_NAME="deploy-haproxy"
readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly ENV_FILE="${SCRIPT_DIR}/../env"
# ---------------------------------------------------------------------------
# Defaults (overridden by config file or CLI flags)
# ---------------------------------------------------------------------------
CLUSTER_REMOTE="oc-node-01"
CLUSTER_ID="52"
OVN_NETWORK="net-prod"
VIP="192.168.103.200"
HAPROXY_01_IP="10.10.10.50"
HAPROXY_02_IP="10.10.10.51"
HAPROXY_CPU="2"
HAPROXY_MEMORY="1GB"
BACKEND_IPS="10.10.10.60,10.10.10.61,10.10.10.62"
BACKEND_PORT="80"
SERVICE_NAME="web-test"
SERVICE_MODE="http"
SERVICE_BALANCE="roundrobin"
IMAGE_VERSION="1.0.0"
AETHER_URL="https://192.168.102.160:8443"
AETHER_USER="admin"
AETHER_PASSWORD=""
# Runtime flags
DRY_RUN=false
VERBOSE=false
QUIET=false
CONFIG_FILE=""
ACTION=""
SKIP_BACKENDS=false
SKIP_IMAGE=false
# Session state
COOKIE_JAR=""
CSRF_TOKEN=""
# ---------------------------------------------------------------------------
# Color and formatting
# ---------------------------------------------------------------------------
setup_colors() {
if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[0;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
BOLD='\033[1m'
DIM='\033[2m'
RESET='\033[0m'
else
RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' RESET=''
fi
}
info() { [[ "$QUIET" == true ]] && return; echo -e "${BLUE}[info]${RESET} $*"; }
success() { [[ "$QUIET" == true ]] && return; echo -e "${GREEN}[ok]${RESET} $*"; }
warn() { echo -e "${YELLOW}[warn]${RESET} $*" >&2; }
error() { echo -e "${RED}[error]${RESET} $*" >&2; }
step() { [[ "$QUIET" == true ]] && return; echo -e "${CYAN}[step]${RESET} ${BOLD}$*${RESET}"; }
detail() { [[ "$VERBOSE" != true ]] && return; echo -e "${DIM} $*${RESET}"; }
dry_run_guard() {
if [[ "$DRY_RUN" == true ]]; then
info "(dry-run) $*"
return 1
fi
return 0
}
# ---------------------------------------------------------------------------
# Help
# ---------------------------------------------------------------------------
usage() {
cat <<EOF
${BOLD}${SCRIPT_NAME}${RESET} v${VERSION} - Deploy HAProxy load balancing on an Incus cluster
${BOLD}USAGE${RESET}
${SCRIPT_NAME} [OPTIONS]
${BOLD}ACTIONS${RESET}
--deploy Full deploy: backends, image, infrastructure, service
--status Check HAProxy health, backends, service stats
--cleanup Remove HAProxy infrastructure and test backends
--doctor Check prerequisites (incus CLI, curl, Aether access)
${BOLD}OPTIONS${RESET}
-c, --config FILE Configuration file
-p, --password PASS Aether admin password (default: from env file)
--skip-backends Skip backend deployment (use existing)
--skip-image Skip image build (use existing)
-n, --dry-run Preview actions without executing
-v, --verbose Show detailed output
-q, --quiet Suppress informational output
-h, --help Show this help message
${BOLD}CONFIGURATION${RESET}
The script uses built-in defaults for the lab environment. Override with
a config file or CLI flags. Password is read from the 'env' file
(AETHER_ADMIN_PASSWORD) or passed via --password.
Defaults:
Cluster remote: ${CLUSTER_REMOTE}
Cluster ID: ${CLUSTER_ID}
OVN network: ${OVN_NETWORK}
VIP: ${VIP}
HAProxy IPs: ${HAPROXY_01_IP}, ${HAPROXY_02_IP}
Backend IPs: ${BACKEND_IPS}
Service: ${SERVICE_NAME} (${SERVICE_MODE}, ${SERVICE_BALANCE})
Image version: ${IMAGE_VERSION}
Aether: ${AETHER_URL}
${BOLD}EXAMPLES${RESET}
# Check prerequisites
${SCRIPT_NAME} --doctor
# Full deployment with defaults
${SCRIPT_NAME} --deploy
# Preview deployment
${SCRIPT_NAME} --deploy --dry-run
# Deploy without rebuilding image
${SCRIPT_NAME} --deploy --skip-image
# Deploy without creating new backends
${SCRIPT_NAME} --deploy --skip-backends
# Check health
${SCRIPT_NAME} --status
# Clean up everything
${SCRIPT_NAME} --cleanup
EOF
}
# ---------------------------------------------------------------------------
# Config parsing
# ---------------------------------------------------------------------------
parse_config() {
local file="$1"
if [[ ! -f "$file" ]]; then
error "Config file not found: $file"
exit 1
fi
local val
val=$(grep -A50 '^haproxy:' "$file" | grep '^ *cluster_remote:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
[[ -n "$val" ]] && CLUSTER_REMOTE="$val"
val=$(grep -A50 '^haproxy:' "$file" | grep '^ *cluster_id:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
[[ -n "$val" ]] && CLUSTER_ID="$val"
val=$(grep -A50 '^haproxy:' "$file" | grep '^ *ovn_network:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
[[ -n "$val" ]] && OVN_NETWORK="$val"
val=$(grep -A50 '^haproxy:' "$file" | grep '^ *vip:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
[[ -n "$val" ]] && VIP="$val"
val=$(grep -A50 '^haproxy:' "$file" | grep '^ *haproxy_01_ip:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
[[ -n "$val" ]] && HAPROXY_01_IP="$val"
val=$(grep -A50 '^haproxy:' "$file" | grep '^ *haproxy_02_ip:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
[[ -n "$val" ]] && HAPROXY_02_IP="$val"
val=$(grep -A50 '^haproxy:' "$file" | grep '^ *backend_ips:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
[[ -n "$val" ]] && BACKEND_IPS="$val"
val=$(grep -A50 '^haproxy:' "$file" | grep '^ *service_name:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
[[ -n "$val" ]] && SERVICE_NAME="$val"
val=$(grep -A50 '^haproxy:' "$file" | grep '^ *image_version:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
[[ -n "$val" ]] && IMAGE_VERSION="$val"
val=$(grep -A50 '^haproxy:' "$file" | grep '^ *aether_url:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
[[ -n "$val" ]] && AETHER_URL="$val"
}
# ---------------------------------------------------------------------------
# Load password from env file
# ---------------------------------------------------------------------------
load_password() {
if [[ -n "$AETHER_PASSWORD" ]]; then
return 0
fi
if [[ -f "$ENV_FILE" ]]; then
local val
val=$(grep 'AETHER_ADMIN_PASSWORD=' "$ENV_FILE" | head -1 | sed 's/^.*AETHER_ADMIN_PASSWORD=//' | tr -d '"' | tr -d "'" || true)
if [[ -n "$val" ]]; then
AETHER_PASSWORD="$val"
detail "Password loaded from env file"
return 0
fi
fi
error "Aether password not found"
error "Set AETHER_ADMIN_PASSWORD in ${ENV_FILE} or use --password"
return 1
}
# ---------------------------------------------------------------------------
# Aether session authentication
# ---------------------------------------------------------------------------
aether_login() {
step "Authenticating with Aether"
COOKIE_JAR=$(mktemp /tmp/aether-haproxy-cookies.XXXXXX)
detail "Cookie jar: ${COOKIE_JAR}"
# Step 1: Get login page and extract CSRF token from hidden form field
# Aether serves the login form at / (root), not /login
# The form POSTs to /login
local login_page
login_page=$(curl -sSk -c "$COOKIE_JAR" "${AETHER_URL}/" 2>/dev/null)
CSRF_TOKEN=$(echo "$login_page" \
| grep -o 'name="csrf_token" value="[^"]*"' \
| head -1 \
| sed 's/.*value="//;s/"//' || true)
if [[ -z "$CSRF_TOKEN" ]]; then
# Try alternate pattern (some versions use different attribute order)
CSRF_TOKEN=$(echo "$login_page" \
| grep -o 'value="[^"]*" name="csrf_token"' \
| head -1 \
| sed 's/value="//;s/" name=.*//' || true)
fi
if [[ -z "$CSRF_TOKEN" ]]; then
error "Failed to extract CSRF token from login page"
error "Check that Aether is accessible at ${AETHER_URL}"
return 1
fi
detail "CSRF token: ${CSRF_TOKEN:0:20}..."
# Step 2: POST login (form-urlencoded, NOT JSON)
local login_code
login_code=$(curl -sSk -b "$COOKIE_JAR" -c "$COOKIE_JAR" \
-X POST "${AETHER_URL}/login" \
-d "csrf_token=${CSRF_TOKEN}&username=${AETHER_USER}&password=${AETHER_PASSWORD}" \
-o /dev/null -w '%{http_code}' \
-L 2>/dev/null)
detail "Login response: HTTP ${login_code}"
# Step 3: Refresh CSRF token from an authenticated page
local haproxy_page
haproxy_page=$(curl -sSk -b "$COOKIE_JAR" "${AETHER_URL}/haproxy" 2>/dev/null)
CSRF_TOKEN=$(echo "$haproxy_page" \
| grep -o 'name="csrf_token" value="[^"]*"' \
| head -1 \
| sed 's/.*value="//;s/"//' || true)
if [[ -z "$CSRF_TOKEN" ]]; then
error "Failed to get CSRF token after login — authentication may have failed"
error "Verify password is correct (AETHER_ADMIN_PASSWORD in env file)"
return 1
fi
success "Authenticated with Aether (session + CSRF)"
detail "CSRF token: ${CSRF_TOKEN:0:20}..."
}
aether_api() {
local method="$1" path="$2"
shift 2
curl -sSk -b "$COOKIE_JAR" \
-H "X-CSRF-Token: ${CSRF_TOKEN}" \
-H "Referer: ${AETHER_URL}/haproxy" \
-X "$method" "${AETHER_URL}${path}" \
"$@"
}
aether_cleanup() {
if [[ -n "${COOKIE_JAR:-}" && -f "${COOKIE_JAR:-}" ]]; then
rm -f "$COOKIE_JAR"
fi
}
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
# Split BACKEND_IPS into array
get_backend_ips() {
echo "$BACKEND_IPS" | tr ',' ' '
}
get_backend_count() {
local ips
ips=$(get_backend_ips)
echo "$ips" | wc -w
}
backend_name() {
local index="$1"
printf "nginx-lb-%02d" "$index"
}
haproxy_container_01() {
echo "ffsdn-haproxy-${CLUSTER_ID}-01"
}
haproxy_container_02() {
echo "ffsdn-haproxy-${CLUSTER_ID}-02"
}
container_exists() {
local name="$1"
incus info "${CLUSTER_REMOTE}:${name}" &>/dev/null 2>&1
}
container_running() {
local name="$1"
local state
state=$(incus info "${CLUSTER_REMOTE}:${name}" 2>/dev/null | grep -i '^status:' | awk '{print $2}')
[[ "$state" == "RUNNING" || "$state" == "Running" ]]
}
# ---------------------------------------------------------------------------
# Doctor — prerequisite checks
# ---------------------------------------------------------------------------
action_doctor() {
step "Checking prerequisites"
local ok=true
# incus CLI
if command -v incus &>/dev/null; then
success "incus CLI: $(incus version 2>/dev/null || echo 'installed')"
else
error "incus CLI not found — install the Incus client"
ok=false
fi
# curl
if command -v curl &>/dev/null; then
success "curl: $(curl --version | head -1 | awk '{print $2}')"
else
error "curl not found"
ok=false
fi
# python3
if command -v python3 &>/dev/null; then
success "python3: $(python3 --version 2>&1 | awk '{print $2}')"
else
error "python3 not found"
ok=false
fi
# Aether password
if load_password 2>/dev/null; then
success "Aether password: configured"
else
error "Aether password: not found (set AETHER_ADMIN_PASSWORD in env file)"
ok=false
fi
# Cluster remote
if incus remote list -f csv 2>/dev/null | grep -q "^${CLUSTER_REMOTE},"; then
success "Incus remote '${CLUSTER_REMOTE}': reachable"
else
warn "Incus remote '${CLUSTER_REMOTE}' not found (needed for deployment)"
fi
# Aether accessibility (login page is at root /)
local aether_code
aether_code=$(curl -sk -o /dev/null -w '%{http_code}' "${AETHER_URL}/" 2>/dev/null || echo "000")
if [[ "$aether_code" == "200" || "$aether_code" == "302" ]]; then
success "Aether: reachable at ${AETHER_URL} (HTTP ${aether_code})"
else
error "Aether: not reachable at ${AETHER_URL} (HTTP ${aether_code})"
ok=false
fi
# Existing HAProxy containers
if container_exists "$(haproxy_container_01)"; then
info "HAProxy infrastructure exists on cluster ${CLUSTER_ID}"
if container_running "$(haproxy_container_01)"; then
success " $(haproxy_container_01): running"
else
warn " $(haproxy_container_01): not running"
fi
if container_running "$(haproxy_container_02)"; then
success " $(haproxy_container_02): running"
else
warn " $(haproxy_container_02): not running"
fi
else
info "No HAProxy infrastructure on cluster ${CLUSTER_ID} (--deploy to create)"
fi
# Existing backends
local idx=1
for ip in $(get_backend_ips); do
local name
name=$(backend_name "$idx")
if container_exists "$name"; then
success " Backend ${name} (${ip}): exists"
else
info " Backend ${name} (${ip}): not deployed"
fi
idx=$((idx + 1))
done
echo
if [[ "$ok" == true ]]; then
success "All prerequisites satisfied"
else
error "Some prerequisites missing — fix the issues above"
return 1
fi
}
# ---------------------------------------------------------------------------
# Deploy — full deployment pipeline
# ---------------------------------------------------------------------------
action_deploy() {
step "Deploying HAProxy load balancing on cluster ${CLUSTER_ID}"
echo
info " Cluster: ${CLUSTER_REMOTE} (ID: ${CLUSTER_ID})"
info " OVN network: ${OVN_NETWORK}"
info " VIP: ${VIP}"
info " HAProxy: ${HAPROXY_01_IP}, ${HAPROXY_02_IP}"
info " Backends: ${BACKEND_IPS}"
info " Service: ${SERVICE_NAME} (${SERVICE_MODE}, ${SERVICE_BALANCE})"
echo
load_password
# Check existing infrastructure
if container_exists "$(haproxy_container_01)"; then
error "HAProxy infrastructure already exists ($(haproxy_container_01))"
error "Use --status to check health, or --cleanup first."
return 1
fi
if [[ "$SKIP_BACKENDS" != true ]]; then
phase_deploy_backends
else
info "Skipping backend deployment (--skip-backends)"
fi
# Authenticate with Aether for remaining phases
if [[ "$DRY_RUN" != true ]]; then
aether_login
trap aether_cleanup EXIT
fi
if [[ "$SKIP_IMAGE" != true ]]; then
phase_build_image
else
info "Skipping image build (--skip-image)"
fi
phase_deploy_infrastructure
phase_create_service
phase_verify
echo
success "HAProxy deployment complete!"
info " VIP: http://${VIP}/"
info " Service: ${SERVICE_NAME}"
info " Backends: $(get_backend_count)"
echo
info "Next steps:"
info " curl http://${VIP}/ # test load balancing"
info " ${SCRIPT_NAME} --status # check health"
}
# ---------------------------------------------------------------------------
# Phase 1: Deploy test backends
# ---------------------------------------------------------------------------
phase_deploy_backends() {
step "Phase 1: Deploy test backends"
local idx=1
for ip in $(get_backend_ips); do
local name
name=$(backend_name "$idx")
if container_exists "$name"; then
if container_running "$name"; then
success "Backend ${name} already running"
idx=$((idx + 1))
continue
else
warn "Backend ${name} exists but not running — starting"
if dry_run_guard "incus start ${CLUSTER_REMOTE}:${name}"; then
incus start "${CLUSTER_REMOTE}:${name}"
fi
idx=$((idx + 1))
continue
fi
fi
if ! dry_run_guard "Launch ${name} on ${OVN_NETWORK} at ${ip}"; then
idx=$((idx + 1))
continue
fi
info "Launching ${name} (${ip})..."
incus launch images:debian/12 "${CLUSTER_REMOTE}:${name}" \
-n "$OVN_NETWORK"
# Wait for container agent
local elapsed=0
while ! incus exec "${CLUSTER_REMOTE}:${name}" -- true &>/dev/null 2>&1; do
sleep 2
elapsed=$((elapsed + 2))
if [[ $elapsed -ge 60 ]]; then
error "Container agent not available after 60s"
return 1
fi
done
# Configure backend
incus exec "${CLUSTER_REMOTE}:${name}" -- bash -c "
# Wait for initial network
sleep 3
# Install nginx + curl (curl needed for verification)
export DEBIAN_FRONTEND=noninteractive
apt-get update -qq
apt-get install -y -qq nginx curl 2>&1 | tail -1
# Create unique index page
cat > /var/www/html/index.html << HTML
<html><body>
<h1>Backend: ${name}</h1>
<p>IP: ${ip}</p>
</body></html>
HTML
systemctl enable nginx
systemctl start nginx
"
# Set static IP
incus exec "${CLUSTER_REMOTE}:${name}" -- bash -c "
cat > /etc/systemd/network/10-static.network << NETCFG
[Match]
Name=eth0
[Network]
Address=${ip}/24
Gateway=10.10.10.1
DNS=10.10.10.1
NETCFG
systemctl restart systemd-networkd
"
success "Backend ${name} deployed at ${ip}"
idx=$((idx + 1))
done
# Verify backends
info "Waiting for backends to stabilize..."
sleep 5
local all_ok=true
idx=1
for ip in $(get_backend_ips); do
local name
name=$(backend_name "$idx")
if container_running "$name"; then
detail "Backend ${name}: running"
else
warn "Backend ${name}: not running"
all_ok=false
fi
idx=$((idx + 1))
done
if [[ "$all_ok" == true ]]; then
success "All $(get_backend_count) backends deployed"
else
warn "Some backends may not be ready"
fi
}
# ---------------------------------------------------------------------------
# Phase 2: Build HAProxy image
# ---------------------------------------------------------------------------
phase_build_image() {
step "Phase 2: Build HAProxy image"
if ! dry_run_guard "Build HAProxy image v${IMAGE_VERSION} on cluster ${CLUSTER_ID}"; then
return 0
fi
# Check if image already exists
local images_response
images_response=$(aether_api GET "/haproxy/images" 2>/dev/null || echo "[]")
local existing_version
existing_version=$(echo "$images_response" \
| python3 -c "
import sys, json
try:
imgs = json.load(sys.stdin)
if isinstance(imgs, list):
for img in imgs:
if img.get('version') == '${IMAGE_VERSION}':
print(img.get('id', ''))
break
except: pass
" 2>/dev/null || true)
if [[ -n "$existing_version" ]]; then
success "Image v${IMAGE_VERSION} already exists (ID: ${existing_version})"
info "Ensuring image is pushed to cluster and set as current..."
aether_api POST "/haproxy/image/push" \
-H "Content-Type: application/json" \
-d "{\"image_id\": ${existing_version}, \"cluster_id\": ${CLUSTER_ID}}" \
2>/dev/null || true
aether_api POST "/haproxy/image/set-current" \
-H "Content-Type: application/json" \
-d "{\"image_id\": ${existing_version}}" \
2>/dev/null || true
success "Image v${IMAGE_VERSION} ready on cluster ${CLUSTER_ID}"
return 0
fi
# Build new image
info "Building HAProxy image v${IMAGE_VERSION} on cluster ${CLUSTER_ID}..."
info "This takes 5-10 minutes..."
local build_response
build_response=$(aether_api POST "/haproxy/image/build" \
-H "Content-Type: application/json" \
-d "{\"cluster_id\": ${CLUSTER_ID}, \"version\": \"${IMAGE_VERSION}\"}" \
2>/dev/null)
detail "Build response: ${build_response}"
# Poll build status
local elapsed=0
local timeout=600
local status=""
while true; do
status=$(aether_api GET "/haproxy/image/build-status" 2>/dev/null \
| python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('status','unknown'))" \
2>/dev/null || echo "unknown")
detail "Build status: ${status} (${elapsed}s)"
if [[ "$status" == "completed" || "$status" == "success" ]]; then
success "Image build completed"
break
elif [[ "$status" == "failed" || "$status" == "error" ]]; then
error "Image build failed"
local build_error
build_error=$(aether_api GET "/haproxy/image/build-status" 2>/dev/null \
| python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('error','unknown'))" \
2>/dev/null || true)
[[ -n "$build_error" ]] && error "Error: ${build_error}"
return 1
fi
sleep 15
elapsed=$((elapsed + 15))
if [[ $elapsed -ge $timeout ]]; then
error "Image build timed out after ${timeout}s"
return 1
fi
done
# Get new image ID
local image_id
image_id=$(aether_api GET "/haproxy/images" 2>/dev/null \
| python3 -c "
import sys, json
try:
imgs = json.load(sys.stdin)
if isinstance(imgs, list):
for img in imgs:
if img.get('version') == '${IMAGE_VERSION}':
print(img['id'])
break
except: pass
" 2>/dev/null || true)
if [[ -z "$image_id" ]]; then
error "Cannot find built image — check Aether UI"
return 1
fi
# Push to cluster
info "Pushing image to cluster ${CLUSTER_ID}..."
aether_api POST "/haproxy/image/push" \
-H "Content-Type: application/json" \
-d "{\"image_id\": ${image_id}, \"cluster_id\": ${CLUSTER_ID}}" \
2>/dev/null || true
# Set as current
info "Setting as current version..."
aether_api POST "/haproxy/image/set-current" \
-H "Content-Type: application/json" \
-d "{\"image_id\": ${image_id}}" \
2>/dev/null || true
success "Image v${IMAGE_VERSION} built and pushed (ID: ${image_id})"
}
# ---------------------------------------------------------------------------
# Phase 3: Deploy HAProxy infrastructure
# ---------------------------------------------------------------------------
phase_deploy_infrastructure() {
step "Phase 3: Deploy HAProxy infrastructure"
if ! dry_run_guard "Deploy infrastructure: VIP=${VIP}, HAProxy=${HAPROXY_01_IP}/${HAPROXY_02_IP}"; then
return 0
fi
# Check existing infrastructure
local infra_response
infra_response=$(aether_api GET "/haproxy/infrastructure/${CLUSTER_ID}" 2>/dev/null || echo "{}")
local infra_exists
infra_exists=$(echo "$infra_response" \
| python3 -c "
import sys, json
try:
d = json.load(sys.stdin)
if d.get('deployed') or d.get('haproxy_01_ip'):
print('yes')
else:
print('no')
except: print('no')
" 2>/dev/null || echo "no")
if [[ "$infra_exists" == "yes" ]]; then
success "Infrastructure already deployed on cluster ${CLUSTER_ID}"
return 0
fi
info "Deploying infrastructure..."
info " OVN network: ${OVN_NETWORK}"
info " VIP: ${VIP}"
info " HAProxy 01: ${HAPROXY_01_IP}"
info " HAProxy 02: ${HAPROXY_02_IP}"
info " CPU: ${HAPROXY_CPU}"
info " Memory: ${HAPROXY_MEMORY}"
local deploy_response
deploy_response=$(aether_api POST "/haproxy/infrastructure/deploy" \
-H "Content-Type: application/json" \
-d "{
\"cluster_id\": ${CLUSTER_ID},
\"ovn_network\": \"${OVN_NETWORK}\",
\"vip\": \"${VIP}\",
\"haproxy_01_ip\": \"${HAPROXY_01_IP}\",
\"haproxy_02_ip\": \"${HAPROXY_02_IP}\",
\"cpu_limit\": \"${HAPROXY_CPU}\",
\"memory_limit\": \"${HAPROXY_MEMORY}\"
}" 2>/dev/null)
detail "Deploy response: ${deploy_response}"
# Verify containers appeared
info "Waiting for HAProxy containers to start..."
local elapsed=0
local timeout=120
while true; do
if container_exists "$(haproxy_container_01)" && container_exists "$(haproxy_container_02)"; then
break
fi
sleep 5
elapsed=$((elapsed + 5))
if [[ $elapsed -ge $timeout ]]; then
error "HAProxy containers not found after ${timeout}s"
error "Check Aether UI for deployment errors"
return 1
fi
detail "Waiting for containers... (${elapsed}s)"
done
# Wait for running state
elapsed=0
while true; do
if container_running "$(haproxy_container_01)" && container_running "$(haproxy_container_02)"; then
break
fi
sleep 5
elapsed=$((elapsed + 5))
if [[ $elapsed -ge $timeout ]]; then
warn "HAProxy containers exist but not all running after ${timeout}s"
break
fi
done
success "Infrastructure deployed"
success " $(haproxy_container_01): $(container_running "$(haproxy_container_01)" && echo "running" || echo "not running")"
success " $(haproxy_container_02): $(container_running "$(haproxy_container_02)" && echo "running" || echo "not running")"
}
# ---------------------------------------------------------------------------
# Phase 4: Create HTTP service
# ---------------------------------------------------------------------------
phase_create_service() {
step "Phase 4: Create HTTP service"
if ! dry_run_guard "Create service: ${SERVICE_NAME} on ${VIP}:${BACKEND_PORT}"; then
return 0
fi
# Build backends JSON
local backends_json="["
local idx=1
local first=true
for ip in $(get_backend_ips); do
local name
name=$(backend_name "$idx")
if [[ "$first" == true ]]; then
first=false
else
backends_json+=","
fi
backends_json+="{\"name\":\"${name}\",\"ip\":\"${ip}\",\"port\":${BACKEND_PORT},\"weight\":100}"
idx=$((idx + 1))
done
backends_json+="]"
detail "Backends JSON: ${backends_json}"
info "Creating service '${SERVICE_NAME}'..."
info " VIP: ${VIP}:${BACKEND_PORT}"
info " Mode: ${SERVICE_MODE}"
info " Balance: ${SERVICE_BALANCE}"
info " Health: enabled"
info " Backends: $(get_backend_count)"
local service_response
service_response=$(aether_api POST "/haproxy/service" \
-H "Content-Type: application/json" \
-d "{
\"cluster_id\": ${CLUSTER_ID},
\"name\": \"${SERVICE_NAME}\",
\"description\": \"HTTP load balancer for nginx test backends\",
\"vip\": \"${VIP}\",
\"listen_port\": ${BACKEND_PORT},
\"mode\": \"${SERVICE_MODE}\",
\"balance\": \"${SERVICE_BALANCE}\",
\"health_check\": true,
\"sticky_sessions\": false,
\"backends\": ${backends_json}
}" 2>/dev/null)
detail "Service response: ${service_response}"
success "Service '${SERVICE_NAME}' created"
}
# ---------------------------------------------------------------------------
# Phase 5: Verify
# ---------------------------------------------------------------------------
phase_verify() {
step "Phase 5: Verify deployment"
info "Waiting for HAProxy to configure backends..."
sleep 10
# Test VIP
info "Testing VIP at http://${VIP}/..."
local response
response=$(curl -sk --connect-timeout 5 --max-time 10 "http://${VIP}/" 2>/dev/null || true)
if [[ -n "$response" ]]; then
local backend_hit
backend_hit=$(echo "$response" | grep -o 'Backend: [^<]*' || echo "unknown")
success "VIP responding: ${backend_hit}"
else
warn "VIP not responding yet — HAProxy may still be configuring"
warn "Try: curl http://${VIP}/"
fi
# Test distribution
if [[ -n "$response" ]]; then
info "Testing traffic distribution (6 requests)..."
local hits=""
for i in $(seq 1 6); do
local hit
hit=$(curl -sk --connect-timeout 5 --max-time 10 "http://${VIP}/" 2>/dev/null \
| grep -o 'Backend: [^<]*' || echo "no response")
detail " Request ${i}: ${hit}"
hits="${hits}${hit}\n"
done
local unique_backends
unique_backends=$(echo -e "$hits" | sort -u | grep -c 'Backend:' || echo 0)
if [[ "$unique_backends" -gt 1 ]]; then
success "Traffic distributed across ${unique_backends} backends"
elif [[ "$unique_backends" -eq 1 ]]; then
info "Traffic going to 1 backend (health checks may not have completed yet)"
else
warn "No successful responses — check HAProxy and backend health"
fi
fi
}
# ---------------------------------------------------------------------------
# Status — health checks
# ---------------------------------------------------------------------------
action_status() {
step "HAProxy Status for cluster ${CLUSTER_ID}"
echo
# HAProxy containers
if container_exists "$(haproxy_container_01)"; then
if container_running "$(haproxy_container_01)"; then
success "$(haproxy_container_01): running"
else
error "$(haproxy_container_01): not running"
fi
else
error "$(haproxy_container_01): not found (no infrastructure deployed)"
return 1
fi
if container_exists "$(haproxy_container_02)"; then
if container_running "$(haproxy_container_02)"; then
success "$(haproxy_container_02): running"
else
warn "$(haproxy_container_02): not running"
fi
else
warn "$(haproxy_container_02): not found"
fi
# Backend containers
echo
info "Backends:"
local idx=1
for ip in $(get_backend_ips); do
local name
name=$(backend_name "$idx")
if container_exists "$name"; then
if container_running "$name"; then
success " ${name} (${ip}): running"
else
warn " ${name} (${ip}): not running"
fi
else
info " ${name} (${ip}): not deployed"
fi
idx=$((idx + 1))
done
# VIP connectivity
echo
info "VIP connectivity:"
local vip_code
vip_code=$(curl -sk -o /dev/null -w '%{http_code}' --connect-timeout 5 \
"http://${VIP}/" 2>/dev/null || echo "000")
if [[ "$vip_code" == "200" ]]; then
success "VIP http://${VIP}/ responding (HTTP ${vip_code})"
else
warn "VIP http://${VIP}/ not responding (HTTP ${vip_code})"
fi
# Aether health stats (requires session auth)
echo
if load_password 2>/dev/null; then
aether_login 2>/dev/null
trap aether_cleanup EXIT
local health_response
health_response=$(aether_api GET \
"/haproxy/service/health-stats?cluster_id=${CLUSTER_ID}&service_name=${SERVICE_NAME}" \
2>/dev/null || echo "{}")
local has_stats
has_stats=$(echo "$health_response" \
| python3 -c "
import sys, json
try:
d = json.load(sys.stdin)
if d and not d.get('error'):
print('yes')
else:
print('no')
except: print('no')
" 2>/dev/null || echo "no")
if [[ "$has_stats" == "yes" ]]; then
info "Service health (from Aether):"
echo "$health_response" | python3 -c "
import sys, json
try:
d = json.load(sys.stdin)
# Print backend stats if available
backends = d.get('backends', d.get('backend_stats', []))
if isinstance(backends, list):
for b in backends:
name = b.get('name', b.get('server', 'unknown'))
status = b.get('status', 'unknown')
icon = 'UP' if status.upper() == 'UP' else 'DOWN'
print(f' {name}: {icon}')
# Print frontend stats if available
frontend = d.get('frontend', d.get('frontend_stats', {}))
if isinstance(frontend, dict) and frontend:
sessions = frontend.get('scur', frontend.get('current_sessions', '?'))
print(f' Frontend sessions: {sessions}')
except Exception as e:
print(f' (could not parse stats: {e})')
" 2>/dev/null || warn " Could not parse health stats"
else
info "No health stats available (service may not exist yet)"
fi
else
info "Skipping Aether health stats (no password configured)"
fi
}
# ---------------------------------------------------------------------------
# Cleanup — remove everything
# ---------------------------------------------------------------------------
action_cleanup() {
step "Cleaning up HAProxy deployment"
echo
local has_infra=false
local has_backends=false
if container_exists "$(haproxy_container_01)"; then
has_infra=true
fi
local idx=1
for ip in $(get_backend_ips); do
if container_exists "$(backend_name "$idx")"; then
has_backends=true
break
fi
idx=$((idx + 1))
done
if [[ "$has_infra" == false && "$has_backends" == false ]]; then
info "Nothing to clean up"
return 0
fi
echo
if [[ "$has_infra" == true ]]; then
warn "This will delete HAProxy infrastructure ($(haproxy_container_01), $(haproxy_container_02))"
warn "All services and VIPs will be removed."
fi
if [[ "$has_backends" == true ]]; then
warn "This will delete test backend containers."
fi
echo
if [[ "$DRY_RUN" == true ]]; then
info "(dry-run) Would delete infrastructure and backends"
return 0
fi
read -r -p "Type 'yes' to confirm: " confirm
if [[ "$confirm" != "yes" ]]; then
info "Aborted"
return 0
fi
# Delete infrastructure via Aether API (removes containers, VIPs, services)
if [[ "$has_infra" == true ]]; then
step "Removing HAProxy infrastructure via Aether"
if load_password 2>/dev/null; then
aether_login
trap aether_cleanup EXIT
local delete_response
delete_response=$(aether_api DELETE "/haproxy/infrastructure/${CLUSTER_ID}" \
-H "Content-Type: application/json" \
2>/dev/null || echo "")
detail "Delete response: ${delete_response}"
# Wait for containers to be removed
local elapsed=0
while container_exists "$(haproxy_container_01)" && [[ $elapsed -lt 60 ]]; do
sleep 5
elapsed=$((elapsed + 5))
detail "Waiting for cleanup... (${elapsed}s)"
done
if ! container_exists "$(haproxy_container_01)"; then
success "HAProxy infrastructure removed"
else
warn "Infrastructure may not be fully removed — check Aether UI"
fi
else
warn "Cannot authenticate with Aether — deleting containers directly"
if container_running "$(haproxy_container_01)"; then
incus stop "${CLUSTER_REMOTE}:$(haproxy_container_01)" --force 2>/dev/null || true
fi
incus delete "${CLUSTER_REMOTE}:$(haproxy_container_01)" --force 2>/dev/null || true
if container_running "$(haproxy_container_02)"; then
incus stop "${CLUSTER_REMOTE}:$(haproxy_container_02)" --force 2>/dev/null || true
fi
incus delete "${CLUSTER_REMOTE}:$(haproxy_container_02)" --force 2>/dev/null || true
success "HAProxy containers deleted directly"
fi
fi
# Delete backends
if [[ "$has_backends" == true ]]; then
step "Removing test backends"
idx=1
for ip in $(get_backend_ips); do
local name
name=$(backend_name "$idx")
if container_exists "$name"; then
info "Deleting ${name}..."
incus delete "${CLUSTER_REMOTE}:${name}" --force 2>/dev/null || true
success "Deleted ${name}"
fi
idx=$((idx + 1))
done
fi
echo
success "Cleanup complete"
}
# ---------------------------------------------------------------------------
# Argument parsing
# ---------------------------------------------------------------------------
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
--deploy) ACTION="deploy";;
--status) ACTION="status";;
--cleanup) ACTION="cleanup";;
--doctor) ACTION="doctor";;
--skip-backends) SKIP_BACKENDS=true;;
--skip-image) SKIP_IMAGE=true;;
-c|--config) CONFIG_FILE="$2"; shift;;
-p|--password) AETHER_PASSWORD="$2"; shift;;
-n|--dry-run) DRY_RUN=true;;
-v|--verbose) VERBOSE=true;;
-q|--quiet) QUIET=true;;
-h|--help) usage; exit 0;;
*)
error "Unknown option: $1"
echo "Run '${SCRIPT_NAME} --help' for usage."
exit 1
;;
esac
shift
done
if [[ -z "$ACTION" ]]; then
error "No action specified"
echo "Run '${SCRIPT_NAME} --help' for usage."
exit 1
fi
}
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
main() {
setup_colors
parse_args "$@"
# Load config file if provided
if [[ -n "$CONFIG_FILE" ]]; then
parse_config "$CONFIG_FILE"
fi
echo
case "$ACTION" in
deploy) action_deploy;;
status) action_status;;
cleanup) action_cleanup;;
doctor) action_doctor;;
esac
}
main "$@"
Reference in New Issue View Git Blame Copy Permalink