1042 lines
34 KiB
Bash
Executable File
1042 lines
34 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# lab-test - Guided lab validation for IncusOS deployments
|
|
#
|
|
# Walks through lab validation step by step: deploy VMs, test single-node
|
|
# workloads, form clusters, test migration and evacuation.
|
|
#
|
|
# Operates on already-deployed VMs (expects incus remotes to exist) or
|
|
# can deploy them via incusos-proxmox.
|
|
#
|
|
# Usage: lab-test [OPTIONS] CONFIG_FILE
|
|
# Run 'lab-test --help' for full usage information.
|
|
|
|
set -euo pipefail
|
|
|
|
readonly VERSION="0.1.0"
|
|
readonly SCRIPT_NAME="lab-test"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Color and formatting
|
|
# ---------------------------------------------------------------------------
|
|
|
|
setup_colors() {
|
|
if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[0;33m'
|
|
BLUE='\033[0;34m'
|
|
CYAN='\033[0;36m'
|
|
BOLD='\033[1m'
|
|
DIM='\033[2m'
|
|
RESET='\033[0m'
|
|
else
|
|
RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' RESET=''
|
|
fi
|
|
}
|
|
|
|
info() { [[ "$QUIET" == true ]] && return; echo -e "${BLUE}[info]${RESET} $*"; }
|
|
success() { [[ "$QUIET" == true ]] && return; echo -e "${GREEN}[ok]${RESET} $*"; }
|
|
warn() { echo -e "${YELLOW}[warn]${RESET} $*" >&2; }
|
|
error() { echo -e "${RED}[error]${RESET} $*" >&2; }
|
|
step() { [[ "$QUIET" == true ]] && return; echo -e "${CYAN}[step]${RESET} ${BOLD}$*${RESET}"; }
|
|
detail() { [[ "$QUIET" == true ]] && return; echo -e "${DIM} $*${RESET}"; }
|
|
|
|
pass() { echo -e " ${GREEN}PASS${RESET} $*"; PASSES=$((PASSES + 1)); }
|
|
fail() { echo -e " ${RED}FAIL${RESET} $*"; FAILURES=$((FAILURES + 1)); }
|
|
skip() { echo -e " ${YELLOW}SKIP${RESET} $*"; SKIPS=$((SKIPS + 1)); }
|
|
|
|
# Test result counters
|
|
PASSES=0
|
|
FAILURES=0
|
|
SKIPS=0
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Help
|
|
# ---------------------------------------------------------------------------
|
|
|
|
usage() {
|
|
cat <<EOF
|
|
${BOLD}${SCRIPT_NAME}${RESET} v${VERSION} - Guided lab validation for IncusOS deployments
|
|
|
|
${BOLD}USAGE${RESET}
|
|
${SCRIPT_NAME} [OPTIONS] CONFIG_FILE
|
|
|
|
${BOLD}DESCRIPTION${RESET}
|
|
Walks through lab validation step by step. Operates on an already-deployed
|
|
set of IncusOS VMs (expects 'incus' remotes to exist). Interactive by
|
|
default -- prints what it will do, confirms, executes, reports.
|
|
|
|
${BOLD}OPTIONS${RESET}
|
|
${BOLD}-p, --phase${RESET} PHASE Run specific phase:
|
|
${CYAN}deploy${RESET} -- deploy VMs and add incus remotes
|
|
${CYAN}single${RESET} -- single-node workload tests
|
|
${CYAN}cluster${RESET} -- form a 3-node cluster
|
|
${CYAN}workload${RESET} -- cluster workload tests
|
|
${CYAN}migrate${RESET} -- migration and evacuation tests
|
|
${CYAN}all${RESET} -- run all phases (default)
|
|
${BOLD}--skip-deploy${RESET} Skip deploy/status check (assume VMs running)
|
|
${BOLD}--cleanup${RESET} Tear down test artifacts without destroying VMs
|
|
${BOLD}-y, --yes${RESET} Skip confirmation prompts
|
|
${BOLD}--dry-run${RESET} Preview actions without executing
|
|
${BOLD}-q, --quiet${RESET} Suppress informational output
|
|
${BOLD}-h, --help${RESET} Show this help message
|
|
${BOLD}-V, --version${RESET} Show version
|
|
|
|
${BOLD}EXAMPLES${RESET}
|
|
# Run all phases
|
|
${SCRIPT_NAME} examples/lab-cluster.yaml
|
|
|
|
# Deploy and verify only
|
|
${SCRIPT_NAME} --phase deploy examples/lab-cluster.yaml
|
|
|
|
# Test single-node workloads (assumes VMs already running)
|
|
${SCRIPT_NAME} --phase single --skip-deploy examples/lab-cluster.yaml
|
|
|
|
# Form cluster
|
|
${SCRIPT_NAME} --phase cluster examples/lab-cluster.yaml
|
|
|
|
# Full run without prompts
|
|
${SCRIPT_NAME} --yes examples/lab-cluster.yaml
|
|
|
|
# Clean up test instances (keeps VMs)
|
|
${SCRIPT_NAME} --cleanup examples/lab-cluster.yaml
|
|
|
|
${BOLD}PREREQUISITES${RESET}
|
|
- incus client (for remote management)
|
|
- incusos-proxmox sibling script (for deploy phase)
|
|
- Already-deployed IncusOS VMs (or use deploy phase)
|
|
|
|
EOF
|
|
exit 0
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Argument parsing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
PHASE="all"
|
|
SKIP_DEPLOY=false
|
|
CLEANUP_MODE=false
|
|
YES=false
|
|
DRY_RUN=false
|
|
QUIET=false
|
|
CONFIG_FILE=""
|
|
|
|
parse_args() {
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
-p|--phase)
|
|
PHASE="${2:?'--phase requires a value'}"
|
|
shift 2
|
|
;;
|
|
--skip-deploy)
|
|
SKIP_DEPLOY=true
|
|
shift
|
|
;;
|
|
--cleanup)
|
|
CLEANUP_MODE=true
|
|
shift
|
|
;;
|
|
-y|--yes)
|
|
YES=true
|
|
shift
|
|
;;
|
|
--dry-run)
|
|
DRY_RUN=true
|
|
shift
|
|
;;
|
|
-q|--quiet)
|
|
QUIET=true
|
|
shift
|
|
;;
|
|
-h|--help)
|
|
usage
|
|
;;
|
|
-V|--version)
|
|
echo "${SCRIPT_NAME} v${VERSION}"
|
|
exit 0
|
|
;;
|
|
-*)
|
|
error "Unknown option: $1"
|
|
echo "Run '${SCRIPT_NAME} --help' for usage information." >&2
|
|
exit 1
|
|
;;
|
|
*)
|
|
if [[ -z "$CONFIG_FILE" ]]; then
|
|
CONFIG_FILE="$1"
|
|
else
|
|
error "Unexpected argument: $1 (config file already set to ${CONFIG_FILE})"
|
|
exit 1
|
|
fi
|
|
shift
|
|
;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
validate_args() {
|
|
if [[ -z "$CONFIG_FILE" ]]; then
|
|
error "No configuration file specified"
|
|
echo "Usage: ${SCRIPT_NAME} [OPTIONS] CONFIG_FILE" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ ! -f "$CONFIG_FILE" ]]; then
|
|
error "Configuration file not found: ${CONFIG_FILE}"
|
|
exit 1
|
|
fi
|
|
|
|
case "$PHASE" in
|
|
deploy|single|cluster|workload|migrate|all) ;;
|
|
*)
|
|
error "Invalid phase: ${PHASE}"
|
|
error "Supported: deploy, single, cluster, workload, migrate, all"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
if ! command -v incus &>/dev/null; then
|
|
error "incus client is required but not found"
|
|
error "Install the Incus client (https://github.com/lxc/incus)"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Config parsing (reuse python3 YAML->JSON approach)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
CONFIG_JSON=""
|
|
PROXMOX_TOOL=""
|
|
WORKDIR=""
|
|
|
|
declare -a VM_NAMES=()
|
|
|
|
parse_config() {
|
|
step "Parsing configuration: ${CONFIG_FILE}"
|
|
|
|
WORKDIR=$(mktemp -d "${TMPDIR:-/tmp}/${SCRIPT_NAME}-XXXXXX")
|
|
trap 'rm -rf "$WORKDIR"' EXIT
|
|
|
|
CONFIG_JSON="${WORKDIR}/config.json"
|
|
|
|
if ! python3 -c "
|
|
import json, sys
|
|
try:
|
|
import yaml
|
|
with open(sys.argv[1]) as f:
|
|
data = yaml.safe_load(f)
|
|
except ImportError:
|
|
import re
|
|
data = {}
|
|
with open(sys.argv[1]) as f:
|
|
lines = f.readlines()
|
|
processed = []
|
|
for raw in lines:
|
|
s = raw.rstrip()
|
|
if not s or s.lstrip().startswith('#'):
|
|
continue
|
|
indent = len(s) - len(s.lstrip())
|
|
content = s.lstrip()
|
|
processed.append((indent, content))
|
|
# Simple parse for top-level sections and vms list
|
|
result = {}
|
|
section = None
|
|
current_list = None
|
|
current_item = None
|
|
for indent, content in processed:
|
|
m = re.match(r'^-\s+(.+?):\s*(.*)$', content)
|
|
if m and current_list is not None:
|
|
if current_item is not None:
|
|
current_list.append(current_item)
|
|
current_item = {}
|
|
key = m.group(1).strip()
|
|
val = m.group(2).strip()
|
|
if val:
|
|
if val in ('true', 'false'):
|
|
val = val == 'true'
|
|
elif val.isdigit():
|
|
val = int(val)
|
|
elif val.startswith(('\"', \"'\")):
|
|
val = val[1:-1]
|
|
current_item[key] = val
|
|
continue
|
|
if indent > 0 and current_item is not None and not content.startswith('-'):
|
|
m2 = re.match(r'^([^:]+?):\s*(.*)$', content)
|
|
if m2:
|
|
key = m2.group(1).strip()
|
|
val = m2.group(2).strip()
|
|
if val in ('true', 'false'):
|
|
val = val == 'true'
|
|
elif val.isdigit():
|
|
val = int(val)
|
|
elif val.startswith(('\"', \"'\")):
|
|
val = val[1:-1]
|
|
elif '#' in val:
|
|
val = re.sub(r'\s+#.*$', '', val)
|
|
current_item[key] = val
|
|
continue
|
|
m3 = re.match(r'^([^:]+?):\s*(.*)$', content)
|
|
if m3:
|
|
key = m3.group(1).strip()
|
|
val = m3.group(2).strip()
|
|
if key == 'vms':
|
|
current_list = []
|
|
current_item = None
|
|
result['vms'] = current_list
|
|
section = 'vms'
|
|
elif not val:
|
|
section = key
|
|
result[section] = {}
|
|
current_list = None
|
|
if current_item is not None:
|
|
pass
|
|
current_item = None
|
|
elif section and section != 'vms':
|
|
if val in ('true', 'false'):
|
|
val = val == 'true'
|
|
elif val.isdigit():
|
|
val = int(val)
|
|
elif val.startswith(('\"', \"'\")):
|
|
val = val[1:-1]
|
|
elif '#' in val:
|
|
val = re.sub(r'\s+#.*$', '', val)
|
|
result[section][key] = val
|
|
if current_item is not None and current_list is not None:
|
|
current_list.append(current_item)
|
|
data = result
|
|
json.dump(data, sys.stdout, indent=2)
|
|
" "$CONFIG_FILE" > "$CONFIG_JSON" 2>/dev/null; then
|
|
error "Failed to parse configuration: ${CONFIG_FILE}"
|
|
exit 1
|
|
fi
|
|
|
|
# Load VM names
|
|
local count
|
|
if command -v jq &>/dev/null; then
|
|
count=$(jq -r '.vms | length' "$CONFIG_JSON" 2>/dev/null || echo "0")
|
|
else
|
|
count=$(python3 -c "
|
|
import json
|
|
data = json.load(open('$CONFIG_JSON'))
|
|
print(len(data.get('vms', [])))
|
|
" 2>/dev/null || echo "0")
|
|
fi
|
|
|
|
local i=0
|
|
while [[ $i -lt $count ]]; do
|
|
local name
|
|
if command -v jq &>/dev/null; then
|
|
name=$(jq -r ".vms[${i}].name // empty" "$CONFIG_JSON" 2>/dev/null)
|
|
else
|
|
name=$(python3 -c "
|
|
import json
|
|
data = json.load(open('$CONFIG_JSON'))
|
|
print(data.get('vms', [])[${i}].get('name', ''))
|
|
" 2>/dev/null)
|
|
fi
|
|
VM_NAMES+=("$name")
|
|
i=$((i + 1))
|
|
done
|
|
|
|
# Find sibling incusos-proxmox
|
|
local script_dir
|
|
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROXMOX_TOOL="${script_dir}/incusos-proxmox"
|
|
if [[ ! -x "$PROXMOX_TOOL" ]]; then
|
|
warn "incusos-proxmox not found at ${PROXMOX_TOOL}"
|
|
warn "Deploy phase will be unavailable"
|
|
PROXMOX_TOOL=""
|
|
fi
|
|
|
|
success "Loaded ${#VM_NAMES[@]} VM(s): ${VM_NAMES[*]}"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Confirmation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
confirm() {
|
|
local message="$1"
|
|
if [[ "$YES" == true ]] || [[ "$DRY_RUN" == true ]]; then
|
|
return 0
|
|
fi
|
|
echo -n -e "${BOLD}${message} [y/N] ${RESET}"
|
|
local reply
|
|
read -r reply
|
|
if [[ ! "$reply" =~ ^[Yy]$ ]]; then
|
|
info "Skipped"
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Utility: run or preview
|
|
# ---------------------------------------------------------------------------
|
|
|
|
run_cmd() {
|
|
local desc="$1"
|
|
shift
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] $*"
|
|
return 0
|
|
fi
|
|
|
|
detail "Running: $*"
|
|
if "$@"; then
|
|
return 0
|
|
else
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Wait for incus remote to be reachable (port 8443)
|
|
wait_for_remote() {
|
|
local remote="$1"
|
|
local timeout="${2:-120}"
|
|
local elapsed=0
|
|
|
|
while [[ $elapsed -lt $timeout ]]; do
|
|
if incus info "${remote}:" &>/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
sleep 5
|
|
elapsed=$((elapsed + 5))
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 1: Deploy & Verify
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_deploy() {
|
|
step "Phase 1: Deploy & Verify"
|
|
echo ""
|
|
|
|
if [[ "$SKIP_DEPLOY" == true ]]; then
|
|
info "Skipping deploy (--skip-deploy)"
|
|
elif [[ -n "$PROXMOX_TOOL" ]]; then
|
|
step "Deploying VMs via incusos-proxmox"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] ${PROXMOX_TOOL} --yes ${CONFIG_FILE}"
|
|
else
|
|
"$PROXMOX_TOOL" --yes "$CONFIG_FILE" || true
|
|
fi
|
|
echo ""
|
|
else
|
|
warn "incusos-proxmox not found, skipping deployment"
|
|
fi
|
|
|
|
# Check status
|
|
if [[ -n "$PROXMOX_TOOL" ]] && [[ "$DRY_RUN" != true ]]; then
|
|
step "Checking deployment status"
|
|
"$PROXMOX_TOOL" --status "$CONFIG_FILE" || true
|
|
echo ""
|
|
fi
|
|
|
|
# Verify incus remotes exist for each VM
|
|
step "Verifying incus remotes"
|
|
local all_remotes_ok=true
|
|
|
|
for name in "${VM_NAMES[@]}"; do
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] Would check incus remote '${name}'"
|
|
continue
|
|
fi
|
|
|
|
if incus remote list --format csv 2>/dev/null | grep -q "^${name},"; then
|
|
if incus info "${name}:" &>/dev/null 2>&1; then
|
|
pass "Remote '${name}' connected"
|
|
else
|
|
fail "Remote '${name}' exists but connection failed"
|
|
all_remotes_ok=false
|
|
fi
|
|
else
|
|
warn "Remote '${name}' not configured"
|
|
detail " Try: incus remote add ${name} <IP> --accept-certificate"
|
|
all_remotes_ok=false
|
|
fi
|
|
done
|
|
|
|
if [[ "$all_remotes_ok" != true ]] && [[ "$DRY_RUN" != true ]]; then
|
|
warn "Not all remotes are configured. Some phases may fail."
|
|
if ! confirm "Continue anyway?"; then
|
|
return 1
|
|
fi
|
|
fi
|
|
|
|
echo ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 2: Single-Node Workloads
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_single() {
|
|
step "Phase 2: Single-Node Workloads"
|
|
echo ""
|
|
|
|
# Pick first node
|
|
local target="${VM_NAMES[0]}"
|
|
info "Using node: ${target}"
|
|
echo ""
|
|
|
|
if [[ "$DRY_RUN" != true ]] && ! incus info "${target}:" &>/dev/null 2>&1; then
|
|
fail "Cannot connect to remote '${target}'"
|
|
return 1
|
|
fi
|
|
|
|
# Test 1: Launch container
|
|
step "Test: Launch container"
|
|
local ct_name="test-ct-$$"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus launch images:debian/12 ${target}:${ct_name}"
|
|
detail "[dry-run] incus exec ${target}:${ct_name} -- hostname"
|
|
detail "[dry-run] incus delete ${target}:${ct_name} --force"
|
|
pass "Container lifecycle (dry-run)"
|
|
else
|
|
if incus launch images:debian/12 "${target}:${ct_name}" 2>/dev/null; then
|
|
# Wait for container to be running
|
|
sleep 5
|
|
local ct_hostname
|
|
ct_hostname=$(incus exec "${target}:${ct_name}" -- hostname 2>/dev/null) || ct_hostname=""
|
|
if [[ -n "$ct_hostname" ]]; then
|
|
pass "Container launch + exec (hostname: ${ct_hostname})"
|
|
else
|
|
fail "Container launched but exec failed"
|
|
fi
|
|
# Cleanup
|
|
incus delete "${target}:${ct_name}" --force 2>/dev/null || true
|
|
detail "Container cleaned up"
|
|
else
|
|
fail "Container launch failed"
|
|
detail " This might mean the image server is unreachable from the node"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Test 2: Launch VM
|
|
step "Test: Launch VM"
|
|
local vm_name="test-vm-$$"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus launch images:debian/12 ${target}:${vm_name} --vm"
|
|
detail "[dry-run] (wait for agent)"
|
|
detail "[dry-run] incus exec ${target}:${vm_name} -- hostname"
|
|
detail "[dry-run] incus delete ${target}:${vm_name} --force"
|
|
pass "VM lifecycle (dry-run)"
|
|
else
|
|
if incus launch images:debian/12 "${target}:${vm_name}" --vm 2>/dev/null; then
|
|
# VMs take longer to boot -- wait for agent
|
|
detail "Waiting for VM agent (up to 120s)..."
|
|
local vm_ready=false
|
|
local elapsed=0
|
|
while [[ $elapsed -lt 120 ]]; do
|
|
if incus exec "${target}:${vm_name}" -- true &>/dev/null 2>&1; then
|
|
vm_ready=true
|
|
break
|
|
fi
|
|
sleep 10
|
|
elapsed=$((elapsed + 10))
|
|
detail " waiting... (${elapsed}s)"
|
|
done
|
|
|
|
if [[ "$vm_ready" == true ]]; then
|
|
local vm_hostname
|
|
vm_hostname=$(incus exec "${target}:${vm_name}" -- hostname 2>/dev/null) || vm_hostname=""
|
|
if [[ -n "$vm_hostname" ]]; then
|
|
pass "VM launch + exec (hostname: ${vm_hostname})"
|
|
else
|
|
fail "VM agent ready but exec failed"
|
|
fi
|
|
else
|
|
fail "VM launched but agent did not become ready within 120s"
|
|
fi
|
|
# Cleanup
|
|
incus delete "${target}:${vm_name}" --force 2>/dev/null || true
|
|
detail "VM cleaned up"
|
|
else
|
|
fail "VM launch failed"
|
|
detail " Nested virtualization may not be available"
|
|
fi
|
|
fi
|
|
echo ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 3: Cluster Formation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_cluster() {
|
|
step "Phase 3: Cluster Formation"
|
|
echo ""
|
|
|
|
if [[ ${#VM_NAMES[@]} -lt 2 ]]; then
|
|
skip "Clustering requires at least 2 nodes (have ${#VM_NAMES[@]})"
|
|
echo ""
|
|
return 0
|
|
fi
|
|
|
|
local init_node="${VM_NAMES[0]}"
|
|
info "Init node: ${init_node}"
|
|
echo ""
|
|
|
|
if [[ "$DRY_RUN" != true ]] && ! incus info "${init_node}:" &>/dev/null 2>&1; then
|
|
fail "Cannot connect to init node '${init_node}'"
|
|
return 1
|
|
fi
|
|
|
|
# Check if already clustered
|
|
if [[ "$DRY_RUN" != true ]]; then
|
|
local cluster_list
|
|
cluster_list=$(incus cluster list "${init_node}:" --format csv 2>/dev/null) || cluster_list=""
|
|
if [[ -n "$cluster_list" ]]; then
|
|
local member_count
|
|
member_count=$(echo "$cluster_list" | wc -l)
|
|
info "Cluster already exists with ${member_count} member(s)"
|
|
incus cluster list "${init_node}:" 2>/dev/null || true
|
|
pass "Cluster already formed"
|
|
echo ""
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
# Step 1: Enable clustering on init node
|
|
step "Enabling clustering on ${init_node}"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus cluster enable ${init_node}: ${init_node}"
|
|
else
|
|
if incus cluster enable "${init_node}:" "${init_node}" 2>/dev/null; then
|
|
pass "Clustering enabled on ${init_node}"
|
|
else
|
|
fail "Failed to enable clustering on ${init_node}"
|
|
return 1
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Step 2-N: Join remaining nodes
|
|
local i=1
|
|
while [[ $i -lt ${#VM_NAMES[@]} ]]; do
|
|
local join_node="${VM_NAMES[$i]}"
|
|
|
|
step "Joining ${join_node} to cluster"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus cluster add ${init_node}: ${join_node}"
|
|
detail "[dry-run] incus cluster join ${init_node}: ${join_node}:"
|
|
pass "Node ${join_node} joined (dry-run)"
|
|
else
|
|
if ! incus info "${join_node}:" &>/dev/null 2>&1; then
|
|
fail "Cannot connect to '${join_node}'"
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
# Generate join token
|
|
local token
|
|
token=$(incus cluster add "${init_node}:" "${join_node}" 2>/dev/null) || token=""
|
|
if [[ -z "$token" ]]; then
|
|
fail "Failed to generate join token for ${join_node}"
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
detail "Join token generated"
|
|
|
|
# Join the cluster
|
|
if incus cluster join "${init_node}:" "${join_node}:" 2>/dev/null; then
|
|
pass "Node ${join_node} joined the cluster"
|
|
else
|
|
fail "Failed to join ${join_node} to cluster"
|
|
fi
|
|
fi
|
|
|
|
echo ""
|
|
i=$((i + 1))
|
|
done
|
|
|
|
# Verify cluster
|
|
step "Verifying cluster membership"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus cluster list ${init_node}:"
|
|
pass "Cluster verification (dry-run)"
|
|
else
|
|
local cluster_output
|
|
cluster_output=$(incus cluster list "${init_node}:" 2>/dev/null) || cluster_output=""
|
|
if [[ -n "$cluster_output" ]]; then
|
|
echo "$cluster_output"
|
|
local online_count
|
|
online_count=$(incus cluster list "${init_node}:" --format csv 2>/dev/null | grep -c "ONLINE" || echo "0")
|
|
if [[ $online_count -ge ${#VM_NAMES[@]} ]]; then
|
|
pass "All ${online_count} cluster members online"
|
|
else
|
|
warn "Only ${online_count}/${#VM_NAMES[@]} members online"
|
|
fi
|
|
else
|
|
fail "Could not list cluster members"
|
|
fi
|
|
fi
|
|
echo ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 4: Cluster Workloads
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_workload() {
|
|
step "Phase 4: Cluster Workloads"
|
|
echo ""
|
|
|
|
local cluster_remote="${VM_NAMES[0]}"
|
|
|
|
if [[ "$DRY_RUN" != true ]] && ! incus info "${cluster_remote}:" &>/dev/null 2>&1; then
|
|
fail "Cannot connect to cluster remote '${cluster_remote}'"
|
|
return 1
|
|
fi
|
|
|
|
# Check if this node is clustered
|
|
if [[ "$DRY_RUN" != true ]]; then
|
|
local is_clustered=false
|
|
incus cluster list "${cluster_remote}:" &>/dev/null 2>&1 && is_clustered=true
|
|
if [[ "$is_clustered" != true ]]; then
|
|
warn "Node '${cluster_remote}' is not clustered"
|
|
warn "Running workloads as single-node tests"
|
|
fi
|
|
fi
|
|
|
|
# Test 1: Container on cluster
|
|
step "Test: Launch container on cluster"
|
|
local ct_name="test-cluster-ct-$$"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus launch images:debian/12 ${cluster_remote}:${ct_name}"
|
|
pass "Cluster container (dry-run)"
|
|
else
|
|
if incus launch images:debian/12 "${cluster_remote}:${ct_name}" 2>/dev/null; then
|
|
sleep 5
|
|
local location
|
|
location=$(incus list "${cluster_remote}:" --format csv -c nL 2>/dev/null | grep "^${ct_name}," | cut -d',' -f2) || location="unknown"
|
|
pass "Cluster container launched (location: ${location})"
|
|
else
|
|
fail "Cluster container launch failed"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Test 2: VM on specific node (if we have a second node)
|
|
if [[ ${#VM_NAMES[@]} -ge 2 ]]; then
|
|
local target_node="${VM_NAMES[1]}"
|
|
step "Test: Launch VM on specific node (${target_node})"
|
|
local vm_name="test-cluster-vm-$$"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus launch images:debian/12 ${cluster_remote}:${vm_name} --vm --target ${target_node}"
|
|
pass "Targeted VM (dry-run)"
|
|
else
|
|
if incus launch images:debian/12 "${cluster_remote}:${vm_name}" --vm --target "${target_node}" 2>/dev/null; then
|
|
detail "Waiting for VM agent..."
|
|
local elapsed=0
|
|
local vm_ready=false
|
|
while [[ $elapsed -lt 120 ]]; do
|
|
if incus exec "${cluster_remote}:${vm_name}" -- true &>/dev/null 2>&1; then
|
|
vm_ready=true
|
|
break
|
|
fi
|
|
sleep 10
|
|
elapsed=$((elapsed + 10))
|
|
done
|
|
local location
|
|
location=$(incus list "${cluster_remote}:" --format csv -c nL 2>/dev/null | grep "^${vm_name}," | cut -d',' -f2) || location="unknown"
|
|
if [[ "$vm_ready" == true ]]; then
|
|
pass "Targeted VM running on ${location}"
|
|
else
|
|
warn "VM launched on ${location} but agent not ready"
|
|
pass "Targeted VM launched (agent timeout)"
|
|
fi
|
|
else
|
|
fail "Targeted VM launch failed"
|
|
fi
|
|
fi
|
|
echo ""
|
|
fi
|
|
|
|
# Show cluster instances
|
|
if [[ "$DRY_RUN" != true ]]; then
|
|
step "Current instances on cluster"
|
|
incus list "${cluster_remote}:" 2>/dev/null || true
|
|
echo ""
|
|
fi
|
|
|
|
# Cleanup test instances
|
|
step "Cleaning up test instances"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] Would delete test instances"
|
|
else
|
|
incus delete "${cluster_remote}:${ct_name}" --force 2>/dev/null || true
|
|
local vm_name="test-cluster-vm-$$"
|
|
incus delete "${cluster_remote}:${vm_name}" --force 2>/dev/null || true
|
|
detail "Test instances cleaned up"
|
|
fi
|
|
echo ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 5: Migration & Evacuation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_migrate() {
|
|
step "Phase 5: Migration & Evacuation"
|
|
echo ""
|
|
|
|
local cluster_remote="${VM_NAMES[0]}"
|
|
|
|
if [[ ${#VM_NAMES[@]} -lt 2 ]]; then
|
|
skip "Migration tests require at least 2 nodes"
|
|
echo ""
|
|
return 0
|
|
fi
|
|
|
|
if [[ "$DRY_RUN" != true ]] && ! incus info "${cluster_remote}:" &>/dev/null 2>&1; then
|
|
fail "Cannot connect to cluster remote '${cluster_remote}'"
|
|
return 1
|
|
fi
|
|
|
|
# Check if clustered
|
|
if [[ "$DRY_RUN" != true ]]; then
|
|
if ! incus cluster list "${cluster_remote}:" &>/dev/null 2>&1; then
|
|
skip "Node is not clustered, skipping migration tests"
|
|
echo ""
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
local target_node="${VM_NAMES[1]}"
|
|
local ct_name="test-migrate-ct-$$"
|
|
|
|
# Test 1: Container migration (stop/move)
|
|
step "Test: Container migration"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus launch images:debian/12 ${cluster_remote}:${ct_name}"
|
|
detail "[dry-run] incus stop ${cluster_remote}:${ct_name}"
|
|
detail "[dry-run] incus move ${cluster_remote}:${ct_name} --target ${target_node}"
|
|
detail "[dry-run] incus start ${cluster_remote}:${ct_name}"
|
|
pass "Container migration (dry-run)"
|
|
else
|
|
if incus launch images:debian/12 "${cluster_remote}:${ct_name}" 2>/dev/null; then
|
|
sleep 5
|
|
local orig_location
|
|
orig_location=$(incus list "${cluster_remote}:" --format csv -c nL 2>/dev/null | grep "^${ct_name}," | cut -d',' -f2) || orig_location="unknown"
|
|
detail "Container on: ${orig_location}"
|
|
|
|
# Stop and move
|
|
incus stop "${cluster_remote}:${ct_name}" 2>/dev/null || true
|
|
sleep 2
|
|
|
|
if incus move "${cluster_remote}:${ct_name}" --target "${target_node}" 2>/dev/null; then
|
|
incus start "${cluster_remote}:${ct_name}" 2>/dev/null || true
|
|
sleep 3
|
|
|
|
local new_location
|
|
new_location=$(incus list "${cluster_remote}:" --format csv -c nL 2>/dev/null | grep "^${ct_name}," | cut -d',' -f2) || new_location="unknown"
|
|
|
|
if [[ "$new_location" == "$target_node" ]]; then
|
|
pass "Container migrated: ${orig_location} -> ${new_location}"
|
|
else
|
|
warn "Container location: ${new_location} (expected: ${target_node})"
|
|
pass "Container move completed (location may differ due to naming)"
|
|
fi
|
|
else
|
|
fail "Container move failed"
|
|
fi
|
|
|
|
# Cleanup
|
|
incus delete "${cluster_remote}:${ct_name}" --force 2>/dev/null || true
|
|
else
|
|
fail "Container launch failed for migration test"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Test 2: Node evacuation (requires 3+ nodes)
|
|
if [[ ${#VM_NAMES[@]} -ge 3 ]]; then
|
|
local evacuate_node="${VM_NAMES[2]}"
|
|
step "Test: Node evacuation (${evacuate_node})"
|
|
|
|
local evac_ct="test-evac-ct-$$"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus launch images:debian/12 ${cluster_remote}:${evac_ct} --target ${evacuate_node}"
|
|
detail "[dry-run] incus cluster evacuate ${evacuate_node} ${cluster_remote}:"
|
|
detail "[dry-run] incus cluster restore ${evacuate_node} ${cluster_remote}:"
|
|
pass "Node evacuation (dry-run)"
|
|
else
|
|
# Place a container on the node we'll evacuate
|
|
if incus launch images:debian/12 "${cluster_remote}:${evac_ct}" --target "${evacuate_node}" 2>/dev/null; then
|
|
sleep 5
|
|
detail "Container placed on ${evacuate_node}"
|
|
|
|
# Evacuate
|
|
if incus cluster evacuate "${evacuate_node}" "${cluster_remote}:" --force 2>/dev/null; then
|
|
sleep 5
|
|
# Check that nothing is on the evacuated node
|
|
local remaining
|
|
remaining=$(incus list "${cluster_remote}:" --format csv -c nL 2>/dev/null | grep ",${evacuate_node}$" | wc -l) || remaining="0"
|
|
if [[ "$remaining" -eq 0 ]]; then
|
|
pass "Node ${evacuate_node} evacuated (0 instances remaining)"
|
|
else
|
|
warn "${remaining} instance(s) still on ${evacuate_node} after evacuation"
|
|
fi
|
|
|
|
# Restore
|
|
if incus cluster restore "${evacuate_node}" "${cluster_remote}:" 2>/dev/null; then
|
|
pass "Node ${evacuate_node} restored"
|
|
else
|
|
warn "Node restore failed (may need manual intervention)"
|
|
fi
|
|
else
|
|
fail "Evacuation of ${evacuate_node} failed"
|
|
fi
|
|
|
|
# Cleanup
|
|
incus delete "${cluster_remote}:${evac_ct}" --force 2>/dev/null || true
|
|
else
|
|
fail "Could not place container on ${evacuate_node} for evacuation test"
|
|
fi
|
|
fi
|
|
echo ""
|
|
else
|
|
skip "Evacuation test requires 3+ nodes (have ${#VM_NAMES[@]})"
|
|
echo ""
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cleanup mode: remove test artifacts
|
|
# ---------------------------------------------------------------------------
|
|
|
|
do_cleanup() {
|
|
step "Cleaning up test artifacts"
|
|
echo ""
|
|
|
|
local cluster_remote="${VM_NAMES[0]}"
|
|
|
|
if [[ "$DRY_RUN" != true ]] && ! incus info "${cluster_remote}:" &>/dev/null 2>&1; then
|
|
warn "Cannot connect to '${cluster_remote}', skipping instance cleanup"
|
|
return
|
|
fi
|
|
|
|
# Delete any test instances
|
|
if [[ "$DRY_RUN" != true ]]; then
|
|
local instances
|
|
instances=$(incus list "${cluster_remote}:" --format csv -c n 2>/dev/null | grep "^test-" || true)
|
|
if [[ -n "$instances" ]]; then
|
|
while IFS= read -r inst; do
|
|
[[ -z "$inst" ]] && continue
|
|
detail "Deleting: ${inst}"
|
|
incus delete "${cluster_remote}:${inst}" --force 2>/dev/null || true
|
|
done <<< "$instances"
|
|
success "Test instances cleaned up"
|
|
else
|
|
info "No test instances found"
|
|
fi
|
|
else
|
|
detail "[dry-run] Would delete instances matching 'test-*'"
|
|
fi
|
|
|
|
echo ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Summary
|
|
# ---------------------------------------------------------------------------
|
|
|
|
print_summary() {
|
|
echo ""
|
|
echo -e "${BOLD}Test Summary${RESET}"
|
|
echo -e " ${GREEN}Passed: ${PASSES}${RESET}"
|
|
if [[ $FAILURES -gt 0 ]]; then
|
|
echo -e " ${RED}Failed: ${FAILURES}${RESET}"
|
|
else
|
|
echo -e " Failed: 0"
|
|
fi
|
|
if [[ $SKIPS -gt 0 ]]; then
|
|
echo -e " ${YELLOW}Skipped: ${SKIPS}${RESET}"
|
|
fi
|
|
echo ""
|
|
|
|
if [[ $FAILURES -gt 0 ]]; then
|
|
echo -e " ${RED}${BOLD}Some tests failed${RESET}"
|
|
else
|
|
echo -e " ${GREEN}${BOLD}All tests passed${RESET}"
|
|
fi
|
|
echo ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main
|
|
# ---------------------------------------------------------------------------
|
|
|
|
main() {
|
|
setup_colors
|
|
parse_args "$@"
|
|
validate_args
|
|
parse_config
|
|
|
|
if [[ "$QUIET" != true ]]; then
|
|
echo -e "${BOLD}${SCRIPT_NAME}${RESET} v${VERSION}"
|
|
echo ""
|
|
fi
|
|
|
|
# Handle cleanup mode
|
|
if [[ "$CLEANUP_MODE" == true ]]; then
|
|
do_cleanup
|
|
return
|
|
fi
|
|
|
|
# Run phases
|
|
case "$PHASE" in
|
|
deploy)
|
|
phase_deploy
|
|
;;
|
|
single)
|
|
if [[ "$SKIP_DEPLOY" != true ]]; then
|
|
phase_deploy
|
|
fi
|
|
phase_single
|
|
;;
|
|
cluster)
|
|
if [[ "$SKIP_DEPLOY" != true ]]; then
|
|
phase_deploy
|
|
fi
|
|
phase_cluster
|
|
;;
|
|
workload)
|
|
if [[ "$SKIP_DEPLOY" != true ]]; then
|
|
phase_deploy
|
|
fi
|
|
phase_workload
|
|
;;
|
|
migrate)
|
|
if [[ "$SKIP_DEPLOY" != true ]]; then
|
|
phase_deploy
|
|
fi
|
|
phase_migrate
|
|
;;
|
|
all)
|
|
phase_deploy
|
|
phase_single
|
|
phase_cluster
|
|
phase_workload
|
|
phase_migrate
|
|
;;
|
|
esac
|
|
|
|
print_summary
|
|
}
|
|
|
|
main "$@"
|