1878 lines
67 KiB
Bash
Executable File
1878 lines
67 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# lab-test - Guided lab validation for IncusOS deployments
|
|
#
|
|
# Walks through lab validation step by step: deploy VMs, test single-node
|
|
# workloads, form clusters, test migration and evacuation.
|
|
#
|
|
# Operates on already-deployed VMs (expects incus remotes to exist) or
|
|
# can deploy them via incusos-proxmox.
|
|
#
|
|
# Usage: lab-test [OPTIONS] CONFIG_FILE
|
|
# Run 'lab-test --help' for full usage information.
|
|
|
|
set -euo pipefail
|
|
|
|
readonly VERSION="0.1.0"
|
|
readonly SCRIPT_NAME="lab-test"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Color and formatting
|
|
# ---------------------------------------------------------------------------
|
|
|
|
setup_colors() {
|
|
if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[0;33m'
|
|
BLUE='\033[0;34m'
|
|
CYAN='\033[0;36m'
|
|
BOLD='\033[1m'
|
|
DIM='\033[2m'
|
|
RESET='\033[0m'
|
|
else
|
|
RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' RESET=''
|
|
fi
|
|
}
|
|
|
|
info() { [[ "$QUIET" == true ]] && return; echo -e "${BLUE}[info]${RESET} $*"; }
|
|
success() { [[ "$QUIET" == true ]] && return; echo -e "${GREEN}[ok]${RESET} $*"; }
|
|
warn() { echo -e "${YELLOW}[warn]${RESET} $*" >&2; }
|
|
error() { echo -e "${RED}[error]${RESET} $*" >&2; }
|
|
step() { [[ "$QUIET" == true ]] && return; echo -e "${CYAN}[step]${RESET} ${BOLD}$*${RESET}"; }
|
|
detail() { [[ "$QUIET" == true ]] && return; echo -e "${DIM} $*${RESET}"; }
|
|
|
|
pass() { echo -e " ${GREEN}PASS${RESET} $*"; PASSES=$((PASSES + 1)); }
|
|
fail() { echo -e " ${RED}FAIL${RESET} $*"; FAILURES=$((FAILURES + 1)); }
|
|
skip() { echo -e " ${YELLOW}SKIP${RESET} $*"; SKIPS=$((SKIPS + 1)); }
|
|
|
|
# Test result counters
|
|
PASSES=0
|
|
FAILURES=0
|
|
SKIPS=0
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Help
|
|
# ---------------------------------------------------------------------------
|
|
|
|
usage() {
|
|
cat <<EOF
|
|
${BOLD}${SCRIPT_NAME}${RESET} v${VERSION} - Guided lab validation for IncusOS deployments
|
|
|
|
${BOLD}USAGE${RESET}
|
|
${SCRIPT_NAME} [OPTIONS] CONFIG_FILE
|
|
|
|
${BOLD}DESCRIPTION${RESET}
|
|
Walks through lab validation step by step. Operates on an already-deployed
|
|
set of IncusOS VMs (expects 'incus' remotes to exist). Interactive by
|
|
default -- prints what it will do, confirms, executes, reports.
|
|
|
|
${BOLD}OPTIONS${RESET}
|
|
${BOLD}-p, --phase${RESET} PHASE Run specific phase:
|
|
${CYAN}deploy${RESET} -- deploy VMs and add incus remotes
|
|
${CYAN}single${RESET} -- single-node workload tests
|
|
${CYAN}cluster${RESET} -- form a 3-node cluster
|
|
${CYAN}workload${RESET} -- cluster workload tests
|
|
${CYAN}migrate${RESET} -- migration and evacuation tests
|
|
${CYAN}storage${RESET} -- storage pool and volume operations
|
|
${CYAN}network${RESET} -- network features and ACLs
|
|
${CYAN}projects${RESET} -- project isolation and profiles
|
|
${CYAN}backup${RESET} -- backup, snapshot, and restore
|
|
${CYAN}limits${RESET} -- resource limits and QoS enforcement
|
|
${CYAN}security${RESET} -- container security and namespaces
|
|
${CYAN}resilience${RESET} -- HA, failover, and cluster resilience
|
|
${CYAN}all${RESET} -- run all phases (default)
|
|
${BOLD}--skip-deploy${RESET} Skip deploy/status check (assume VMs running)
|
|
${BOLD}--cleanup${RESET} Tear down test artifacts without destroying VMs
|
|
${BOLD}-y, --yes${RESET} Skip confirmation prompts
|
|
${BOLD}--dry-run${RESET} Preview actions without executing
|
|
${BOLD}-q, --quiet${RESET} Suppress informational output
|
|
${BOLD}-h, --help${RESET} Show this help message
|
|
${BOLD}-V, --version${RESET} Show version
|
|
|
|
${BOLD}EXAMPLES${RESET}
|
|
# Run all phases
|
|
${SCRIPT_NAME} examples/lab-cluster.yaml
|
|
|
|
# Deploy and verify only
|
|
${SCRIPT_NAME} --phase deploy examples/lab-cluster.yaml
|
|
|
|
# Test single-node workloads (assumes VMs already running)
|
|
${SCRIPT_NAME} --phase single --skip-deploy examples/lab-cluster.yaml
|
|
|
|
# Form cluster
|
|
${SCRIPT_NAME} --phase cluster examples/lab-cluster.yaml
|
|
|
|
# Full run without prompts
|
|
${SCRIPT_NAME} --yes examples/lab-cluster.yaml
|
|
|
|
# Clean up test instances (keeps VMs)
|
|
${SCRIPT_NAME} --cleanup examples/lab-cluster.yaml
|
|
|
|
${BOLD}PREREQUISITES${RESET}
|
|
- incus client (for remote management)
|
|
- incusos-proxmox sibling script (for deploy phase)
|
|
- Already-deployed IncusOS VMs (or use deploy phase)
|
|
|
|
EOF
|
|
exit 0
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Argument parsing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
PHASE="all"
|
|
SKIP_DEPLOY=false
|
|
CLEANUP_MODE=false
|
|
YES=false
|
|
DRY_RUN=false
|
|
QUIET=false
|
|
CONFIG_FILE=""
|
|
|
|
parse_args() {
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
-p|--phase)
|
|
PHASE="${2:?'--phase requires a value'}"
|
|
shift 2
|
|
;;
|
|
--skip-deploy)
|
|
SKIP_DEPLOY=true
|
|
shift
|
|
;;
|
|
--cleanup)
|
|
CLEANUP_MODE=true
|
|
shift
|
|
;;
|
|
-y|--yes)
|
|
YES=true
|
|
shift
|
|
;;
|
|
--dry-run)
|
|
DRY_RUN=true
|
|
shift
|
|
;;
|
|
-q|--quiet)
|
|
QUIET=true
|
|
shift
|
|
;;
|
|
-h|--help)
|
|
usage
|
|
;;
|
|
-V|--version)
|
|
echo "${SCRIPT_NAME} v${VERSION}"
|
|
exit 0
|
|
;;
|
|
-*)
|
|
error "Unknown option: $1"
|
|
echo "Run '${SCRIPT_NAME} --help' for usage information." >&2
|
|
exit 1
|
|
;;
|
|
*)
|
|
if [[ -z "$CONFIG_FILE" ]]; then
|
|
CONFIG_FILE="$1"
|
|
else
|
|
error "Unexpected argument: $1 (config file already set to ${CONFIG_FILE})"
|
|
exit 1
|
|
fi
|
|
shift
|
|
;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
validate_args() {
|
|
if [[ -z "$CONFIG_FILE" ]]; then
|
|
error "No configuration file specified"
|
|
echo "Usage: ${SCRIPT_NAME} [OPTIONS] CONFIG_FILE" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ ! -f "$CONFIG_FILE" ]]; then
|
|
error "Configuration file not found: ${CONFIG_FILE}"
|
|
exit 1
|
|
fi
|
|
|
|
case "$PHASE" in
|
|
deploy|single|cluster|workload|migrate|storage|network|projects|backup|limits|security|resilience|all) ;;
|
|
*)
|
|
error "Invalid phase: ${PHASE}"
|
|
error "Supported: deploy, single, cluster, workload, migrate, storage, network, projects, backup, limits, security, resilience, all"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
if ! command -v incus &>/dev/null; then
|
|
error "incus client is required but not found"
|
|
error "Install the Incus client (https://github.com/lxc/incus)"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Config parsing (reuse python3 YAML->JSON approach)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
CONFIG_JSON=""
|
|
PROXMOX_TOOL=""
|
|
WORKDIR=""
|
|
|
|
declare -a VM_NAMES=()
|
|
|
|
parse_config() {
|
|
step "Parsing configuration: ${CONFIG_FILE}"
|
|
|
|
WORKDIR=$(mktemp -d "${TMPDIR:-/tmp}/${SCRIPT_NAME}-XXXXXX")
|
|
trap 'rm -rf "$WORKDIR"' EXIT
|
|
|
|
CONFIG_JSON="${WORKDIR}/config.json"
|
|
|
|
if ! python3 -c "
|
|
import json, sys
|
|
try:
|
|
import yaml
|
|
with open(sys.argv[1]) as f:
|
|
data = yaml.safe_load(f)
|
|
except ImportError:
|
|
import re
|
|
data = {}
|
|
with open(sys.argv[1]) as f:
|
|
lines = f.readlines()
|
|
processed = []
|
|
for raw in lines:
|
|
s = raw.rstrip()
|
|
if not s or s.lstrip().startswith('#'):
|
|
continue
|
|
indent = len(s) - len(s.lstrip())
|
|
content = s.lstrip()
|
|
processed.append((indent, content))
|
|
# Simple parse for top-level sections and vms list
|
|
result = {}
|
|
section = None
|
|
current_list = None
|
|
current_item = None
|
|
for indent, content in processed:
|
|
m = re.match(r'^-\s+(.+?):\s*(.*)$', content)
|
|
if m and current_list is not None:
|
|
if current_item is not None:
|
|
current_list.append(current_item)
|
|
current_item = {}
|
|
key = m.group(1).strip()
|
|
val = m.group(2).strip()
|
|
if val:
|
|
if val in ('true', 'false'):
|
|
val = val == 'true'
|
|
elif val.isdigit():
|
|
val = int(val)
|
|
elif val.startswith(('\"', \"'\")):
|
|
val = val[1:-1]
|
|
current_item[key] = val
|
|
continue
|
|
if indent > 0 and current_item is not None and not content.startswith('-'):
|
|
m2 = re.match(r'^([^:]+?):\s*(.*)$', content)
|
|
if m2:
|
|
key = m2.group(1).strip()
|
|
val = m2.group(2).strip()
|
|
if val in ('true', 'false'):
|
|
val = val == 'true'
|
|
elif val.isdigit():
|
|
val = int(val)
|
|
elif val.startswith(('\"', \"'\")):
|
|
val = val[1:-1]
|
|
elif '#' in val:
|
|
val = re.sub(r'\s+#.*$', '', val)
|
|
current_item[key] = val
|
|
continue
|
|
m3 = re.match(r'^([^:]+?):\s*(.*)$', content)
|
|
if m3:
|
|
key = m3.group(1).strip()
|
|
val = m3.group(2).strip()
|
|
if key == 'vms':
|
|
current_list = []
|
|
current_item = None
|
|
result['vms'] = current_list
|
|
section = 'vms'
|
|
elif not val:
|
|
section = key
|
|
result[section] = {}
|
|
current_list = None
|
|
if current_item is not None:
|
|
pass
|
|
current_item = None
|
|
elif section and section != 'vms':
|
|
if val in ('true', 'false'):
|
|
val = val == 'true'
|
|
elif val.isdigit():
|
|
val = int(val)
|
|
elif val.startswith(('\"', \"'\")):
|
|
val = val[1:-1]
|
|
elif '#' in val:
|
|
val = re.sub(r'\s+#.*$', '', val)
|
|
result[section][key] = val
|
|
if current_item is not None and current_list is not None:
|
|
current_list.append(current_item)
|
|
data = result
|
|
json.dump(data, sys.stdout, indent=2)
|
|
" "$CONFIG_FILE" > "$CONFIG_JSON" 2>/dev/null; then
|
|
error "Failed to parse configuration: ${CONFIG_FILE}"
|
|
exit 1
|
|
fi
|
|
|
|
# Load VM names
|
|
local count
|
|
if command -v jq &>/dev/null; then
|
|
count=$(jq -r '.vms | length' "$CONFIG_JSON" 2>/dev/null || echo "0")
|
|
else
|
|
count=$(python3 -c "
|
|
import json
|
|
data = json.load(open('$CONFIG_JSON'))
|
|
print(len(data.get('vms', [])))
|
|
" 2>/dev/null || echo "0")
|
|
fi
|
|
|
|
local i=0
|
|
while [[ $i -lt $count ]]; do
|
|
local name
|
|
if command -v jq &>/dev/null; then
|
|
name=$(jq -r ".vms[${i}].name // empty" "$CONFIG_JSON" 2>/dev/null)
|
|
else
|
|
name=$(python3 -c "
|
|
import json
|
|
data = json.load(open('$CONFIG_JSON'))
|
|
print(data.get('vms', [])[${i}].get('name', ''))
|
|
" 2>/dev/null)
|
|
fi
|
|
VM_NAMES+=("$name")
|
|
i=$((i + 1))
|
|
done
|
|
|
|
# Find sibling incusos-proxmox
|
|
local script_dir
|
|
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROXMOX_TOOL="${script_dir}/incusos-proxmox"
|
|
if [[ ! -x "$PROXMOX_TOOL" ]]; then
|
|
warn "incusos-proxmox not found at ${PROXMOX_TOOL}"
|
|
warn "Deploy phase will be unavailable"
|
|
PROXMOX_TOOL=""
|
|
fi
|
|
|
|
success "Loaded ${#VM_NAMES[@]} VM(s): ${VM_NAMES[*]}"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Confirmation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
confirm() {
|
|
local message="$1"
|
|
if [[ "$YES" == true ]] || [[ "$DRY_RUN" == true ]]; then
|
|
return 0
|
|
fi
|
|
echo -n -e "${BOLD}${message} [y/N] ${RESET}"
|
|
local reply
|
|
read -r reply
|
|
if [[ ! "$reply" =~ ^[Yy]$ ]]; then
|
|
info "Skipped"
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Utility: run or preview
|
|
# ---------------------------------------------------------------------------
|
|
|
|
run_cmd() {
|
|
local desc="$1"
|
|
shift
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] $*"
|
|
return 0
|
|
fi
|
|
|
|
detail "Running: $*"
|
|
if "$@"; then
|
|
return 0
|
|
else
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Wait for incus remote to be reachable (port 8443)
|
|
wait_for_remote() {
|
|
local remote="$1"
|
|
local timeout="${2:-120}"
|
|
local elapsed=0
|
|
|
|
while [[ $elapsed -lt $timeout ]]; do
|
|
if incus info "${remote}:" &>/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
sleep 5
|
|
elapsed=$((elapsed + 5))
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 1: Deploy & Verify
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_deploy() {
|
|
step "Phase 1: Deploy & Verify"
|
|
echo ""
|
|
|
|
if [[ "$SKIP_DEPLOY" == true ]]; then
|
|
info "Skipping deploy (--skip-deploy)"
|
|
elif [[ -n "$PROXMOX_TOOL" ]]; then
|
|
step "Deploying VMs via incusos-proxmox"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] ${PROXMOX_TOOL} --yes ${CONFIG_FILE}"
|
|
else
|
|
"$PROXMOX_TOOL" --yes "$CONFIG_FILE" || true
|
|
fi
|
|
echo ""
|
|
else
|
|
warn "incusos-proxmox not found, skipping deployment"
|
|
fi
|
|
|
|
# Check status
|
|
if [[ -n "$PROXMOX_TOOL" ]] && [[ "$DRY_RUN" != true ]]; then
|
|
step "Checking deployment status"
|
|
"$PROXMOX_TOOL" --status "$CONFIG_FILE" || true
|
|
echo ""
|
|
fi
|
|
|
|
# Verify incus remotes exist for each VM
|
|
step "Verifying incus remotes"
|
|
local all_remotes_ok=true
|
|
|
|
for name in "${VM_NAMES[@]}"; do
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] Would check incus remote '${name}'"
|
|
continue
|
|
fi
|
|
|
|
if incus remote list --format csv 2>/dev/null | grep -q "^${name},"; then
|
|
if incus info "${name}:" &>/dev/null 2>&1; then
|
|
pass "Remote '${name}' connected"
|
|
else
|
|
fail "Remote '${name}' exists but connection failed"
|
|
all_remotes_ok=false
|
|
fi
|
|
else
|
|
warn "Remote '${name}' not configured"
|
|
detail " Try: incus remote add ${name} <IP> --accept-certificate"
|
|
all_remotes_ok=false
|
|
fi
|
|
done
|
|
|
|
if [[ "$all_remotes_ok" != true ]] && [[ "$DRY_RUN" != true ]]; then
|
|
warn "Not all remotes are configured. Some phases may fail."
|
|
if ! confirm "Continue anyway?"; then
|
|
return 1
|
|
fi
|
|
fi
|
|
|
|
echo ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 2: Single-Node Workloads
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_single() {
|
|
step "Phase 2: Single-Node Workloads"
|
|
echo ""
|
|
|
|
# Pick first node
|
|
local target="${VM_NAMES[0]}"
|
|
info "Using node: ${target}"
|
|
echo ""
|
|
|
|
if [[ "$DRY_RUN" != true ]] && ! incus info "${target}:" &>/dev/null 2>&1; then
|
|
fail "Cannot connect to remote '${target}'"
|
|
return 1
|
|
fi
|
|
|
|
# Test 1: Launch container
|
|
step "Test: Launch container"
|
|
local ct_name="test-ct-$$"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus launch images:debian/12 ${target}:${ct_name}"
|
|
detail "[dry-run] incus exec ${target}:${ct_name} -- hostname"
|
|
detail "[dry-run] incus delete ${target}:${ct_name} --force"
|
|
pass "Container lifecycle (dry-run)"
|
|
else
|
|
if incus launch images:debian/12 "${target}:${ct_name}" 2>/dev/null; then
|
|
# Wait for container to be running
|
|
sleep 5
|
|
local ct_hostname
|
|
ct_hostname=$(incus exec "${target}:${ct_name}" -- hostname 2>/dev/null) || ct_hostname=""
|
|
if [[ -n "$ct_hostname" ]]; then
|
|
pass "Container launch + exec (hostname: ${ct_hostname})"
|
|
else
|
|
fail "Container launched but exec failed"
|
|
fi
|
|
# Cleanup
|
|
incus delete "${target}:${ct_name}" --force 2>/dev/null || true
|
|
detail "Container cleaned up"
|
|
else
|
|
fail "Container launch failed"
|
|
detail " This might mean the image server is unreachable from the node"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Test 2: Launch VM
|
|
step "Test: Launch VM"
|
|
local vm_name="test-vm-$$"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus launch images:debian/12 ${target}:${vm_name} --vm"
|
|
detail "[dry-run] (wait for agent)"
|
|
detail "[dry-run] incus exec ${target}:${vm_name} -- hostname"
|
|
detail "[dry-run] incus delete ${target}:${vm_name} --force"
|
|
pass "VM lifecycle (dry-run)"
|
|
else
|
|
if incus launch images:debian/12 "${target}:${vm_name}" --vm 2>/dev/null; then
|
|
# VMs take longer to boot -- wait for agent
|
|
detail "Waiting for VM agent (up to 120s)..."
|
|
local vm_ready=false
|
|
local elapsed=0
|
|
while [[ $elapsed -lt 120 ]]; do
|
|
if incus exec "${target}:${vm_name}" -- true &>/dev/null 2>&1; then
|
|
vm_ready=true
|
|
break
|
|
fi
|
|
sleep 10
|
|
elapsed=$((elapsed + 10))
|
|
detail " waiting... (${elapsed}s)"
|
|
done
|
|
|
|
if [[ "$vm_ready" == true ]]; then
|
|
local vm_hostname
|
|
vm_hostname=$(incus exec "${target}:${vm_name}" -- hostname 2>/dev/null) || vm_hostname=""
|
|
if [[ -n "$vm_hostname" ]]; then
|
|
pass "VM launch + exec (hostname: ${vm_hostname})"
|
|
else
|
|
fail "VM agent ready but exec failed"
|
|
fi
|
|
else
|
|
fail "VM launched but agent did not become ready within 120s"
|
|
fi
|
|
# Cleanup
|
|
incus delete "${target}:${vm_name}" --force 2>/dev/null || true
|
|
detail "VM cleaned up"
|
|
else
|
|
fail "VM launch failed"
|
|
detail " Nested virtualization may not be available"
|
|
fi
|
|
fi
|
|
echo ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 3: Cluster Formation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_cluster() {
|
|
step "Phase 3: Cluster Formation"
|
|
echo ""
|
|
|
|
if [[ ${#VM_NAMES[@]} -lt 2 ]]; then
|
|
skip "Clustering requires at least 2 nodes (have ${#VM_NAMES[@]})"
|
|
echo ""
|
|
return 0
|
|
fi
|
|
|
|
local init_node="${VM_NAMES[0]}"
|
|
info "Init node: ${init_node}"
|
|
echo ""
|
|
|
|
if [[ "$DRY_RUN" != true ]] && ! incus info "${init_node}:" &>/dev/null 2>&1; then
|
|
fail "Cannot connect to init node '${init_node}'"
|
|
return 1
|
|
fi
|
|
|
|
# Check if already clustered
|
|
if [[ "$DRY_RUN" != true ]]; then
|
|
local cluster_list
|
|
cluster_list=$(incus cluster list "${init_node}:" --format csv 2>/dev/null) || cluster_list=""
|
|
if [[ -n "$cluster_list" ]]; then
|
|
local member_count
|
|
member_count=$(echo "$cluster_list" | wc -l)
|
|
info "Cluster already exists with ${member_count} member(s)"
|
|
incus cluster list "${init_node}:" 2>/dev/null || true
|
|
pass "Cluster already formed"
|
|
echo ""
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
# Step 1: Set core.https_address on all nodes (required for clustering)
|
|
step "Setting core.https_address on all nodes"
|
|
local -A NODE_IPS=()
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] Would query and set core.https_address on all nodes"
|
|
else
|
|
local n=0
|
|
while [[ $n -lt ${#VM_NAMES[@]} ]]; do
|
|
local node="${VM_NAMES[$n]}"
|
|
local node_ip
|
|
node_ip=$(incus query "${node}:/1.0" 2>/dev/null | \
|
|
python3 -c "
|
|
import sys, json
|
|
d = json.load(sys.stdin)
|
|
for a in d.get('environment', {}).get('addresses', []):
|
|
if not a.startswith('10.') and not a.startswith('fd42:') and not a.startswith('['):
|
|
print(a)
|
|
break
|
|
" 2>/dev/null) || node_ip=""
|
|
if [[ -z "$node_ip" ]]; then
|
|
fail "Cannot determine IP for ${node}"
|
|
return 1
|
|
fi
|
|
NODE_IPS["$node"]="$node_ip"
|
|
incus config set "${node}:" core.https_address "${node_ip}:8443" 2>/dev/null
|
|
detail "${node}: ${node_ip}:8443"
|
|
n=$((n + 1))
|
|
done
|
|
pass "core.https_address set on all nodes"
|
|
fi
|
|
echo ""
|
|
|
|
# Step 2: Enable clustering on init node
|
|
step "Enabling clustering on ${init_node}"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus cluster enable ${init_node}: ${init_node}"
|
|
else
|
|
if incus cluster enable "${init_node}:" "${init_node}" 2>/dev/null; then
|
|
pass "Clustering enabled on ${init_node}"
|
|
# Fix remote (cluster enable regenerates TLS cert)
|
|
incus remote remove "${init_node}" 2>/dev/null || true
|
|
incus remote add "${init_node}" "https://${NODE_IPS[$init_node]}:8443" \
|
|
--accept-certificate 2>/dev/null || true
|
|
detail "Remote ${init_node} re-added with new cluster cert"
|
|
else
|
|
fail "Failed to enable clustering on ${init_node}"
|
|
return 1
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Step 3-N: Join remaining nodes
|
|
local i=1
|
|
while [[ $i -lt ${#VM_NAMES[@]} ]]; do
|
|
local join_node="${VM_NAMES[$i]}"
|
|
|
|
step "Joining ${join_node} to cluster"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus cluster add ${init_node}: ${join_node}"
|
|
detail "[dry-run] incus cluster join ${init_node}: ${join_node}:"
|
|
pass "Node ${join_node} joined (dry-run)"
|
|
else
|
|
if ! incus info "${join_node}:" &>/dev/null 2>&1; then
|
|
fail "Cannot connect to '${join_node}'"
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
# Check if joining node has storage pool (apply_defaults: true)
|
|
# and clean up if needed for cluster join
|
|
if incus storage show "${join_node}:local" &>/dev/null 2>&1; then
|
|
detail "Cleaning up defaults on ${join_node} for cluster join"
|
|
incus config unset "${join_node}:" storage.backups_volume 2>/dev/null || true
|
|
incus config unset "${join_node}:" storage.images_volume 2>/dev/null || true
|
|
incus storage volume delete "${join_node}:local" backups 2>/dev/null || true
|
|
incus storage volume delete "${join_node}:local" images 2>/dev/null || true
|
|
incus profile device remove "${join_node}:default" root 2>/dev/null || true
|
|
incus profile device remove "${join_node}:default" eth0 2>/dev/null || true
|
|
incus storage delete "${join_node}:local" 2>/dev/null || true
|
|
incus network delete "${join_node}:incusbr0" 2>/dev/null || true
|
|
detail "Defaults cleaned up"
|
|
fi
|
|
|
|
# Generate join token
|
|
local token
|
|
token=$(incus cluster add "${init_node}:" "${join_node}" 2>/dev/null) || token=""
|
|
if [[ -z "$token" ]]; then
|
|
fail "Failed to generate join token for ${join_node}"
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
detail "Join token generated"
|
|
|
|
# Join the cluster (automated: accept defaults, answer prompts)
|
|
if printf '\n\nyes\nlocal/incus\nlocal/incus\n' | \
|
|
incus cluster join "${init_node}:" "${join_node}:" 2>/dev/null; then
|
|
pass "Node ${join_node} joined the cluster"
|
|
# Fix remote (join regenerates TLS cert)
|
|
incus remote remove "${join_node}" 2>/dev/null || true
|
|
incus remote add "${join_node}" "https://${NODE_IPS[$join_node]}:8443" \
|
|
--accept-certificate 2>/dev/null || true
|
|
detail "Remote ${join_node} re-added with cluster cert"
|
|
else
|
|
fail "Failed to join ${join_node} to cluster"
|
|
fi
|
|
fi
|
|
|
|
echo ""
|
|
i=$((i + 1))
|
|
done
|
|
|
|
# Verify cluster
|
|
step "Verifying cluster membership"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus cluster list ${init_node}:"
|
|
pass "Cluster verification (dry-run)"
|
|
else
|
|
local cluster_output
|
|
cluster_output=$(incus cluster list "${init_node}:" 2>/dev/null) || cluster_output=""
|
|
if [[ -n "$cluster_output" ]]; then
|
|
echo "$cluster_output"
|
|
local online_count
|
|
online_count=$(incus cluster list "${init_node}:" --format csv 2>/dev/null | grep -c "ONLINE" || echo "0")
|
|
if [[ $online_count -ge ${#VM_NAMES[@]} ]]; then
|
|
pass "All ${online_count} cluster members online"
|
|
else
|
|
warn "Only ${online_count}/${#VM_NAMES[@]} members online"
|
|
fi
|
|
else
|
|
fail "Could not list cluster members"
|
|
fi
|
|
fi
|
|
echo ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 4: Cluster Workloads
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_workload() {
|
|
step "Phase 4: Cluster Workloads"
|
|
echo ""
|
|
|
|
local cluster_remote="${VM_NAMES[0]}"
|
|
|
|
if [[ "$DRY_RUN" != true ]] && ! incus info "${cluster_remote}:" &>/dev/null 2>&1; then
|
|
fail "Cannot connect to cluster remote '${cluster_remote}'"
|
|
return 1
|
|
fi
|
|
|
|
# Check if this node is clustered
|
|
if [[ "$DRY_RUN" != true ]]; then
|
|
local is_clustered=false
|
|
incus cluster list "${cluster_remote}:" &>/dev/null 2>&1 && is_clustered=true
|
|
if [[ "$is_clustered" != true ]]; then
|
|
warn "Node '${cluster_remote}' is not clustered"
|
|
warn "Running workloads as single-node tests"
|
|
fi
|
|
fi
|
|
|
|
# Test 1: Container on cluster
|
|
step "Test: Launch container on cluster"
|
|
local ct_name="test-cluster-ct-$$"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus launch images:debian/12 ${cluster_remote}:${ct_name}"
|
|
pass "Cluster container (dry-run)"
|
|
else
|
|
if incus launch images:debian/12 "${cluster_remote}:${ct_name}" 2>/dev/null; then
|
|
sleep 5
|
|
local location
|
|
location=$(incus list "${cluster_remote}:" --format csv -c nL 2>/dev/null | grep "^${ct_name}," | cut -d',' -f2) || location="unknown"
|
|
pass "Cluster container launched (location: ${location})"
|
|
else
|
|
fail "Cluster container launch failed"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Test 2: VM on specific node (if we have a second node)
|
|
if [[ ${#VM_NAMES[@]} -ge 2 ]]; then
|
|
local target_node="${VM_NAMES[1]}"
|
|
step "Test: Launch VM on specific node (${target_node})"
|
|
local vm_name="test-cluster-vm-$$"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus launch images:debian/12 ${cluster_remote}:${vm_name} --vm --target ${target_node}"
|
|
pass "Targeted VM (dry-run)"
|
|
else
|
|
if incus launch images:debian/12 "${cluster_remote}:${vm_name}" --vm --target "${target_node}" 2>/dev/null; then
|
|
detail "Waiting for VM agent..."
|
|
local elapsed=0
|
|
local vm_ready=false
|
|
while [[ $elapsed -lt 120 ]]; do
|
|
if incus exec "${cluster_remote}:${vm_name}" -- true &>/dev/null 2>&1; then
|
|
vm_ready=true
|
|
break
|
|
fi
|
|
sleep 10
|
|
elapsed=$((elapsed + 10))
|
|
done
|
|
local location
|
|
location=$(incus list "${cluster_remote}:" --format csv -c nL 2>/dev/null | grep "^${vm_name}," | cut -d',' -f2) || location="unknown"
|
|
if [[ "$vm_ready" == true ]]; then
|
|
pass "Targeted VM running on ${location}"
|
|
else
|
|
warn "VM launched on ${location} but agent not ready"
|
|
pass "Targeted VM launched (agent timeout)"
|
|
fi
|
|
else
|
|
fail "Targeted VM launch failed"
|
|
fi
|
|
fi
|
|
echo ""
|
|
fi
|
|
|
|
# Show cluster instances
|
|
if [[ "$DRY_RUN" != true ]]; then
|
|
step "Current instances on cluster"
|
|
incus list "${cluster_remote}:" 2>/dev/null || true
|
|
echo ""
|
|
fi
|
|
|
|
# Cleanup test instances
|
|
step "Cleaning up test instances"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] Would delete test instances"
|
|
else
|
|
incus delete "${cluster_remote}:${ct_name}" --force 2>/dev/null || true
|
|
local vm_name="test-cluster-vm-$$"
|
|
incus delete "${cluster_remote}:${vm_name}" --force 2>/dev/null || true
|
|
detail "Test instances cleaned up"
|
|
fi
|
|
echo ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 5: Migration & Evacuation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_migrate() {
|
|
step "Phase 5: Migration & Evacuation"
|
|
echo ""
|
|
|
|
local cluster_remote="${VM_NAMES[0]}"
|
|
|
|
if [[ ${#VM_NAMES[@]} -lt 2 ]]; then
|
|
skip "Migration tests require at least 2 nodes"
|
|
echo ""
|
|
return 0
|
|
fi
|
|
|
|
if [[ "$DRY_RUN" != true ]] && ! incus info "${cluster_remote}:" &>/dev/null 2>&1; then
|
|
fail "Cannot connect to cluster remote '${cluster_remote}'"
|
|
return 1
|
|
fi
|
|
|
|
# Check if clustered
|
|
if [[ "$DRY_RUN" != true ]]; then
|
|
if ! incus cluster list "${cluster_remote}:" &>/dev/null 2>&1; then
|
|
skip "Node is not clustered, skipping migration tests"
|
|
echo ""
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
local target_node="${VM_NAMES[1]}"
|
|
local ct_name="test-migrate-ct-$$"
|
|
|
|
# Test 1: Container migration (stop/move)
|
|
step "Test: Container migration"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus launch images:debian/12 ${cluster_remote}:${ct_name}"
|
|
detail "[dry-run] incus stop ${cluster_remote}:${ct_name}"
|
|
detail "[dry-run] incus move ${cluster_remote}:${ct_name} --target ${target_node}"
|
|
detail "[dry-run] incus start ${cluster_remote}:${ct_name}"
|
|
pass "Container migration (dry-run)"
|
|
else
|
|
if incus launch images:debian/12 "${cluster_remote}:${ct_name}" 2>/dev/null; then
|
|
sleep 5
|
|
local orig_location
|
|
orig_location=$(incus list "${cluster_remote}:" --format csv -c nL 2>/dev/null | grep "^${ct_name}," | cut -d',' -f2) || orig_location="unknown"
|
|
detail "Container on: ${orig_location}"
|
|
|
|
# Stop and move
|
|
incus stop "${cluster_remote}:${ct_name}" 2>/dev/null || true
|
|
sleep 2
|
|
|
|
if incus move "${cluster_remote}:${ct_name}" --target "${target_node}" 2>/dev/null; then
|
|
incus start "${cluster_remote}:${ct_name}" 2>/dev/null || true
|
|
sleep 3
|
|
|
|
local new_location
|
|
new_location=$(incus list "${cluster_remote}:" --format csv -c nL 2>/dev/null | grep "^${ct_name}," | cut -d',' -f2) || new_location="unknown"
|
|
|
|
if [[ "$new_location" == "$target_node" ]]; then
|
|
pass "Container migrated: ${orig_location} -> ${new_location}"
|
|
else
|
|
warn "Container location: ${new_location} (expected: ${target_node})"
|
|
pass "Container move completed (location may differ due to naming)"
|
|
fi
|
|
else
|
|
fail "Container move failed"
|
|
fi
|
|
|
|
# Cleanup
|
|
incus delete "${cluster_remote}:${ct_name}" --force 2>/dev/null || true
|
|
else
|
|
fail "Container launch failed for migration test"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Test 2: Node evacuation (requires 3+ nodes)
|
|
if [[ ${#VM_NAMES[@]} -ge 3 ]]; then
|
|
local evacuate_node="${VM_NAMES[2]}"
|
|
step "Test: Node evacuation (${evacuate_node})"
|
|
|
|
local evac_ct="test-evac-ct-$$"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus launch images:debian/12 ${cluster_remote}:${evac_ct} --target ${evacuate_node}"
|
|
detail "[dry-run] incus cluster evacuate ${evacuate_node} ${cluster_remote}:"
|
|
detail "[dry-run] incus cluster restore ${evacuate_node} ${cluster_remote}:"
|
|
pass "Node evacuation (dry-run)"
|
|
else
|
|
# Place a container on the node we'll evacuate
|
|
if incus launch images:debian/12 "${cluster_remote}:${evac_ct}" --target "${evacuate_node}" 2>/dev/null; then
|
|
sleep 5
|
|
detail "Container placed on ${evacuate_node}"
|
|
|
|
# Evacuate
|
|
if incus cluster evacuate "${evacuate_node}" "${cluster_remote}:" --force 2>/dev/null; then
|
|
sleep 5
|
|
# Check that nothing is on the evacuated node
|
|
local remaining
|
|
remaining=$(incus list "${cluster_remote}:" --format csv -c nL 2>/dev/null | grep ",${evacuate_node}$" | wc -l) || remaining="0"
|
|
if [[ "$remaining" -eq 0 ]]; then
|
|
pass "Node ${evacuate_node} evacuated (0 instances remaining)"
|
|
else
|
|
warn "${remaining} instance(s) still on ${evacuate_node} after evacuation"
|
|
fi
|
|
|
|
# Restore
|
|
if incus cluster restore "${evacuate_node}" "${cluster_remote}:" 2>/dev/null; then
|
|
pass "Node ${evacuate_node} restored"
|
|
else
|
|
warn "Node restore failed (may need manual intervention)"
|
|
fi
|
|
else
|
|
fail "Evacuation of ${evacuate_node} failed"
|
|
fi
|
|
|
|
# Cleanup
|
|
incus delete "${cluster_remote}:${evac_ct}" --force 2>/dev/null || true
|
|
else
|
|
fail "Could not place container on ${evacuate_node} for evacuation test"
|
|
fi
|
|
fi
|
|
echo ""
|
|
else
|
|
skip "Evacuation test requires 3+ nodes (have ${#VM_NAMES[@]})"
|
|
echo ""
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cleanup mode: remove test artifacts
|
|
# ---------------------------------------------------------------------------
|
|
|
|
do_cleanup() {
|
|
step "Cleaning up test artifacts"
|
|
echo ""
|
|
|
|
local cluster_remote="${VM_NAMES[0]}"
|
|
|
|
if [[ "$DRY_RUN" != true ]] && ! incus info "${cluster_remote}:" &>/dev/null 2>&1; then
|
|
warn "Cannot connect to '${cluster_remote}', skipping instance cleanup"
|
|
return
|
|
fi
|
|
|
|
# Delete any test instances
|
|
if [[ "$DRY_RUN" != true ]]; then
|
|
local instances
|
|
instances=$(incus list "${cluster_remote}:" --format csv -c n 2>/dev/null | grep "^test-" || true)
|
|
if [[ -n "$instances" ]]; then
|
|
while IFS= read -r inst; do
|
|
[[ -z "$inst" ]] && continue
|
|
detail "Deleting: ${inst}"
|
|
incus delete "${cluster_remote}:${inst}" --force 2>/dev/null || true
|
|
done <<< "$instances"
|
|
success "Test instances cleaned up"
|
|
else
|
|
info "No test instances found"
|
|
fi
|
|
else
|
|
detail "[dry-run] Would delete instances matching 'test-*'"
|
|
fi
|
|
|
|
echo ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 6: Storage Operations
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_storage() {
|
|
step "Phase 6: Storage Operations"
|
|
echo ""
|
|
|
|
local target="${VM_NAMES[0]}"
|
|
info "Using node: ${target}"
|
|
|
|
if [[ "$DRY_RUN" != true ]] && ! incus info "${target}:" &>/dev/null 2>&1; then
|
|
fail "Cannot connect to remote '${target}'"
|
|
return 1
|
|
fi
|
|
|
|
# Test 1: List storage pools
|
|
step "Test: List storage pools"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
pass "Storage pool listing (dry-run)"
|
|
else
|
|
local pools
|
|
pools=$(incus storage list "${target}:" --format csv 2>/dev/null) || pools=""
|
|
if [[ -n "$pools" ]]; then
|
|
pass "Storage pool listing (found pools)"
|
|
detail "$pools"
|
|
else
|
|
fail "No storage pools found"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Test 2: Create custom storage volume
|
|
step "Test: Create and manage storage volume"
|
|
local vol_name="test-vol-$$"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus storage volume create ${target}:local ${vol_name}"
|
|
detail "[dry-run] incus storage volume show ${target}:local ${vol_name}"
|
|
detail "[dry-run] incus storage volume delete ${target}:local ${vol_name}"
|
|
pass "Storage volume lifecycle (dry-run)"
|
|
else
|
|
if incus storage volume create "${target}:local" "${vol_name}" 2>/dev/null; then
|
|
local vol_info
|
|
vol_info=$(incus storage volume show "${target}:local" "${vol_name}" 2>/dev/null) || vol_info=""
|
|
if [[ -n "$vol_info" ]]; then
|
|
pass "Storage volume create + show"
|
|
else
|
|
fail "Volume created but show failed"
|
|
fi
|
|
incus storage volume delete "${target}:local" "${vol_name}" 2>/dev/null || true
|
|
detail "Volume cleaned up"
|
|
else
|
|
fail "Storage volume create failed"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Test 3: Volume snapshot
|
|
step "Test: Volume snapshot lifecycle"
|
|
local snap_vol="test-snapvol-$$"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] create volume, snapshot, restore, delete"
|
|
pass "Volume snapshot lifecycle (dry-run)"
|
|
else
|
|
if incus storage volume create "${target}:local" "${snap_vol}" 2>/dev/null; then
|
|
if incus storage volume snapshot create "${target}:local" "${snap_vol}" test-snap 2>/dev/null; then
|
|
pass "Volume snapshot create"
|
|
# Restore
|
|
if incus storage volume snapshot restore "${target}:local" "${snap_vol}" test-snap 2>/dev/null; then
|
|
pass "Volume snapshot restore"
|
|
else
|
|
fail "Volume snapshot restore failed"
|
|
fi
|
|
else
|
|
fail "Volume snapshot create failed"
|
|
fi
|
|
incus storage volume delete "${target}:local" "${snap_vol}" 2>/dev/null || true
|
|
detail "Snapshot volume cleaned up"
|
|
else
|
|
fail "Storage volume create failed (for snapshot test)"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Test 4: Pool usage reporting
|
|
step "Test: Storage pool usage"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
pass "Pool usage reporting (dry-run)"
|
|
else
|
|
local usage
|
|
usage=$(incus storage info "${target}:local" 2>/dev/null) || usage=""
|
|
if [[ -n "$usage" ]]; then
|
|
pass "Storage pool usage reporting"
|
|
detail "$(echo "$usage" | head -5)"
|
|
else
|
|
fail "Could not get pool usage info"
|
|
fi
|
|
fi
|
|
echo ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 7: Network Features
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_network() {
|
|
step "Phase 7: Network Features"
|
|
echo ""
|
|
|
|
local target="${VM_NAMES[0]}"
|
|
info "Using node: ${target}"
|
|
|
|
if [[ "$DRY_RUN" != true ]] && ! incus info "${target}:" &>/dev/null 2>&1; then
|
|
fail "Cannot connect to remote '${target}'"
|
|
return 1
|
|
fi
|
|
|
|
# Test 1: List networks
|
|
step "Test: List networks"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
pass "Network listing (dry-run)"
|
|
else
|
|
local nets
|
|
nets=$(incus network list "${target}:" --format csv 2>/dev/null) || nets=""
|
|
if [[ -n "$nets" ]]; then
|
|
pass "Network listing"
|
|
detail "$nets"
|
|
else
|
|
fail "No networks found"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Test 2: Create and delete a managed network bridge
|
|
step "Test: Create managed network bridge"
|
|
local net_name="test-br-$$"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus network create ${target}:${net_name}"
|
|
detail "[dry-run] incus network delete ${target}:${net_name}"
|
|
pass "Network bridge lifecycle (dry-run)"
|
|
else
|
|
if incus network create "${target}:${net_name}" \
|
|
ipv4.address=10.99.99.1/24 ipv4.nat=true ipv6.address=none 2>/dev/null; then
|
|
local net_info
|
|
net_info=$(incus network show "${target}:${net_name}" 2>/dev/null) || net_info=""
|
|
if [[ -n "$net_info" ]]; then
|
|
pass "Network bridge create + show"
|
|
else
|
|
fail "Network created but show failed"
|
|
fi
|
|
incus network delete "${target}:${net_name}" 2>/dev/null || true
|
|
detail "Network cleaned up"
|
|
else
|
|
fail "Network bridge create failed"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Test 3: Network ACL
|
|
step "Test: Network ACL create/delete"
|
|
local acl_name="test-acl-$$"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus network acl create/delete"
|
|
pass "Network ACL lifecycle (dry-run)"
|
|
else
|
|
if incus network acl create "${target}:${acl_name}" 2>/dev/null; then
|
|
pass "Network ACL create"
|
|
incus network acl delete "${target}:${acl_name}" 2>/dev/null || true
|
|
detail "ACL cleaned up"
|
|
else
|
|
skip "Network ACL not supported (may require OVN)"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Test 4: Network forward (port forwarding)
|
|
step "Test: Network forward"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus network forward create/delete"
|
|
pass "Network forward (dry-run)"
|
|
else
|
|
# Create a temporary instance to test forwarding
|
|
local fw_ct="test-fw-$$"
|
|
if incus launch images:debian/12 "${target}:${fw_ct}" 2>/dev/null; then
|
|
sleep 5
|
|
local ct_ip
|
|
ct_ip=$(incus list "${target}:${fw_ct}" --format csv -c 4 2>/dev/null | cut -d' ' -f1) || ct_ip=""
|
|
if [[ -n "$ct_ip" ]] && [[ "$ct_ip" != *"()"* ]]; then
|
|
if incus network forward create "${target}:incusbr0" 10.66.66.100 2>/dev/null; then
|
|
pass "Network forward create"
|
|
incus network forward delete "${target}:incusbr0" 10.66.66.100 2>/dev/null || true
|
|
else
|
|
skip "Network forward not supported on this network type"
|
|
fi
|
|
else
|
|
skip "Could not get container IP for forward test"
|
|
fi
|
|
incus delete "${target}:${fw_ct}" --force 2>/dev/null || true
|
|
else
|
|
skip "Could not create container for forward test"
|
|
fi
|
|
fi
|
|
echo ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 8: Projects & Profiles
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_projects() {
|
|
step "Phase 8: Projects & Profiles"
|
|
echo ""
|
|
|
|
local target="${VM_NAMES[0]}"
|
|
info "Using node: ${target}"
|
|
|
|
if [[ "$DRY_RUN" != true ]] && ! incus info "${target}:" &>/dev/null 2>&1; then
|
|
fail "Cannot connect to remote '${target}'"
|
|
return 1
|
|
fi
|
|
|
|
# Test 1: Create project with limits
|
|
step "Test: Create project with resource limits"
|
|
local proj_name="test-proj-$$"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus project create/delete"
|
|
pass "Project lifecycle (dry-run)"
|
|
else
|
|
if incus project create "${target}:${proj_name}" \
|
|
-c features.images=false \
|
|
-c features.profiles=true \
|
|
-c limits.instances=5 2>/dev/null; then
|
|
local proj_info
|
|
proj_info=$(incus project show "${target}:${proj_name}" 2>/dev/null) || proj_info=""
|
|
if [[ -n "$proj_info" ]]; then
|
|
pass "Project create with limits"
|
|
else
|
|
fail "Project created but show failed"
|
|
fi
|
|
incus project delete "${target}:${proj_name}" 2>/dev/null || true
|
|
detail "Project cleaned up"
|
|
else
|
|
fail "Project create failed"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Test 2: Custom profile
|
|
step "Test: Create and apply custom profile"
|
|
local profile_name="test-profile-$$"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus profile create/delete"
|
|
pass "Profile lifecycle (dry-run)"
|
|
else
|
|
if incus profile create "${target}:${profile_name}" 2>/dev/null; then
|
|
# Set some config on the profile
|
|
incus profile set "${target}:${profile_name}" limits.cpu=1 limits.memory=256MiB 2>/dev/null || true
|
|
local prof_info
|
|
prof_info=$(incus profile show "${target}:${profile_name}" 2>/dev/null) || prof_info=""
|
|
if echo "$prof_info" | grep -q "limits.cpu"; then
|
|
pass "Profile create + configure"
|
|
else
|
|
fail "Profile created but config not applied"
|
|
fi
|
|
incus profile delete "${target}:${profile_name}" 2>/dev/null || true
|
|
detail "Profile cleaned up"
|
|
else
|
|
fail "Profile create failed"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Test 3: Profile inheritance
|
|
step "Test: Launch container with custom profile"
|
|
local ct_name="test-prof-ct-$$"
|
|
local prof2="test-prof2-$$"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] create profile, launch container with profile, verify config"
|
|
pass "Profile inheritance (dry-run)"
|
|
else
|
|
if incus profile create "${target}:${prof2}" 2>/dev/null; then
|
|
incus profile set "${target}:${prof2}" limits.cpu=1 2>/dev/null || true
|
|
if incus launch images:debian/12 "${target}:${ct_name}" --profile default --profile "${prof2}" 2>/dev/null; then
|
|
sleep 5
|
|
local ct_config
|
|
ct_config=$(incus config show "${target}:${ct_name}" 2>/dev/null) || ct_config=""
|
|
if echo "$ct_config" | grep -q "limits.cpu"; then
|
|
pass "Container inherits profile config"
|
|
else
|
|
# Profile config may be in expanded config
|
|
local expanded
|
|
expanded=$(incus config show "${target}:${ct_name}" --expanded 2>/dev/null) || expanded=""
|
|
if echo "$expanded" | grep -q "limits.cpu"; then
|
|
pass "Container inherits profile config (expanded)"
|
|
else
|
|
fail "Profile config not inherited"
|
|
fi
|
|
fi
|
|
incus delete "${target}:${ct_name}" --force 2>/dev/null || true
|
|
else
|
|
fail "Container launch with profile failed"
|
|
fi
|
|
incus profile delete "${target}:${prof2}" 2>/dev/null || true
|
|
detail "Cleaned up"
|
|
else
|
|
fail "Profile create failed"
|
|
fi
|
|
fi
|
|
echo ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 9: Backup & Restore
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_backup() {
|
|
step "Phase 9: Backup & Restore"
|
|
echo ""
|
|
|
|
local target="${VM_NAMES[0]}"
|
|
info "Using node: ${target}"
|
|
|
|
if [[ "$DRY_RUN" != true ]] && ! incus info "${target}:" &>/dev/null 2>&1; then
|
|
fail "Cannot connect to remote '${target}'"
|
|
return 1
|
|
fi
|
|
|
|
# Test 1: Instance snapshot lifecycle
|
|
step "Test: Instance snapshot create/restore/delete"
|
|
local ct_name="test-snap-$$"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] create instance, snapshot, restore, delete"
|
|
pass "Instance snapshot lifecycle (dry-run)"
|
|
else
|
|
if incus launch images:debian/12 "${target}:${ct_name}" 2>/dev/null; then
|
|
sleep 5
|
|
# Create a marker file before snapshot
|
|
incus exec "${target}:${ct_name}" -- touch /tmp/before-snap 2>/dev/null || true
|
|
|
|
# Create snapshot
|
|
if incus snapshot create "${target}:${ct_name}" test-snap1 2>/dev/null; then
|
|
pass "Instance snapshot create"
|
|
|
|
# Modify instance after snapshot
|
|
incus exec "${target}:${ct_name}" -- touch /tmp/after-snap 2>/dev/null || true
|
|
|
|
# Restore snapshot
|
|
incus stop "${target}:${ct_name}" --force 2>/dev/null || true
|
|
if incus snapshot restore "${target}:${ct_name}" test-snap1 2>/dev/null; then
|
|
incus start "${target}:${ct_name}" 2>/dev/null || true
|
|
sleep 3
|
|
# Verify marker file exists (from before snapshot)
|
|
if incus exec "${target}:${ct_name}" -- test -f /tmp/before-snap 2>/dev/null; then
|
|
pass "Instance snapshot restore (marker file preserved)"
|
|
else
|
|
fail "Snapshot restored but marker file missing"
|
|
fi
|
|
# Verify after-snap file is gone (rolled back)
|
|
if ! incus exec "${target}:${ct_name}" -- test -f /tmp/after-snap 2>/dev/null; then
|
|
pass "Snapshot rollback (post-snapshot changes reverted)"
|
|
else
|
|
fail "Post-snapshot file still exists after restore"
|
|
fi
|
|
else
|
|
fail "Instance snapshot restore failed"
|
|
fi
|
|
|
|
# Delete snapshot
|
|
if incus snapshot delete "${target}:${ct_name}" test-snap1 2>/dev/null; then
|
|
pass "Instance snapshot delete"
|
|
else
|
|
fail "Instance snapshot delete failed"
|
|
fi
|
|
else
|
|
fail "Instance snapshot create failed"
|
|
fi
|
|
incus delete "${target}:${ct_name}" --force 2>/dev/null || true
|
|
detail "Cleaned up"
|
|
else
|
|
fail "Container launch failed (for snapshot test)"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Test 2: Export/import round-trip
|
|
step "Test: Export and import instance"
|
|
local exp_ct="test-export-$$"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] export/import container"
|
|
pass "Export/import round-trip (dry-run)"
|
|
else
|
|
if incus launch images:debian/12 "${target}:${exp_ct}" 2>/dev/null; then
|
|
sleep 5
|
|
incus exec "${target}:${exp_ct}" -- sh -c "echo test-data > /tmp/export-marker" 2>/dev/null || true
|
|
|
|
local export_file="/tmp/test-export-$$.tar.gz"
|
|
incus stop "${target}:${exp_ct}" --force 2>/dev/null || true
|
|
if incus export "${target}:${exp_ct}" "$export_file" 2>/dev/null; then
|
|
pass "Instance export"
|
|
incus delete "${target}:${exp_ct}" --force 2>/dev/null || true
|
|
|
|
# Import
|
|
if incus import "${target}:" "$export_file" 2>/dev/null; then
|
|
incus start "${target}:${exp_ct}" 2>/dev/null || true
|
|
sleep 5
|
|
local marker
|
|
marker=$(incus exec "${target}:${exp_ct}" -- cat /tmp/export-marker 2>/dev/null) || marker=""
|
|
if [[ "$marker" == "test-data" ]]; then
|
|
pass "Instance import (data preserved)"
|
|
else
|
|
fail "Instance imported but data not preserved"
|
|
fi
|
|
else
|
|
fail "Instance import failed"
|
|
fi
|
|
else
|
|
fail "Instance export failed"
|
|
fi
|
|
incus delete "${target}:${exp_ct}" --force 2>/dev/null || true
|
|
rm -f "$export_file"
|
|
detail "Cleaned up"
|
|
else
|
|
fail "Container launch failed (for export test)"
|
|
fi
|
|
fi
|
|
echo ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 10: Resource Limits
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_limits() {
|
|
step "Phase 10: Resource Limits & QoS"
|
|
echo ""
|
|
|
|
local target="${VM_NAMES[0]}"
|
|
info "Using node: ${target}"
|
|
|
|
if [[ "$DRY_RUN" != true ]] && ! incus info "${target}:" &>/dev/null 2>&1; then
|
|
fail "Cannot connect to remote '${target}'"
|
|
return 1
|
|
fi
|
|
|
|
# Test 1: CPU limit
|
|
step "Test: CPU limit enforcement"
|
|
local ct_name="test-limits-$$"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] launch container with CPU/memory limits"
|
|
pass "CPU limit (dry-run)"
|
|
else
|
|
if incus launch images:debian/12 "${target}:${ct_name}" \
|
|
-c limits.cpu=1 -c limits.memory=256MiB 2>/dev/null; then
|
|
sleep 5
|
|
# Verify limits are applied
|
|
local cpus
|
|
cpus=$(incus exec "${target}:${ct_name}" -- nproc 2>/dev/null) || cpus=""
|
|
if [[ "$cpus" == "1" ]]; then
|
|
pass "CPU limit enforced (nproc=${cpus})"
|
|
else
|
|
fail "CPU limit not enforced (nproc=${cpus}, expected 1)"
|
|
fi
|
|
|
|
# Test 2: Memory limit
|
|
step "Test: Memory limit enforcement"
|
|
local mem_limit
|
|
mem_limit=$(incus exec "${target}:${ct_name}" -- sh -c \
|
|
"cat /sys/fs/cgroup/memory.max 2>/dev/null || cat /sys/fs/cgroup/memory/memory.limit_in_bytes 2>/dev/null || echo unknown" 2>/dev/null) || mem_limit="unknown"
|
|
if [[ "$mem_limit" != "unknown" ]] && [[ "$mem_limit" != "max" ]]; then
|
|
local mem_mib=$((mem_limit / 1024 / 1024))
|
|
if [[ $mem_mib -le 260 ]] && [[ $mem_mib -ge 240 ]]; then
|
|
pass "Memory limit enforced (~${mem_mib}MiB)"
|
|
else
|
|
fail "Memory limit unexpected: ${mem_mib}MiB (expected ~256)"
|
|
fi
|
|
else
|
|
skip "Could not read memory limit (cgroup v1/v2 difference)"
|
|
fi
|
|
|
|
incus delete "${target}:${ct_name}" --force 2>/dev/null || true
|
|
detail "Cleaned up"
|
|
else
|
|
fail "Container launch with limits failed"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Test 3: Disk I/O limits
|
|
step "Test: Disk I/O limit configuration"
|
|
local io_ct="test-io-$$"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] set disk I/O limits"
|
|
pass "Disk I/O limits (dry-run)"
|
|
else
|
|
if incus launch images:debian/12 "${target}:${io_ct}" 2>/dev/null; then
|
|
sleep 5
|
|
# Set I/O limits on root disk
|
|
if incus config device set "${target}:${io_ct}" root limits.read=10MB limits.write=10MB 2>/dev/null; then
|
|
pass "Disk I/O limit configuration accepted"
|
|
else
|
|
skip "Disk I/O limits not supported on this storage driver"
|
|
fi
|
|
incus delete "${target}:${io_ct}" --force 2>/dev/null || true
|
|
detail "Cleaned up"
|
|
else
|
|
fail "Container launch failed (for I/O test)"
|
|
fi
|
|
fi
|
|
echo ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 11: Security
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_security() {
|
|
step "Phase 11: Security"
|
|
echo ""
|
|
|
|
local target="${VM_NAMES[0]}"
|
|
info "Using node: ${target}"
|
|
|
|
if [[ "$DRY_RUN" != true ]] && ! incus info "${target}:" &>/dev/null 2>&1; then
|
|
fail "Cannot connect to remote '${target}'"
|
|
return 1
|
|
fi
|
|
|
|
# Test 1: Unprivileged container (default)
|
|
step "Test: Unprivileged container (default)"
|
|
local ct_name="test-sec-$$"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] verify container is unprivileged"
|
|
pass "Unprivileged container (dry-run)"
|
|
else
|
|
if incus launch images:debian/12 "${target}:${ct_name}" 2>/dev/null; then
|
|
sleep 5
|
|
# Check that the container is running unprivileged (uid mapping)
|
|
local uid
|
|
uid=$(incus exec "${target}:${ct_name}" -- id -u 2>/dev/null) || uid=""
|
|
local config
|
|
config=$(incus config show "${target}:${ct_name}" 2>/dev/null) || config=""
|
|
if echo "$config" | grep -q "security.privileged: \"true\""; then
|
|
fail "Container is privileged (unexpected default)"
|
|
else
|
|
pass "Container is unprivileged (default)"
|
|
fi
|
|
incus delete "${target}:${ct_name}" --force 2>/dev/null || true
|
|
else
|
|
fail "Container launch failed"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Test 2: Privileged container
|
|
step "Test: Privileged container"
|
|
local priv_ct="test-priv-$$"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] launch privileged container"
|
|
pass "Privileged container (dry-run)"
|
|
else
|
|
if incus launch images:debian/12 "${target}:${priv_ct}" \
|
|
-c security.privileged=true 2>/dev/null; then
|
|
sleep 5
|
|
local config
|
|
config=$(incus config show "${target}:${priv_ct}" 2>/dev/null) || config=""
|
|
if echo "$config" | grep -q "security.privileged"; then
|
|
pass "Privileged container created"
|
|
else
|
|
fail "Privileged flag not set"
|
|
fi
|
|
incus delete "${target}:${priv_ct}" --force 2>/dev/null || true
|
|
detail "Cleaned up"
|
|
else
|
|
fail "Privileged container launch failed"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Test 3: Container nesting
|
|
step "Test: Container nesting support"
|
|
local nest_ct="test-nest-$$"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] launch container with nesting enabled"
|
|
pass "Container nesting (dry-run)"
|
|
else
|
|
if incus launch images:debian/12 "${target}:${nest_ct}" \
|
|
-c security.nesting=true 2>/dev/null; then
|
|
sleep 5
|
|
local config
|
|
config=$(incus config show "${target}:${nest_ct}" 2>/dev/null) || config=""
|
|
if echo "$config" | grep -q "security.nesting"; then
|
|
pass "Nesting support enabled"
|
|
else
|
|
fail "Nesting flag not set"
|
|
fi
|
|
incus delete "${target}:${nest_ct}" --force 2>/dev/null || true
|
|
detail "Cleaned up"
|
|
else
|
|
fail "Container with nesting launch failed"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Test 4: ID mapping (user namespace isolation)
|
|
step "Test: User namespace isolation"
|
|
local idmap_ct="test-idmap-$$"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] verify UID/GID mapping in container"
|
|
pass "User namespace isolation (dry-run)"
|
|
else
|
|
if incus launch images:debian/12 "${target}:${idmap_ct}" 2>/dev/null; then
|
|
sleep 5
|
|
# In unprivileged containers, root inside maps to a high UID outside
|
|
local inner_uid outer_pid
|
|
inner_uid=$(incus exec "${target}:${idmap_ct}" -- id -u 2>/dev/null) || inner_uid=""
|
|
if [[ "$inner_uid" == "0" ]]; then
|
|
# Root inside the container maps to a high UID on the host
|
|
pass "User namespace isolation (root maps to uid 0 inside container)"
|
|
else
|
|
fail "Unexpected uid inside container: ${inner_uid}"
|
|
fi
|
|
incus delete "${target}:${idmap_ct}" --force 2>/dev/null || true
|
|
detail "Cleaned up"
|
|
else
|
|
fail "Container launch failed (for idmap test)"
|
|
fi
|
|
fi
|
|
echo ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 12: HA & Cluster Resilience
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_resilience() {
|
|
step "Phase 12: HA & Cluster Resilience"
|
|
echo ""
|
|
|
|
local count=${#VM_NAMES[@]}
|
|
if [[ $count -lt 3 ]]; then
|
|
skip "Resilience tests require at least 3 nodes (have ${count})"
|
|
return
|
|
fi
|
|
|
|
local cluster_remote="${VM_NAMES[0]}"
|
|
info "Using cluster via: ${cluster_remote}"
|
|
|
|
if [[ "$DRY_RUN" != true ]] && ! incus info "${cluster_remote}:" &>/dev/null 2>&1; then
|
|
fail "Cannot connect to remote '${cluster_remote}'"
|
|
return 1
|
|
fi
|
|
|
|
# Verify we're in a cluster
|
|
if [[ "$DRY_RUN" != true ]]; then
|
|
local member_count
|
|
member_count=$(incus cluster list "${cluster_remote}:" --format csv 2>/dev/null | wc -l) || member_count=0
|
|
if [[ $member_count -lt 2 ]]; then
|
|
skip "Not in a cluster (${member_count} members). Run cluster phase first."
|
|
return
|
|
fi
|
|
info "Cluster has ${member_count} members"
|
|
fi
|
|
echo ""
|
|
|
|
# Test 1: Cluster member listing and health
|
|
step "Test: Cluster health check"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus cluster list"
|
|
pass "Cluster health (dry-run)"
|
|
else
|
|
local members
|
|
members=$(incus cluster list "${cluster_remote}:" --format csv -c ns 2>/dev/null) || members=""
|
|
local all_online=true
|
|
while IFS=',' read -r name status; do
|
|
[[ -z "$name" ]] && continue
|
|
if [[ "$status" != "Online" ]] && [[ "$status" != "ONLINE" ]]; then
|
|
fail "Member ${name} is ${status}"
|
|
all_online=false
|
|
fi
|
|
done <<< "$members"
|
|
if [[ "$all_online" == true ]]; then
|
|
pass "All cluster members online"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Test 2: Database leader identification
|
|
step "Test: Database leader identification"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus cluster list (check roles)"
|
|
pass "Database leader (dry-run)"
|
|
else
|
|
local leader
|
|
leader=$(incus cluster list "${cluster_remote}:" --format csv -c nR 2>/dev/null | grep -i "database-leader" | head -1) || leader=""
|
|
if [[ -n "$leader" ]]; then
|
|
pass "Database leader identified: $(echo "$leader" | cut -d',' -f1)"
|
|
else
|
|
# Older Incus may not show roles in CSV
|
|
skip "Could not identify database leader (format may differ)"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Test 3: Workload survives member reboot (via Proxmox stop/start)
|
|
step "Test: Workload availability during member restart"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] Would test workload survival through member restart"
|
|
detail "[dry-run] This test requires Proxmox access and is not safe in dry-run"
|
|
skip "Workload survival test (dry-run -- requires live infrastructure)"
|
|
else
|
|
# This test just verifies that the cluster can handle member state queries
|
|
# Actual stop/start requires Proxmox and is too destructive for automated tests
|
|
local member_list
|
|
member_list=$(incus cluster list "${cluster_remote}:" --format csv -c n 2>/dev/null) || member_list=""
|
|
if [[ -n "$member_list" ]]; then
|
|
pass "Cluster member state queryable (stop/start test requires manual execution)"
|
|
detail "To test: stop a non-leader node via Proxmox, verify cluster still works"
|
|
else
|
|
fail "Cannot query cluster members"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Test 4: Cluster warnings
|
|
step "Test: Cluster warnings check"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
detail "[dry-run] incus warning list"
|
|
pass "Cluster warnings (dry-run)"
|
|
else
|
|
local warnings
|
|
warnings=$(incus warning list "${cluster_remote}:" --format csv 2>/dev/null) || warnings=""
|
|
if [[ -z "$warnings" ]]; then
|
|
pass "No cluster warnings"
|
|
else
|
|
local warn_count
|
|
warn_count=$(echo "$warnings" | wc -l)
|
|
warn "Cluster has ${warn_count} warning(s):"
|
|
detail "$warnings"
|
|
pass "Cluster warnings retrieved (${warn_count} active)"
|
|
fi
|
|
fi
|
|
echo ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Summary
|
|
# ---------------------------------------------------------------------------
|
|
|
|
print_summary() {
|
|
echo ""
|
|
echo -e "${BOLD}Test Summary${RESET}"
|
|
echo -e " ${GREEN}Passed: ${PASSES}${RESET}"
|
|
if [[ $FAILURES -gt 0 ]]; then
|
|
echo -e " ${RED}Failed: ${FAILURES}${RESET}"
|
|
else
|
|
echo -e " Failed: 0"
|
|
fi
|
|
if [[ $SKIPS -gt 0 ]]; then
|
|
echo -e " ${YELLOW}Skipped: ${SKIPS}${RESET}"
|
|
fi
|
|
echo ""
|
|
|
|
if [[ $FAILURES -gt 0 ]]; then
|
|
echo -e " ${RED}${BOLD}Some tests failed${RESET}"
|
|
else
|
|
echo -e " ${GREEN}${BOLD}All tests passed${RESET}"
|
|
fi
|
|
echo ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main
|
|
# ---------------------------------------------------------------------------
|
|
|
|
main() {
|
|
setup_colors
|
|
parse_args "$@"
|
|
validate_args
|
|
parse_config
|
|
|
|
if [[ "$QUIET" != true ]]; then
|
|
echo -e "${BOLD}${SCRIPT_NAME}${RESET} v${VERSION}"
|
|
echo ""
|
|
fi
|
|
|
|
# Handle cleanup mode
|
|
if [[ "$CLEANUP_MODE" == true ]]; then
|
|
do_cleanup
|
|
return
|
|
fi
|
|
|
|
# Run phases
|
|
case "$PHASE" in
|
|
deploy)
|
|
phase_deploy
|
|
;;
|
|
single)
|
|
if [[ "$SKIP_DEPLOY" != true ]]; then phase_deploy; fi
|
|
phase_single
|
|
;;
|
|
cluster)
|
|
if [[ "$SKIP_DEPLOY" != true ]]; then phase_deploy; fi
|
|
phase_cluster
|
|
;;
|
|
workload)
|
|
if [[ "$SKIP_DEPLOY" != true ]]; then phase_deploy; fi
|
|
phase_workload
|
|
;;
|
|
migrate)
|
|
if [[ "$SKIP_DEPLOY" != true ]]; then phase_deploy; fi
|
|
phase_migrate
|
|
;;
|
|
storage)
|
|
if [[ "$SKIP_DEPLOY" != true ]]; then phase_deploy; fi
|
|
phase_storage
|
|
;;
|
|
network)
|
|
if [[ "$SKIP_DEPLOY" != true ]]; then phase_deploy; fi
|
|
phase_network
|
|
;;
|
|
projects)
|
|
if [[ "$SKIP_DEPLOY" != true ]]; then phase_deploy; fi
|
|
phase_projects
|
|
;;
|
|
backup)
|
|
if [[ "$SKIP_DEPLOY" != true ]]; then phase_deploy; fi
|
|
phase_backup
|
|
;;
|
|
limits)
|
|
if [[ "$SKIP_DEPLOY" != true ]]; then phase_deploy; fi
|
|
phase_limits
|
|
;;
|
|
security)
|
|
if [[ "$SKIP_DEPLOY" != true ]]; then phase_deploy; fi
|
|
phase_security
|
|
;;
|
|
resilience)
|
|
if [[ "$SKIP_DEPLOY" != true ]]; then phase_deploy; fi
|
|
phase_resilience
|
|
;;
|
|
all)
|
|
phase_deploy
|
|
phase_single
|
|
phase_cluster
|
|
phase_workload
|
|
phase_migrate
|
|
phase_storage
|
|
phase_network
|
|
phase_projects
|
|
phase_backup
|
|
phase_limits
|
|
phase_security
|
|
phase_resilience
|
|
;;
|
|
esac
|
|
|
|
print_summary
|
|
}
|
|
|
|
main "$@"
|