#!/usr/bin/env bash # lab-test - Guided lab validation for IncusOS deployments # # Walks through lab validation step by step: deploy VMs, test single-node # workloads, form clusters, test migration and evacuation. # # Operates on already-deployed VMs (expects incus remotes to exist) or # can deploy them via incusos-proxmox. # # Usage: lab-test [OPTIONS] CONFIG_FILE # Run 'lab-test --help' for full usage information. set -euo pipefail readonly VERSION="0.1.0" readonly SCRIPT_NAME="lab-test" # --------------------------------------------------------------------------- # Color and formatting # --------------------------------------------------------------------------- setup_colors() { if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[0;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' BOLD='\033[1m' DIM='\033[2m' RESET='\033[0m' else RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' RESET='' fi } info() { [[ "$QUIET" == true ]] && return; echo -e "${BLUE}[info]${RESET} $*"; } success() { [[ "$QUIET" == true ]] && return; echo -e "${GREEN}[ok]${RESET} $*"; } warn() { echo -e "${YELLOW}[warn]${RESET} $*" >&2; } error() { echo -e "${RED}[error]${RESET} $*" >&2; } step() { [[ "$QUIET" == true ]] && return; echo -e "${CYAN}[step]${RESET} ${BOLD}$*${RESET}"; } detail() { [[ "$QUIET" == true ]] && return; echo -e "${DIM} $*${RESET}"; } pass() { echo -e " ${GREEN}PASS${RESET} $*"; PASSES=$((PASSES + 1)); } fail() { echo -e " ${RED}FAIL${RESET} $*"; FAILURES=$((FAILURES + 1)); } skip() { echo -e " ${YELLOW}SKIP${RESET} $*"; SKIPS=$((SKIPS + 1)); } # Test result counters PASSES=0 FAILURES=0 SKIPS=0 # --------------------------------------------------------------------------- # Help # --------------------------------------------------------------------------- usage() { cat <&2 exit 1 ;; *) if [[ -z "$CONFIG_FILE" ]]; then CONFIG_FILE="$1" else error "Unexpected argument: $1 (config file already set to ${CONFIG_FILE})" exit 1 fi shift ;; esac done } validate_args() { if [[ -z "$CONFIG_FILE" ]]; then error "No configuration file specified" echo "Usage: ${SCRIPT_NAME} [OPTIONS] CONFIG_FILE" >&2 exit 1 fi if [[ ! -f "$CONFIG_FILE" ]]; then error "Configuration file not found: ${CONFIG_FILE}" exit 1 fi case "$PHASE" in deploy|single|cluster|workload|migrate|all) ;; *) error "Invalid phase: ${PHASE}" error "Supported: deploy, single, cluster, workload, migrate, all" exit 1 ;; esac if ! command -v incus &>/dev/null; then error "incus client is required but not found" error "Install the Incus client (https://github.com/lxc/incus)" exit 1 fi } # --------------------------------------------------------------------------- # Config parsing (reuse python3 YAML->JSON approach) # --------------------------------------------------------------------------- CONFIG_JSON="" PROXMOX_TOOL="" WORKDIR="" declare -a VM_NAMES=() parse_config() { step "Parsing configuration: ${CONFIG_FILE}" WORKDIR=$(mktemp -d "${TMPDIR:-/tmp}/${SCRIPT_NAME}-XXXXXX") trap 'rm -rf "$WORKDIR"' EXIT CONFIG_JSON="${WORKDIR}/config.json" if ! python3 -c " import json, sys try: import yaml with open(sys.argv[1]) as f: data = yaml.safe_load(f) except ImportError: import re data = {} with open(sys.argv[1]) as f: lines = f.readlines() processed = [] for raw in lines: s = raw.rstrip() if not s or s.lstrip().startswith('#'): continue indent = len(s) - len(s.lstrip()) content = s.lstrip() processed.append((indent, content)) # Simple parse for top-level sections and vms list result = {} section = None current_list = None current_item = None for indent, content in processed: m = re.match(r'^-\s+(.+?):\s*(.*)$', content) if m and current_list is not None: if current_item is not None: current_list.append(current_item) current_item = {} key = m.group(1).strip() val = m.group(2).strip() if val: if val in ('true', 'false'): val = val == 'true' elif val.isdigit(): val = int(val) elif val.startswith(('\"', \"'\")): val = val[1:-1] current_item[key] = val continue if indent > 0 and current_item is not None and not content.startswith('-'): m2 = re.match(r'^([^:]+?):\s*(.*)$', content) if m2: key = m2.group(1).strip() val = m2.group(2).strip() if val in ('true', 'false'): val = val == 'true' elif val.isdigit(): val = int(val) elif val.startswith(('\"', \"'\")): val = val[1:-1] elif '#' in val: val = re.sub(r'\s+#.*$', '', val) current_item[key] = val continue m3 = re.match(r'^([^:]+?):\s*(.*)$', content) if m3: key = m3.group(1).strip() val = m3.group(2).strip() if key == 'vms': current_list = [] current_item = None result['vms'] = current_list section = 'vms' elif not val: section = key result[section] = {} current_list = None if current_item is not None: pass current_item = None elif section and section != 'vms': if val in ('true', 'false'): val = val == 'true' elif val.isdigit(): val = int(val) elif val.startswith(('\"', \"'\")): val = val[1:-1] elif '#' in val: val = re.sub(r'\s+#.*$', '', val) result[section][key] = val if current_item is not None and current_list is not None: current_list.append(current_item) data = result json.dump(data, sys.stdout, indent=2) " "$CONFIG_FILE" > "$CONFIG_JSON" 2>/dev/null; then error "Failed to parse configuration: ${CONFIG_FILE}" exit 1 fi # Load VM names local count if command -v jq &>/dev/null; then count=$(jq -r '.vms | length' "$CONFIG_JSON" 2>/dev/null || echo "0") else count=$(python3 -c " import json data = json.load(open('$CONFIG_JSON')) print(len(data.get('vms', []))) " 2>/dev/null || echo "0") fi local i=0 while [[ $i -lt $count ]]; do local name if command -v jq &>/dev/null; then name=$(jq -r ".vms[${i}].name // empty" "$CONFIG_JSON" 2>/dev/null) else name=$(python3 -c " import json data = json.load(open('$CONFIG_JSON')) print(data.get('vms', [])[${i}].get('name', '')) " 2>/dev/null) fi VM_NAMES+=("$name") i=$((i + 1)) done # Find sibling incusos-proxmox local script_dir script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROXMOX_TOOL="${script_dir}/incusos-proxmox" if [[ ! -x "$PROXMOX_TOOL" ]]; then warn "incusos-proxmox not found at ${PROXMOX_TOOL}" warn "Deploy phase will be unavailable" PROXMOX_TOOL="" fi success "Loaded ${#VM_NAMES[@]} VM(s): ${VM_NAMES[*]}" } # --------------------------------------------------------------------------- # Confirmation # --------------------------------------------------------------------------- confirm() { local message="$1" if [[ "$YES" == true ]] || [[ "$DRY_RUN" == true ]]; then return 0 fi echo -n -e "${BOLD}${message} [y/N] ${RESET}" local reply read -r reply if [[ ! "$reply" =~ ^[Yy]$ ]]; then info "Skipped" return 1 fi return 0 } # --------------------------------------------------------------------------- # Utility: run or preview # --------------------------------------------------------------------------- run_cmd() { local desc="$1" shift if [[ "$DRY_RUN" == true ]]; then detail "[dry-run] $*" return 0 fi detail "Running: $*" if "$@"; then return 0 else return 1 fi } # Wait for incus remote to be reachable (port 8443) wait_for_remote() { local remote="$1" local timeout="${2:-120}" local elapsed=0 while [[ $elapsed -lt $timeout ]]; do if incus info "${remote}:" &>/dev/null 2>&1; then return 0 fi sleep 5 elapsed=$((elapsed + 5)) done return 1 } # --------------------------------------------------------------------------- # Phase 1: Deploy & Verify # --------------------------------------------------------------------------- phase_deploy() { step "Phase 1: Deploy & Verify" echo "" if [[ "$SKIP_DEPLOY" == true ]]; then info "Skipping deploy (--skip-deploy)" elif [[ -n "$PROXMOX_TOOL" ]]; then step "Deploying VMs via incusos-proxmox" if [[ "$DRY_RUN" == true ]]; then detail "[dry-run] ${PROXMOX_TOOL} --yes ${CONFIG_FILE}" else "$PROXMOX_TOOL" --yes "$CONFIG_FILE" || true fi echo "" else warn "incusos-proxmox not found, skipping deployment" fi # Check status if [[ -n "$PROXMOX_TOOL" ]] && [[ "$DRY_RUN" != true ]]; then step "Checking deployment status" "$PROXMOX_TOOL" --status "$CONFIG_FILE" || true echo "" fi # Verify incus remotes exist for each VM step "Verifying incus remotes" local all_remotes_ok=true for name in "${VM_NAMES[@]}"; do if [[ "$DRY_RUN" == true ]]; then detail "[dry-run] Would check incus remote '${name}'" continue fi if incus remote list --format csv 2>/dev/null | grep -q "^${name},"; then if incus info "${name}:" &>/dev/null 2>&1; then pass "Remote '${name}' connected" else fail "Remote '${name}' exists but connection failed" all_remotes_ok=false fi else warn "Remote '${name}' not configured" detail " Try: incus remote add ${name} --accept-certificate" all_remotes_ok=false fi done if [[ "$all_remotes_ok" != true ]] && [[ "$DRY_RUN" != true ]]; then warn "Not all remotes are configured. Some phases may fail." if ! confirm "Continue anyway?"; then return 1 fi fi echo "" } # --------------------------------------------------------------------------- # Phase 2: Single-Node Workloads # --------------------------------------------------------------------------- phase_single() { step "Phase 2: Single-Node Workloads" echo "" # Pick first node local target="${VM_NAMES[0]}" info "Using node: ${target}" echo "" if [[ "$DRY_RUN" != true ]] && ! incus info "${target}:" &>/dev/null 2>&1; then fail "Cannot connect to remote '${target}'" return 1 fi # Test 1: Launch container step "Test: Launch container" local ct_name="test-ct-$$" if [[ "$DRY_RUN" == true ]]; then detail "[dry-run] incus launch images:debian/12 ${target}:${ct_name}" detail "[dry-run] incus exec ${target}:${ct_name} -- hostname" detail "[dry-run] incus delete ${target}:${ct_name} --force" pass "Container lifecycle (dry-run)" else if incus launch images:debian/12 "${target}:${ct_name}" 2>/dev/null; then # Wait for container to be running sleep 5 local ct_hostname ct_hostname=$(incus exec "${target}:${ct_name}" -- hostname 2>/dev/null) || ct_hostname="" if [[ -n "$ct_hostname" ]]; then pass "Container launch + exec (hostname: ${ct_hostname})" else fail "Container launched but exec failed" fi # Cleanup incus delete "${target}:${ct_name}" --force 2>/dev/null || true detail "Container cleaned up" else fail "Container launch failed" detail " This might mean the image server is unreachable from the node" fi fi echo "" # Test 2: Launch VM step "Test: Launch VM" local vm_name="test-vm-$$" if [[ "$DRY_RUN" == true ]]; then detail "[dry-run] incus launch images:debian/12 ${target}:${vm_name} --vm" detail "[dry-run] (wait for agent)" detail "[dry-run] incus exec ${target}:${vm_name} -- hostname" detail "[dry-run] incus delete ${target}:${vm_name} --force" pass "VM lifecycle (dry-run)" else if incus launch images:debian/12 "${target}:${vm_name}" --vm 2>/dev/null; then # VMs take longer to boot -- wait for agent detail "Waiting for VM agent (up to 120s)..." local vm_ready=false local elapsed=0 while [[ $elapsed -lt 120 ]]; do if incus exec "${target}:${vm_name}" -- true &>/dev/null 2>&1; then vm_ready=true break fi sleep 10 elapsed=$((elapsed + 10)) detail " waiting... (${elapsed}s)" done if [[ "$vm_ready" == true ]]; then local vm_hostname vm_hostname=$(incus exec "${target}:${vm_name}" -- hostname 2>/dev/null) || vm_hostname="" if [[ -n "$vm_hostname" ]]; then pass "VM launch + exec (hostname: ${vm_hostname})" else fail "VM agent ready but exec failed" fi else fail "VM launched but agent did not become ready within 120s" fi # Cleanup incus delete "${target}:${vm_name}" --force 2>/dev/null || true detail "VM cleaned up" else fail "VM launch failed" detail " Nested virtualization may not be available" fi fi echo "" } # --------------------------------------------------------------------------- # Phase 3: Cluster Formation # --------------------------------------------------------------------------- phase_cluster() { step "Phase 3: Cluster Formation" echo "" if [[ ${#VM_NAMES[@]} -lt 2 ]]; then skip "Clustering requires at least 2 nodes (have ${#VM_NAMES[@]})" echo "" return 0 fi local init_node="${VM_NAMES[0]}" info "Init node: ${init_node}" echo "" if [[ "$DRY_RUN" != true ]] && ! incus info "${init_node}:" &>/dev/null 2>&1; then fail "Cannot connect to init node '${init_node}'" return 1 fi # Check if already clustered if [[ "$DRY_RUN" != true ]]; then local cluster_list cluster_list=$(incus cluster list "${init_node}:" --format csv 2>/dev/null) || cluster_list="" if [[ -n "$cluster_list" ]]; then local member_count member_count=$(echo "$cluster_list" | wc -l) info "Cluster already exists with ${member_count} member(s)" incus cluster list "${init_node}:" 2>/dev/null || true pass "Cluster already formed" echo "" return 0 fi fi # Step 1: Enable clustering on init node step "Enabling clustering on ${init_node}" if [[ "$DRY_RUN" == true ]]; then detail "[dry-run] incus cluster enable ${init_node}: ${init_node}" else if incus cluster enable "${init_node}:" "${init_node}" 2>/dev/null; then pass "Clustering enabled on ${init_node}" else fail "Failed to enable clustering on ${init_node}" return 1 fi fi echo "" # Step 2-N: Join remaining nodes local i=1 while [[ $i -lt ${#VM_NAMES[@]} ]]; do local join_node="${VM_NAMES[$i]}" step "Joining ${join_node} to cluster" if [[ "$DRY_RUN" == true ]]; then detail "[dry-run] incus cluster add ${init_node}: ${join_node}" detail "[dry-run] incus cluster join ${init_node}: ${join_node}:" pass "Node ${join_node} joined (dry-run)" else if ! incus info "${join_node}:" &>/dev/null 2>&1; then fail "Cannot connect to '${join_node}'" i=$((i + 1)) continue fi # Generate join token local token token=$(incus cluster add "${init_node}:" "${join_node}" 2>/dev/null) || token="" if [[ -z "$token" ]]; then fail "Failed to generate join token for ${join_node}" i=$((i + 1)) continue fi detail "Join token generated" # Join the cluster if incus cluster join "${init_node}:" "${join_node}:" 2>/dev/null; then pass "Node ${join_node} joined the cluster" else fail "Failed to join ${join_node} to cluster" fi fi echo "" i=$((i + 1)) done # Verify cluster step "Verifying cluster membership" if [[ "$DRY_RUN" == true ]]; then detail "[dry-run] incus cluster list ${init_node}:" pass "Cluster verification (dry-run)" else local cluster_output cluster_output=$(incus cluster list "${init_node}:" 2>/dev/null) || cluster_output="" if [[ -n "$cluster_output" ]]; then echo "$cluster_output" local online_count online_count=$(incus cluster list "${init_node}:" --format csv 2>/dev/null | grep -c "ONLINE" || echo "0") if [[ $online_count -ge ${#VM_NAMES[@]} ]]; then pass "All ${online_count} cluster members online" else warn "Only ${online_count}/${#VM_NAMES[@]} members online" fi else fail "Could not list cluster members" fi fi echo "" } # --------------------------------------------------------------------------- # Phase 4: Cluster Workloads # --------------------------------------------------------------------------- phase_workload() { step "Phase 4: Cluster Workloads" echo "" local cluster_remote="${VM_NAMES[0]}" if [[ "$DRY_RUN" != true ]] && ! incus info "${cluster_remote}:" &>/dev/null 2>&1; then fail "Cannot connect to cluster remote '${cluster_remote}'" return 1 fi # Check if this node is clustered if [[ "$DRY_RUN" != true ]]; then local is_clustered=false incus cluster list "${cluster_remote}:" &>/dev/null 2>&1 && is_clustered=true if [[ "$is_clustered" != true ]]; then warn "Node '${cluster_remote}' is not clustered" warn "Running workloads as single-node tests" fi fi # Test 1: Container on cluster step "Test: Launch container on cluster" local ct_name="test-cluster-ct-$$" if [[ "$DRY_RUN" == true ]]; then detail "[dry-run] incus launch images:debian/12 ${cluster_remote}:${ct_name}" pass "Cluster container (dry-run)" else if incus launch images:debian/12 "${cluster_remote}:${ct_name}" 2>/dev/null; then sleep 5 local location location=$(incus list "${cluster_remote}:" --format csv -c nL 2>/dev/null | grep "^${ct_name}," | cut -d',' -f2) || location="unknown" pass "Cluster container launched (location: ${location})" else fail "Cluster container launch failed" fi fi echo "" # Test 2: VM on specific node (if we have a second node) if [[ ${#VM_NAMES[@]} -ge 2 ]]; then local target_node="${VM_NAMES[1]}" step "Test: Launch VM on specific node (${target_node})" local vm_name="test-cluster-vm-$$" if [[ "$DRY_RUN" == true ]]; then detail "[dry-run] incus launch images:debian/12 ${cluster_remote}:${vm_name} --vm --target ${target_node}" pass "Targeted VM (dry-run)" else if incus launch images:debian/12 "${cluster_remote}:${vm_name}" --vm --target "${target_node}" 2>/dev/null; then detail "Waiting for VM agent..." local elapsed=0 local vm_ready=false while [[ $elapsed -lt 120 ]]; do if incus exec "${cluster_remote}:${vm_name}" -- true &>/dev/null 2>&1; then vm_ready=true break fi sleep 10 elapsed=$((elapsed + 10)) done local location location=$(incus list "${cluster_remote}:" --format csv -c nL 2>/dev/null | grep "^${vm_name}," | cut -d',' -f2) || location="unknown" if [[ "$vm_ready" == true ]]; then pass "Targeted VM running on ${location}" else warn "VM launched on ${location} but agent not ready" pass "Targeted VM launched (agent timeout)" fi else fail "Targeted VM launch failed" fi fi echo "" fi # Show cluster instances if [[ "$DRY_RUN" != true ]]; then step "Current instances on cluster" incus list "${cluster_remote}:" 2>/dev/null || true echo "" fi # Cleanup test instances step "Cleaning up test instances" if [[ "$DRY_RUN" == true ]]; then detail "[dry-run] Would delete test instances" else incus delete "${cluster_remote}:${ct_name}" --force 2>/dev/null || true local vm_name="test-cluster-vm-$$" incus delete "${cluster_remote}:${vm_name}" --force 2>/dev/null || true detail "Test instances cleaned up" fi echo "" } # --------------------------------------------------------------------------- # Phase 5: Migration & Evacuation # --------------------------------------------------------------------------- phase_migrate() { step "Phase 5: Migration & Evacuation" echo "" local cluster_remote="${VM_NAMES[0]}" if [[ ${#VM_NAMES[@]} -lt 2 ]]; then skip "Migration tests require at least 2 nodes" echo "" return 0 fi if [[ "$DRY_RUN" != true ]] && ! incus info "${cluster_remote}:" &>/dev/null 2>&1; then fail "Cannot connect to cluster remote '${cluster_remote}'" return 1 fi # Check if clustered if [[ "$DRY_RUN" != true ]]; then if ! incus cluster list "${cluster_remote}:" &>/dev/null 2>&1; then skip "Node is not clustered, skipping migration tests" echo "" return 0 fi fi local target_node="${VM_NAMES[1]}" local ct_name="test-migrate-ct-$$" # Test 1: Container migration (stop/move) step "Test: Container migration" if [[ "$DRY_RUN" == true ]]; then detail "[dry-run] incus launch images:debian/12 ${cluster_remote}:${ct_name}" detail "[dry-run] incus stop ${cluster_remote}:${ct_name}" detail "[dry-run] incus move ${cluster_remote}:${ct_name} --target ${target_node}" detail "[dry-run] incus start ${cluster_remote}:${ct_name}" pass "Container migration (dry-run)" else if incus launch images:debian/12 "${cluster_remote}:${ct_name}" 2>/dev/null; then sleep 5 local orig_location orig_location=$(incus list "${cluster_remote}:" --format csv -c nL 2>/dev/null | grep "^${ct_name}," | cut -d',' -f2) || orig_location="unknown" detail "Container on: ${orig_location}" # Stop and move incus stop "${cluster_remote}:${ct_name}" 2>/dev/null || true sleep 2 if incus move "${cluster_remote}:${ct_name}" --target "${target_node}" 2>/dev/null; then incus start "${cluster_remote}:${ct_name}" 2>/dev/null || true sleep 3 local new_location new_location=$(incus list "${cluster_remote}:" --format csv -c nL 2>/dev/null | grep "^${ct_name}," | cut -d',' -f2) || new_location="unknown" if [[ "$new_location" == "$target_node" ]]; then pass "Container migrated: ${orig_location} -> ${new_location}" else warn "Container location: ${new_location} (expected: ${target_node})" pass "Container move completed (location may differ due to naming)" fi else fail "Container move failed" fi # Cleanup incus delete "${cluster_remote}:${ct_name}" --force 2>/dev/null || true else fail "Container launch failed for migration test" fi fi echo "" # Test 2: Node evacuation (requires 3+ nodes) if [[ ${#VM_NAMES[@]} -ge 3 ]]; then local evacuate_node="${VM_NAMES[2]}" step "Test: Node evacuation (${evacuate_node})" local evac_ct="test-evac-ct-$$" if [[ "$DRY_RUN" == true ]]; then detail "[dry-run] incus launch images:debian/12 ${cluster_remote}:${evac_ct} --target ${evacuate_node}" detail "[dry-run] incus cluster evacuate ${evacuate_node} ${cluster_remote}:" detail "[dry-run] incus cluster restore ${evacuate_node} ${cluster_remote}:" pass "Node evacuation (dry-run)" else # Place a container on the node we'll evacuate if incus launch images:debian/12 "${cluster_remote}:${evac_ct}" --target "${evacuate_node}" 2>/dev/null; then sleep 5 detail "Container placed on ${evacuate_node}" # Evacuate if incus cluster evacuate "${evacuate_node}" "${cluster_remote}:" --force 2>/dev/null; then sleep 5 # Check that nothing is on the evacuated node local remaining remaining=$(incus list "${cluster_remote}:" --format csv -c nL 2>/dev/null | grep ",${evacuate_node}$" | wc -l) || remaining="0" if [[ "$remaining" -eq 0 ]]; then pass "Node ${evacuate_node} evacuated (0 instances remaining)" else warn "${remaining} instance(s) still on ${evacuate_node} after evacuation" fi # Restore if incus cluster restore "${evacuate_node}" "${cluster_remote}:" 2>/dev/null; then pass "Node ${evacuate_node} restored" else warn "Node restore failed (may need manual intervention)" fi else fail "Evacuation of ${evacuate_node} failed" fi # Cleanup incus delete "${cluster_remote}:${evac_ct}" --force 2>/dev/null || true else fail "Could not place container on ${evacuate_node} for evacuation test" fi fi echo "" else skip "Evacuation test requires 3+ nodes (have ${#VM_NAMES[@]})" echo "" fi } # --------------------------------------------------------------------------- # Cleanup mode: remove test artifacts # --------------------------------------------------------------------------- do_cleanup() { step "Cleaning up test artifacts" echo "" local cluster_remote="${VM_NAMES[0]}" if [[ "$DRY_RUN" != true ]] && ! incus info "${cluster_remote}:" &>/dev/null 2>&1; then warn "Cannot connect to '${cluster_remote}', skipping instance cleanup" return fi # Delete any test instances if [[ "$DRY_RUN" != true ]]; then local instances instances=$(incus list "${cluster_remote}:" --format csv -c n 2>/dev/null | grep "^test-" || true) if [[ -n "$instances" ]]; then while IFS= read -r inst; do [[ -z "$inst" ]] && continue detail "Deleting: ${inst}" incus delete "${cluster_remote}:${inst}" --force 2>/dev/null || true done <<< "$instances" success "Test instances cleaned up" else info "No test instances found" fi else detail "[dry-run] Would delete instances matching 'test-*'" fi echo "" } # --------------------------------------------------------------------------- # Summary # --------------------------------------------------------------------------- print_summary() { echo "" echo -e "${BOLD}Test Summary${RESET}" echo -e " ${GREEN}Passed: ${PASSES}${RESET}" if [[ $FAILURES -gt 0 ]]; then echo -e " ${RED}Failed: ${FAILURES}${RESET}" else echo -e " Failed: 0" fi if [[ $SKIPS -gt 0 ]]; then echo -e " ${YELLOW}Skipped: ${SKIPS}${RESET}" fi echo "" if [[ $FAILURES -gt 0 ]]; then echo -e " ${RED}${BOLD}Some tests failed${RESET}" else echo -e " ${GREEN}${BOLD}All tests passed${RESET}" fi echo "" } # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- main() { setup_colors parse_args "$@" validate_args parse_config if [[ "$QUIET" != true ]]; then echo -e "${BOLD}${SCRIPT_NAME}${RESET} v${VERSION}" echo "" fi # Handle cleanup mode if [[ "$CLEANUP_MODE" == true ]]; then do_cleanup return fi # Run phases case "$PHASE" in deploy) phase_deploy ;; single) if [[ "$SKIP_DEPLOY" != true ]]; then phase_deploy fi phase_single ;; cluster) if [[ "$SKIP_DEPLOY" != true ]]; then phase_deploy fi phase_cluster ;; workload) if [[ "$SKIP_DEPLOY" != true ]]; then phase_deploy fi phase_workload ;; migrate) if [[ "$SKIP_DEPLOY" != true ]]; then phase_deploy fi phase_migrate ;; all) phase_deploy phase_single phase_cluster phase_workload phase_migrate ;; esac print_summary } main "$@"