#!/usr/bin/env bash # incusos-proxmox - Declarative IncusOS deployment on Proxmox VE # # Reads a YAML configuration file defining VMs and deploys them on a Proxmox # host with IncusOS-correct settings (UEFI, TPM 2.0, VirtIO-SCSI, etc.). # Generates per-VM seed images, uploads ISOs, creates VMs, and boots them # through automated installation. # # Usage: incusos-proxmox [OPTIONS] CONFIG_FILE # Run 'incusos-proxmox --help' for full usage information. set -euo pipefail readonly VERSION="0.1.0" readonly SCRIPT_NAME="incusos-proxmox" readonly CDN_INDEX="https://images.linuxcontainers.org/os/index.json" readonly CDN_BASE="https://images.linuxcontainers.org/os" readonly MIN_DISK_GIB=50 readonly MIN_MEMORY_MIB=4096 readonly MANAGED_MARKER="[incusos-lab:managed]" # --------------------------------------------------------------------------- # Color and formatting # --------------------------------------------------------------------------- setup_colors() { if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[0;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' BOLD='\033[1m' DIM='\033[2m' RESET='\033[0m' else RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' RESET='' fi } info() { [[ "$QUIET" == true ]] && return; echo -e "${BLUE}[info]${RESET} $*"; } success() { [[ "$QUIET" == true ]] && return; echo -e "${GREEN}[ok]${RESET} $*"; } warn() { echo -e "${YELLOW}[warn]${RESET} $*" >&2; } error() { echo -e "${RED}[error]${RESET} $*" >&2; } step() { [[ "$QUIET" == true ]] && return; echo -e "${CYAN}[step]${RESET} ${BOLD}$*${RESET}"; } detail() { [[ "$QUIET" == true ]] && return; echo -e "${DIM} $*${RESET}"; } # --------------------------------------------------------------------------- # Help # --------------------------------------------------------------------------- usage() { cat < pvesh get /version ${BOLD}API method:${RESET} Requires an API token. Set up minimal privileges: pveum role add IncusOSDeploy --privs "VM.Allocate VM.Config.Disk \\ VM.Config.CPU VM.Config.Memory VM.Config.Network VM.Config.CDROM \\ VM.Config.Options VM.Config.HWType VM.PowerMgmt VM.Audit \\ Datastore.AllocateSpace Datastore.AllocateTemplate \\ Datastore.Audit SDN.Use" pveum user add automation@pve pveum aclmod / -user automation@pve -role IncusOSDeploy pveum user token add automation@pve deploy --privsep 0 For resource pool isolation (recommended): pveum pool add IncusLab --comment "IncusOS Lab VMs" pveum pool modify IncusLab --storage local-lvm,local pveum aclmod /pool/IncusLab -user automation@pve -role IncusOSDeploy Set the token secret: export PROXMOX_TOKEN_SECRET="" And in your config file: proxmox: method: api api_token_id: automation@pve!deploy ${BOLD}PREREQUISITES${RESET} - python3 with PyYAML (python3-yaml / py3-yaml) - jq (for JSON field access; falls back to python3) - ssh + scp (for SSH method) - curl (for API method and CDN downloads) - incusos-seed sibling script (auto-detected) - Bash 4+ (associative arrays) EOF exit 0 } # --------------------------------------------------------------------------- # Argument parsing # --------------------------------------------------------------------------- # Defaults PHASE="all" METHOD="" HOST_OVERRIDE="" LOCAL_ISO="" CERT_FILE="" NO_CERT=false VM_FILTER="" YES=false DOCTOR=false CLEANUP=false CLEANUP_ALL=false DEEP_CLEANUP=false FORCE_CLEANUP=false DRY_RUN=false QUIET=false CONFIG_FILE="" parse_args() { while [[ $# -gt 0 ]]; do case "$1" in -p|--phase) PHASE="${2:?'--phase requires a value'}" shift 2 ;; -m|--method) METHOD="${2:?'--method requires a value'}" shift 2 ;; -H|--host) HOST_OVERRIDE="${2:?'--host requires a value'}" shift 2 ;; -i|--iso) LOCAL_ISO="${2:?'--iso requires a file path'}" shift 2 ;; -c|--cert) CERT_FILE="${2:?'--cert requires a file path'}" shift 2 ;; --no-cert) NO_CERT=true shift ;; --vm) VM_FILTER="${2:?'--vm requires a name'}" shift 2 ;; -y|--yes) YES=true shift ;; --status) PHASE="status" shift ;; --doctor) DOCTOR=true shift ;; --cleanup) CLEANUP=true shift ;; --deep) DEEP_CLEANUP=true shift ;; --cleanup-all) CLEANUP_ALL=true shift ;; --force-cleanup) CLEANUP=true FORCE_CLEANUP=true shift ;; --dry-run) DRY_RUN=true shift ;; -q|--quiet) QUIET=true shift ;; -h|--help) usage ;; -V|--version) echo "${SCRIPT_NAME} v${VERSION}" exit 0 ;; -*) error "Unknown option: $1" echo "Run '${SCRIPT_NAME} --help' for usage information." >&2 exit 1 ;; *) if [[ -z "$CONFIG_FILE" ]]; then CONFIG_FILE="$1" else error "Unexpected argument: $1 (config file already set to ${CONFIG_FILE})" echo "Run '${SCRIPT_NAME} --help' for usage information." >&2 exit 1 fi shift ;; esac done } # --------------------------------------------------------------------------- # Validation # --------------------------------------------------------------------------- validate_args() { # --doctor can run without a config file if [[ "$DOCTOR" == true ]] && [[ -z "$CONFIG_FILE" ]]; then return 0 fi if [[ -z "$CONFIG_FILE" ]]; then error "No configuration file specified" echo "Usage: ${SCRIPT_NAME} [OPTIONS] CONFIG_FILE" >&2 echo "Run '${SCRIPT_NAME} --help' for usage information." >&2 exit 1 fi if [[ ! -f "$CONFIG_FILE" ]]; then error "Configuration file not found: ${CONFIG_FILE}" exit 1 fi # --deep only makes sense with --cleanup or --cleanup-all if [[ "$DEEP_CLEANUP" == true ]] && [[ "$CLEANUP" != true ]] && [[ "$CLEANUP_ALL" != true ]]; then error "--deep can only be used with --cleanup or --cleanup-all" exit 1 fi case "$PHASE" in validate|prepare|upload|create|install|status|all) ;; *) error "Invalid phase: ${PHASE}" error "Supported: validate, prepare, upload, create, install, status, all" exit 1 ;; esac if [[ -n "$METHOD" ]]; then case "$METHOD" in ssh|api) ;; *) error "Invalid method: ${METHOD}" error "Supported: ssh, api" exit 1 ;; esac fi if [[ -n "$CERT_FILE" ]] && [[ "$NO_CERT" == true ]]; then error "Cannot use both --cert and --no-cert" exit 1 fi if [[ -n "$CERT_FILE" ]] && [[ ! -f "$CERT_FILE" ]]; then error "Certificate file not found: ${CERT_FILE}" exit 1 fi if [[ -n "$LOCAL_ISO" ]] && [[ ! -f "$LOCAL_ISO" ]]; then error "ISO file not found: ${LOCAL_ISO}" exit 1 fi } # --------------------------------------------------------------------------- # Dependency checks # --------------------------------------------------------------------------- SEED_TOOL="" check_dependencies() { step "Checking dependencies" local missing=false # python3 if ! command -v python3 &>/dev/null; then error "python3 is required but not found" error "Install Python 3 for your platform (https://www.python.org/)" missing=true else detail "python3: $(command -v python3)" fi # python3-yaml (optional -- basic YAML fallback built-in) if command -v python3 &>/dev/null; then if python3 -c "import yaml" 2>/dev/null; then detail "PyYAML: available" else detail "PyYAML: not found (using built-in basic parser)" fi fi # jq (optional, python3 fallback) if command -v jq &>/dev/null; then detail "jq: $(command -v jq)" else detail "jq: not found (using python3 fallback)" fi # curl if ! command -v curl &>/dev/null; then error "curl is required but not found" missing=true else detail "curl: $(command -v curl)" fi # ssh/scp (for SSH method) if [[ "${PVE_METHOD:-ssh}" == "ssh" ]]; then if ! command -v ssh &>/dev/null; then error "ssh is required for SSH method but not found" missing=true fi if ! command -v scp &>/dev/null; then error "scp is required for SSH method but not found" missing=true fi fi # incusos-seed sibling script local script_dir script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SEED_TOOL="${script_dir}/incusos-seed" if [[ ! -x "$SEED_TOOL" ]]; then error "incusos-seed not found at: ${SEED_TOOL}" error "The incusos-seed script must be in the same directory as ${SCRIPT_NAME}" missing=true else detail "incusos-seed: ${SEED_TOOL}" fi # bash version if [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then error "Bash 4+ is required (found ${BASH_VERSION})" missing=true else detail "bash: ${BASH_VERSION}" fi if [[ "$missing" == true ]]; then error "Missing dependencies -- cannot continue" exit 1 fi success "All dependencies satisfied" } # --------------------------------------------------------------------------- # Doctor mode # --------------------------------------------------------------------------- run_doctor() { echo -e "${BOLD}${SCRIPT_NAME} doctor${RESET}" echo "" local script_dir script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # -- Tools section -- echo -e " ${BOLD}Tools:${RESET}" # incus client if command -v incus &>/dev/null; then local incus_ver incus_ver=$(incus version 2>/dev/null | grep -oP 'Client version: \K[0-9.]+' || echo "unknown") echo -e " incus client: ${GREEN}${incus_ver} ✓${RESET}" else echo -e " incus client: ${YELLOW}not installed${RESET}" fi # flasher-tool if command -v flasher-tool &>/dev/null; then local ft_ver ft_ver=$(flasher-tool --version 2>/dev/null | head -1 | sed 's/^flasher-tool version //' || echo "installed") if [[ -z "$ft_ver" ]] || [[ "$ft_ver" == *"unknown"* ]]; then ft_ver="dev (installed)" fi echo -e " flasher-tool: ${GREEN}${ft_ver} ✓${RESET}" else echo -e " flasher-tool: ${YELLOW}not installed${RESET}" fi # operations-center CLI if command -v operations-center &>/dev/null; then local oc_ver oc_ver=$(operations-center version 2>/dev/null | head -1 || echo "installed") echo -e " operations-center CLI: ${GREEN}${oc_ver} ✓${RESET}" else echo -e " operations-center CLI: ${DIM}not installed (optional)${RESET}" fi # python3 if command -v python3 &>/dev/null; then local py_ver py_ver=$(python3 --version 2>/dev/null | awk '{print $2}') echo -e " python3: ${GREEN}${py_ver} ✓${RESET}" else echo -e " python3: ${RED}not installed (required)${RESET}" fi # curl if command -v curl &>/dev/null; then local curl_ver curl_ver=$(curl --version 2>/dev/null | head -1 | awk '{print $2}') echo -e " curl: ${GREEN}${curl_ver} ✓${RESET}" else echo -e " curl: ${RED}not installed (required)${RESET}" fi # genisoimage if command -v genisoimage &>/dev/null; then echo -e " genisoimage: ${GREEN}installed ✓${RESET}" else echo -e " genisoimage: ${RED}not installed (required for seed ISOs)${RESET}" fi # jq if command -v jq &>/dev/null; then local jq_ver jq_ver=$(jq --version 2>/dev/null | sed 's/^jq-//') echo -e " jq: ${GREEN}${jq_ver} ✓${RESET}" else echo -e " jq: ${YELLOW}not installed (python3 fallback used)${RESET}" fi # incusos-seed sibling if [[ -x "${script_dir}/incusos-seed" ]]; then echo -e " incusos-seed: ${GREEN}found ✓${RESET}" else echo -e " incusos-seed: ${RED}not found at ${script_dir}/incusos-seed${RESET}" fi # -- IncusOS section -- echo "" echo -e " ${BOLD}IncusOS:${RESET}" local cdn_latest="" local cdn_index cdn_index=$(mktemp) if curl -fsSL --connect-timeout 5 "$CDN_INDEX" -o "$cdn_index" 2>/dev/null; then if command -v jq &>/dev/null; then cdn_latest=$(jq -r '[.updates[] | select(.channels[] == "stable")] | sort_by(.version) | last | .version // empty' "$cdn_index" 2>/dev/null) else cdn_latest=$(python3 -c " import json data = json.load(open('${cdn_index}')) versions = [u['version'] for u in data.get('updates', []) if 'stable' in u.get('channels', [])] if versions: print(sorted(versions)[-1]) " 2>/dev/null) fi fi rm -f "$cdn_index" if [[ -n "$cdn_latest" ]]; then echo -e " Latest stable: ${cdn_latest}" else echo -e " Latest stable: ${YELLOW}could not reach CDN${RESET}" fi # Check local cache local cache_base="${XDG_CACHE_HOME:-${HOME}/.cache}/${SCRIPT_NAME}/iso-cache" if [[ -d "$cache_base" ]]; then local cached_iso cached_iso=$(ls -1 "${cache_base}"/IncusOS_*.iso 2>/dev/null | sort | tail -1) if [[ -n "$cached_iso" ]]; then local cached_name cached_name=$(basename "$cached_iso" .iso | sed 's/^IncusOS_//') if [[ -n "$cdn_latest" ]] && [[ "$cached_name" == "$cdn_latest" ]]; then echo -e " Cached ISO: ${GREEN}${cached_name} (up to date)${RESET}" else echo -e " Cached ISO: ${YELLOW}${cached_name}${RESET}" fi else echo -e " Cached ISO: ${DIM}none${RESET}" fi else echo -e " Cached ISO: ${DIM}none${RESET}" fi # -- Proxmox section (only if config file provided) -- echo "" if [[ -n "$CONFIG_FILE" ]] && [[ -f "$CONFIG_FILE" ]]; then echo -e " ${BOLD}Proxmox:${RESET}" # Minimal config parsing for doctor mode local doctor_workdir doctor_workdir=$(mktemp -d "${TMPDIR:-/tmp}/${SCRIPT_NAME}-doctor-XXXXXX") WORKDIR="$doctor_workdir" CONFIG_JSON="${doctor_workdir}/config.json" if python3 -c " import json, sys try: import yaml with open(sys.argv[1]) as f: data = yaml.safe_load(f) except ImportError: # Minimal inline parse -- just need proxmox section data = {} with open(sys.argv[1]) as f: import re lines = f.readlines() processed = [] for raw in lines: s = raw.rstrip() if not s or s.lstrip().startswith('#'): continue indent = len(s) - len(s.lstrip()) content = s.lstrip() processed.append((indent, content)) result = {} section = None for indent, content in processed: m = re.match(r'^([^:]+?):\s*(.*?)$', content) if m: key = m.group(1).strip() val = m.group(2).strip() if not val: section = key result[section] = {} elif section: if val.startswith('\"') or val.startswith(\"'\"): val = val[1:-1] result[section][key] = val data = result json.dump(data, sys.stdout, indent=2) " "$CONFIG_FILE" > "$CONFIG_JSON" 2>/dev/null; then local doc_host doc_method doc_token_id doc_pool doc_host=$(json_get "$CONFIG_JSON" ".proxmox.host" "") doc_method=$(json_get "$CONFIG_JSON" ".proxmox.method" "ssh") doc_token_id=$(json_get "$CONFIG_JSON" ".proxmox.api_token_id" "") doc_pool=$(json_get "$CONFIG_JSON" ".proxmox.pool" "") if [[ -z "$doc_host" ]]; then echo -e " Host: ${RED}not set in config${RESET}" else echo -e " Host: ${doc_host} (${doc_method})" # Connectivity check local pve_ok=false if [[ "$doc_method" == "api" ]]; then if [[ -n "$doc_token_id" ]] && [[ -n "${PROXMOX_TOKEN_SECRET:-}" ]]; then local auth="Authorization: PVEAPIToken=${doc_token_id}=${PROXMOX_TOKEN_SECRET}" local pve_ver pve_ver=$(curl -fsSk --connect-timeout 5 -H "$auth" \ "https://${doc_host}:8006/api2/json/version" 2>/dev/null | \ python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('version',''))" 2>/dev/null) || pve_ver="" if [[ -n "$pve_ver" ]]; then echo -e " Connectivity: ${GREEN}connected (PVE ${pve_ver}) ✓${RESET}" pve_ok=true else echo -e " Connectivity: ${RED}API connection failed${RESET}" fi else echo -e " Connectivity: ${YELLOW}API token not configured or PROXMOX_TOKEN_SECRET not set${RESET}" fi else local ssh_user ssh_user=$(json_get "$CONFIG_JSON" ".proxmox.ssh_user" "root") if ssh -o BatchMode=yes -o ConnectTimeout=5 "${ssh_user}@${doc_host}" "true" &>/dev/null; then echo -e " Connectivity: ${GREEN}SSH connected ✓${RESET}" pve_ok=true else echo -e " Connectivity: ${RED}SSH connection failed${RESET}" fi fi # Pool check (API only, if connected) if [[ -n "$doc_pool" ]]; then if [[ "$pve_ok" == true ]] && [[ "$doc_method" == "api" ]]; then local auth="Authorization: PVEAPIToken=${doc_token_id}=${PROXMOX_TOKEN_SECRET}" if curl -fsSk --connect-timeout 5 -H "$auth" \ "https://${doc_host}:8006/api2/json/pools/${doc_pool}" &>/dev/null; then echo -e " Pool: ${GREEN}${doc_pool} exists ✓${RESET}" else echo -e " Pool: ${RED}${doc_pool} not found${RESET}" fi else echo -e " Pool: ${doc_pool} (not verified)" fi fi fi else echo -e " ${RED}Failed to parse config file${RESET}" fi rm -rf "$doctor_workdir" else echo -e " ${BOLD}Proxmox:${RESET} ${DIM}(requires config file, skipped)${RESET}" fi echo "" } # --------------------------------------------------------------------------- # Temporary directory management # --------------------------------------------------------------------------- WORKDIR="" CACHEDIR="" setup_workdir() { # Use persistent storage for the work directory -- the ISO download + # decompression can exceed 4 GiB, which overflows tmpfs on most systems. # Honour TMPDIR if explicitly set; otherwise use ~/.cache on real disk. local base="${TMPDIR:-${XDG_CACHE_HOME:-${HOME}/.cache}/${SCRIPT_NAME}}" mkdir -p "$base" # Persistent cache for ISO downloads (survives across runs) CACHEDIR="${base}/iso-cache" mkdir -p "$CACHEDIR" WORKDIR=$(mktemp -d "${base}/${SCRIPT_NAME}-XXXXXX") trap 'rm -rf "$WORKDIR"' EXIT } # --------------------------------------------------------------------------- # YAML parsing via python3 # --------------------------------------------------------------------------- # Convert YAML config to JSON, stored in a file for repeated access CONFIG_JSON="" parse_config() { step "Parsing configuration: ${CONFIG_FILE}" CONFIG_JSON="${WORKDIR}/config.json" # Try PyYAML first, fall back to a basic pure-Python parser if ! python3 -c " import json, sys, re def _parse_value(v): v = v.strip() # Strip inline comments (but not inside quotes) if not v.startswith(('\"', \"'\")): m = re.match(r'^(.*?)\s+#.*$', v) if m: v = m.group(1).rstrip() if not v: return None if (v.startswith('\"') and v.endswith('\"')) or (v.startswith(\"'\") and v.endswith(\"'\")): return v[1:-1] if v == 'true': return True if v == 'false': return False if v == 'null' or v == '~': return None try: return int(v) except ValueError: pass try: return float(v) except ValueError: pass return v def parse_yaml_basic(path): \"\"\"Minimal YAML parser for simple config files. Handles: mappings, lists of mappings, lists of scalars, inline comments. Does NOT handle: anchors, multi-line strings, flow style, merge keys.\"\"\" with open(path) as f: lines = f.readlines() # Pre-process: strip comments and blank lines, track indent + content processed = [] for raw in lines: s = raw.rstrip() if not s or s.lstrip().startswith('#'): continue indent = len(s) - len(s.lstrip()) content = s.lstrip() processed.append((indent, content)) def parse_block(idx, base_indent): \"\"\"Parse a block at the given indentation level, return (result, next_idx).\"\"\" if idx >= len(processed): return None, idx indent, content = processed[idx] # Detect if this block is a list or mapping if content.startswith('- '): return parse_list(idx, indent) else: return parse_mapping(idx, indent) def parse_mapping(idx, base_indent): result = {} while idx < len(processed): indent, content = processed[idx] if indent < base_indent: break if indent > base_indent: break if content.startswith('- '): break # Must be key: value m = re.match(r'^([^:]+?):\s*(.*?)$', content) if not m: idx += 1 continue key = m.group(1).strip() val_str = m.group(2).strip() # Strip inline comment from value if val_str and not val_str.startswith(('\"', \"'\")): cm = re.match(r'^(.*?)\s+#.*$', val_str) if cm: val_str = cm.group(1).rstrip() if val_str and not val_str.startswith('#'): result[key] = _parse_value(val_str) idx += 1 else: # Value is a nested block idx += 1 if idx < len(processed) and processed[idx][0] > base_indent: child_indent = processed[idx][0] child, idx = parse_block(idx, child_indent) result[key] = child if child is not None else {} else: result[key] = {} return result, idx def parse_list(idx, base_indent): result = [] while idx < len(processed): indent, content = processed[idx] if indent < base_indent: break if indent > base_indent: break if not content.startswith('- '): break item_str = content[2:].strip() # Check if it's a mapping item: '- key: value' m = re.match(r'^([^:]+?):\s*(.*?)$', item_str) if m: # It's a mapping -- parse this item and continuation lines obj = {} key = m.group(1).strip() val_str = m.group(2).strip() if val_str and not val_str.startswith('#'): if not val_str.startswith(('\"', \"'\")): cm = re.match(r'^(.*?)\s+#.*$', val_str) if cm: val_str = cm.group(1).rstrip() obj[key] = _parse_value(val_str) else: idx += 1 if idx < len(processed) and processed[idx][0] > indent: child_indent = processed[idx][0] child, idx = parse_block(idx, child_indent) obj[key] = child if child is not None else {} else: obj[key] = {} # Parse remaining keys at item indent + 2 item_indent = indent + 2 while idx < len(processed) and processed[idx][0] == item_indent: ic = processed[idx][1] if ic.startswith('- '): break im = re.match(r'^([^:]+?):\s*(.*?)$', ic) if im: ik = im.group(1).strip() iv = im.group(2).strip() if iv and not iv.startswith('#'): if not iv.startswith(('\"', \"'\")): icm = re.match(r'^(.*?)\s+#.*$', iv) if icm: iv = icm.group(1).rstrip() obj[ik] = _parse_value(iv) idx += 1 else: idx += 1 if idx < len(processed) and processed[idx][0] > item_indent: child_indent = processed[idx][0] child, idx = parse_block(idx, child_indent) obj[ik] = child if child is not None else {} else: obj[ik] = {} else: idx += 1 result.append(obj) continue idx += 1 # Parse remaining keys of this list item item_indent = indent + 2 while idx < len(processed) and processed[idx][0] >= item_indent: if processed[idx][0] == item_indent: ic = processed[idx][1] if ic.startswith('- '): break im = re.match(r'^([^:]+?):\s*(.*?)$', ic) if im: ik = im.group(1).strip() iv = im.group(2).strip() if iv and not iv.startswith('#'): if not iv.startswith(('\"', \"'\")): icm = re.match(r'^(.*?)\s+#.*$', iv) if icm: iv = icm.group(1).rstrip() obj[ik] = _parse_value(iv) idx += 1 else: idx += 1 if idx < len(processed) and processed[idx][0] > item_indent: child_indent = processed[idx][0] child, idx = parse_block(idx, child_indent) obj[ik] = child if child is not None else {} else: obj[ik] = {} else: idx += 1 else: # Deeper nesting handled by recursive calls above break result.append(obj) else: # Scalar list item result.append(_parse_value(item_str)) idx += 1 return result, idx result, _ = parse_block(0, 0) return result if result is not None else {} try: import yaml with open(sys.argv[1]) as f: data = yaml.safe_load(f) except ImportError: data = parse_yaml_basic(sys.argv[1]) json.dump(data, sys.stdout, indent=2) " "$CONFIG_FILE" > "$CONFIG_JSON" 2>/dev/null; then error "Failed to parse YAML configuration: ${CONFIG_FILE}" error "Check that the file is valid YAML" error "For complex YAML features, install PyYAML: pip3 install pyyaml" exit 1 fi success "Configuration parsed" } # JSON field accessor -- uses jq if available, falls back to python3 json_get() { local file="$1" local query="$2" local default="${3:-}" local result if command -v jq &>/dev/null; then result=$(jq -r "$query // empty" "$file" 2>/dev/null) || result="" else result=$(python3 -c " import json, sys data = json.load(open(sys.argv[1])) path = sys.argv[2] # Simple jq-like path traversal for common patterns if path.startswith('.'): path = path[1:] parts = path.replace('[]','').split('.') obj = data for p in parts: if p == '': continue if isinstance(obj, dict): obj = obj.get(p) elif isinstance(obj, list) and p.isdigit(): obj = obj[int(p)] else: obj = None if obj is None: break if obj is None: print('') elif isinstance(obj, bool): print('true' if obj else 'false') elif isinstance(obj, (dict, list)): json.dump(obj, sys.stdout) else: print(obj) " "$file" "$query" 2>/dev/null) || result="" fi if [[ -z "$result" ]] && [[ -n "$default" ]]; then echo "$default" else echo "$result" fi } # Get the number of VMs in config vm_count() { if command -v jq &>/dev/null; then jq -r '.vms | length' "$CONFIG_JSON" 2>/dev/null || echo "0" else python3 -c " import json data = json.load(open('$CONFIG_JSON')) print(len(data.get('vms', []))) " 2>/dev/null || echo "0" fi } # Get a field from a specific VM by index vm_get() { local index="$1" local field="$2" local default="${3:-}" local result if command -v jq &>/dev/null; then result=$(jq -r ".vms[${index}].${field} // empty" "$CONFIG_JSON" 2>/dev/null) || result="" else result=$(python3 -c " import json data = json.load(open('$CONFIG_JSON')) vm = data.get('vms', [])[${index}] val = vm.get('${field}') if val is None: print('') elif isinstance(val, bool): print('true' if val else 'false') else: print(val) " 2>/dev/null) || result="" fi if [[ -z "$result" ]] && [[ -n "$default" ]]; then echo "$default" else echo "$result" fi } # --------------------------------------------------------------------------- # Configuration loading # --------------------------------------------------------------------------- # Proxmox connection settings PVE_HOST="" PVE_NODE="" PVE_STORAGE="" PVE_ISO_STORAGE="" PVE_BRIDGE="" PVE_METHOD="" PVE_SSH_USER="" PVE_API_TOKEN_ID="" PVE_POOL="" # Default VM settings DEF_CORES="" DEF_MEMORY="" DEF_DISK="" DEF_CHANNEL="" DEF_START_VMID="" # VM arrays (indexed by position) declare -a VM_NAMES=() declare -a VM_APPS=() declare -a VM_APPLY_DEFAULTS=() declare -a VM_CORES=() declare -a VM_MEMORY=() declare -a VM_DISK=() declare -a VM_VMIDS=() declare -a VM_CLUSTERS=() declare -a VM_CLUSTER_ROLES=() load_config() { step "Loading configuration values" # Proxmox settings PVE_HOST=$(json_get "$CONFIG_JSON" ".proxmox.host" "") PVE_NODE=$(json_get "$CONFIG_JSON" ".proxmox.node" "pve") PVE_STORAGE=$(json_get "$CONFIG_JSON" ".proxmox.storage" "local-lvm") PVE_ISO_STORAGE=$(json_get "$CONFIG_JSON" ".proxmox.iso_storage" "local") PVE_BRIDGE=$(json_get "$CONFIG_JSON" ".proxmox.bridge" "vmbr0") PVE_SSH_USER=$(json_get "$CONFIG_JSON" ".proxmox.ssh_user" "root") PVE_API_TOKEN_ID=$(json_get "$CONFIG_JSON" ".proxmox.api_token_id" "") PVE_POOL=$(json_get "$CONFIG_JSON" ".proxmox.pool" "") # Method: CLI override > config > default if [[ -z "$PVE_METHOD" ]]; then PVE_METHOD=$(json_get "$CONFIG_JSON" ".proxmox.method" "ssh") fi # CLI override for method if [[ -n "$METHOD" ]]; then PVE_METHOD="$METHOD" fi # Host override if [[ -n "$HOST_OVERRIDE" ]]; then PVE_HOST="$HOST_OVERRIDE" fi # Default VM settings DEF_CORES=$(json_get "$CONFIG_JSON" ".defaults.cores" "4") DEF_MEMORY=$(json_get "$CONFIG_JSON" ".defaults.memory" "8192") DEF_DISK=$(json_get "$CONFIG_JSON" ".defaults.disk" "64") DEF_CHANNEL=$(json_get "$CONFIG_JSON" ".defaults.channel" "stable") DEF_START_VMID=$(json_get "$CONFIG_JSON" ".defaults.start_vmid" "200") # Load VMs local count count=$(vm_count) local i=0 while [[ $i -lt $count ]]; do VM_NAMES+=("$(vm_get "$i" "name" "")") VM_APPS+=("$(vm_get "$i" "app" "incus")") VM_APPLY_DEFAULTS+=("$(vm_get "$i" "apply_defaults" "false")") VM_CORES+=("$(vm_get "$i" "cores" "$DEF_CORES")") VM_MEMORY+=("$(vm_get "$i" "memory" "$DEF_MEMORY")") VM_DISK+=("$(vm_get "$i" "disk" "$DEF_DISK")") VM_VMIDS+=("$(vm_get "$i" "vmid" "")") VM_CLUSTERS+=("$(vm_get "$i" "cluster" "")") VM_CLUSTER_ROLES+=("$(vm_get "$i" "cluster_role" "")") i=$((i + 1)) done success "Loaded ${count} VM definition(s)" } # --------------------------------------------------------------------------- # Configuration validation # --------------------------------------------------------------------------- validate_config() { step "Validating configuration" local errors=0 # Required: Proxmox host if [[ -z "$PVE_HOST" ]]; then error "proxmox.host is required" errors=$((errors + 1)) fi # Method validation case "$PVE_METHOD" in ssh|api) ;; *) error "Invalid proxmox.method: ${PVE_METHOD} (must be ssh or api)" errors=$((errors + 1)) ;; esac # API method requires token if [[ "$PVE_METHOD" == "api" ]]; then if [[ -z "$PVE_API_TOKEN_ID" ]]; then error "proxmox.api_token_id is required for API method" error "Example: automation@pve!deploy" errors=$((errors + 1)) fi if [[ -z "${PROXMOX_TOKEN_SECRET:-}" ]]; then error "PROXMOX_TOKEN_SECRET environment variable is required for API method" error "Set it with: export PROXMOX_TOKEN_SECRET=\"\"" errors=$((errors + 1)) fi fi # Must have at least one VM local count=${#VM_NAMES[@]} if [[ $count -eq 0 ]]; then error "No VMs defined in configuration" errors=$((errors + 1)) fi # Validate each VM local seen_names="" local cluster_inits="" local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" local app="${VM_APPS[$i]}" local memory="${VM_MEMORY[$i]}" local disk="${VM_DISK[$i]}" local cluster="${VM_CLUSTERS[$i]}" local role="${VM_CLUSTER_ROLES[$i]}" # Name required if [[ -z "$name" ]]; then error "VM #$((i + 1)): name is required" errors=$((errors + 1)) i=$((i + 1)) continue fi # Unique names if echo "$seen_names" | grep -qw "$name"; then error "VM '${name}': duplicate VM name" errors=$((errors + 1)) fi seen_names="${seen_names} ${name}" # Valid app case "$app" in incus|operations-center) ;; *) error "VM '${name}': invalid app '${app}' (must be incus or operations-center)" errors=$((errors + 1)) ;; esac # Minimum memory if [[ "$memory" -lt "$MIN_MEMORY_MIB" ]] 2>/dev/null; then error "VM '${name}': memory ${memory} MiB is below minimum ${MIN_MEMORY_MIB} MiB" errors=$((errors + 1)) fi # Minimum disk if [[ "$disk" -lt "$MIN_DISK_GIB" ]] 2>/dev/null; then error "VM '${name}': disk ${disk} GiB is below minimum ${MIN_DISK_GIB} GiB" errors=$((errors + 1)) fi # Cluster role validation if [[ -n "$role" ]] && [[ -z "$cluster" ]]; then error "VM '${name}': cluster_role '${role}' set but no cluster name specified" errors=$((errors + 1)) fi if [[ -n "$role" ]]; then case "$role" in init|join) ;; *) error "VM '${name}': invalid cluster_role '${role}' (must be init or join)" errors=$((errors + 1)) ;; esac fi # Track cluster init nodes if [[ -n "$cluster" ]] && [[ "$role" == "init" ]]; then if echo "$cluster_inits" | grep -qw "$cluster"; then error "Cluster '${cluster}': multiple init nodes defined (only one allowed)" errors=$((errors + 1)) fi cluster_inits="${cluster_inits} ${cluster}" fi i=$((i + 1)) done # VM filter validation if [[ -n "$VM_FILTER" ]]; then local found=false for name in "${VM_NAMES[@]}"; do if [[ "$name" == "$VM_FILTER" ]]; then found=true break fi done if [[ "$found" != true ]]; then error "VM '${VM_FILTER}' not found in configuration" error "Available VMs: ${VM_NAMES[*]}" errors=$((errors + 1)) fi fi if [[ $errors -gt 0 ]]; then error "${errors} validation error(s) found" exit 1 fi success "Configuration valid" } # --------------------------------------------------------------------------- # Proxmox transport layer # --------------------------------------------------------------------------- # Execute a command on the Proxmox host pve_run() { local cmd="$1" if [[ "$DRY_RUN" == true ]]; then detail "[dry-run] ${PVE_METHOD}: ${cmd}" return 0 fi case "$PVE_METHOD" in ssh) ssh -o BatchMode=yes -o ConnectTimeout=10 \ "${PVE_SSH_USER}@${PVE_HOST}" "$cmd" ;; api) error "API execution of arbitrary commands is not supported" error "Use specific API functions instead" return 1 ;; esac } # Execute a Proxmox API call (API method) # Usage: pve_api METHOD ENDPOINT [DATA] [json] # Pass "json" as 4th arg to send data as application/json instead of form-encoded. pve_api() { local method="$1" local endpoint="$2" local data="${3:-}" local content_type="${4:-form}" local url="https://${PVE_HOST}:8006/api2/json${endpoint}" local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET:-}" if [[ "$DRY_RUN" == true ]]; then detail "[dry-run] API ${method} ${endpoint}" if [[ -n "$data" ]]; then detail "[dry-run] data: ${data}" fi return 0 fi local curl_args=(-fsSk -H "$auth") if [[ "$content_type" == "json" ]]; then curl_args+=(-H "Content-Type: application/json") fi case "$method" in GET) curl "${curl_args[@]}" "$url" ;; POST) if [[ -n "$data" ]]; then curl "${curl_args[@]}" -X POST -d "$data" "$url" else curl "${curl_args[@]}" -X POST "$url" fi ;; PUT) if [[ -n "$data" ]]; then curl "${curl_args[@]}" -X PUT -d "$data" "$url" else curl "${curl_args[@]}" -X PUT "$url" fi ;; DELETE) curl "${curl_args[@]}" -X DELETE "$url" ;; esac } # Upload a file to Proxmox ISO storage pve_upload() { local local_path="$1" local remote_filename="$2" if [[ "$DRY_RUN" == true ]]; then detail "[dry-run] upload: ${local_path} -> ${PVE_ISO_STORAGE}:iso/${remote_filename}" return 0 fi case "$PVE_METHOD" in ssh) local iso_dir # Query the actual path of the ISO storage iso_dir=$(pve_run "pvesm path ${PVE_ISO_STORAGE}:iso/test 2>/dev/null" | sed 's|/test$||') || true if [[ -z "$iso_dir" ]]; then # Common default paths iso_dir="/var/lib/vz/template/iso" fi scp -o BatchMode=yes -o ConnectTimeout=10 \ "$local_path" "${PVE_SSH_USER}@${PVE_HOST}:${iso_dir}/${remote_filename}" ;; api) local upload_response upload_rc upload_response=$(curl -sSk \ -H "Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET:-}" \ -X POST \ -F "content=iso" \ -F "filename=@${local_path};filename=${remote_filename}" \ "https://${PVE_HOST}:8006/api2/json/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/upload" \ -w "\n%{http_code}" 2>&1) || true local http_code http_code=$(echo "$upload_response" | tail -1) local body body=$(echo "$upload_response" | sed '$d') if [[ "$http_code" -ge 200 ]] && [[ "$http_code" -lt 300 ]] 2>/dev/null; then return 0 else error "Upload failed (HTTP ${http_code})" # Extract error message from JSON response local api_error api_error=$(echo "$body" | python3 -c " import json, sys try: data = json.load(sys.stdin) errors = data.get('errors', {}) if errors: for k, v in errors.items(): print(f' {k}: {v}') elif data.get('message'): print(f' {data[\"message\"]}') else: print(json.dumps(data, indent=2)) except: print(sys.stdin.read() if hasattr(sys.stdin, 'read') else '') " 2>/dev/null) || api_error="$body" if [[ -n "$api_error" ]]; then error "Proxmox API response:" echo "$api_error" | while IFS= read -r line; do error " ${line}" done fi return 1 fi ;; esac } # Check if an ISO exists on Proxmox storage pve_iso_exists() { local filename="$1" if [[ "$DRY_RUN" == true ]]; then return 1 fi case "$PVE_METHOD" in ssh) pve_run "pvesm list ${PVE_ISO_STORAGE} --content iso 2>/dev/null" | grep -q "$filename" ;; api) pve_api GET "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content?content=iso" | \ grep -q "$filename" ;; esac } # Get list of existing VMIDs pve_list_vmids() { if [[ "$DRY_RUN" == true ]]; then echo "" return 0 fi case "$PVE_METHOD" in ssh) pve_run "qm list 2>/dev/null" | awk 'NR>1 {print $1}' || true ;; api) pve_api GET "/nodes/${PVE_NODE}/qemu" 2>/dev/null | \ python3 -c " import json, sys data = json.load(sys.stdin) for vm in data.get('data', []): print(vm.get('vmid', '')) " 2>/dev/null || true ;; esac } # Create a VM via qm create pve_create_vm() { local vmid="$1" local name="$2" local cores="$3" local memory="$4" local disk_gib="$5" local iso_filename="$6" local seed_filename="$7" local description description=$(build_vm_description) local cmd="qm create ${vmid}" cmd+=" --name ${name}" cmd+=" --description '${description}'" cmd+=" --ostype l26" cmd+=" --bios ovmf" cmd+=" --machine q35" cmd+=" --efidisk0 ${PVE_STORAGE}:0,efitype=4m,pre-enrolled-keys=0" cmd+=" --tpmstate0 ${PVE_STORAGE}:1,version=v2.0" cmd+=" --cpu host" cmd+=" --cores ${cores}" cmd+=" --memory ${memory}" cmd+=" --balloon 0" cmd+=" --scsihw virtio-scsi-pci" cmd+=" --scsi0 ${PVE_STORAGE}:${disk_gib},iothread=1" cmd+=" --ide2 ${PVE_ISO_STORAGE}:iso/${iso_filename},media=cdrom" cmd+=" --ide3 ${PVE_ISO_STORAGE}:iso/${seed_filename},media=cdrom" cmd+=" --net0 virtio,bridge=${PVE_BRIDGE}" cmd+=" --boot order=ide2\\;scsi0" cmd+=" --agent 1" if [[ -n "$PVE_POOL" ]]; then cmd+=" --pool ${PVE_POOL}" fi if [[ "$DRY_RUN" == true ]]; then detail "[dry-run] ${cmd}" return 0 fi case "$PVE_METHOD" in ssh) pve_run "$cmd" ;; api) # Build JSON payload -- the Proxmox API requires JSON for VM # creation because comma-separated disk/net specs are misinterpreted # when sent as form-encoded data. local json_payload json_payload=$(python3 -c " import json payload = { 'vmid': ${vmid}, 'name': '${name}', 'description': '''${description}''', 'ostype': 'l26', 'bios': 'ovmf', 'machine': 'q35', 'efidisk0': '${PVE_STORAGE}:0,efitype=4m,pre-enrolled-keys=0', 'tpmstate0': '${PVE_STORAGE}:1,version=v2.0', 'cpu': 'host', 'cores': ${cores}, 'memory': ${memory}, 'balloon': 0, 'scsihw': 'virtio-scsi-pci', 'scsi0': '${PVE_STORAGE}:${disk_gib},iothread=1', 'ide2': '${PVE_ISO_STORAGE}:iso/${iso_filename},media=cdrom', 'ide3': '${PVE_ISO_STORAGE}:iso/${seed_filename},media=cdrom', 'net0': 'virtio,bridge=${PVE_BRIDGE}', 'boot': 'order=ide2;scsi0', 'agent': 1, } pool = '${PVE_POOL}' if pool: payload['pool'] = pool print(json.dumps(payload)) ") local create_response create_response=$(pve_api POST "/nodes/${PVE_NODE}/qemu" "$json_payload" json) # VM creation is async -- the API returns a UPID. Wait for the # task to finish before returning, otherwise the caller may try # to start or configure the VM while it's still being created. local upid upid=$(echo "$create_response" | python3 -c "import json,sys; print(json.load(sys.stdin).get('data',''))" 2>/dev/null) || upid="" if [[ -n "$upid" ]]; then local encoded_upid encoded_upid=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${upid}', safe=''))" 2>/dev/null) local wait_elapsed=0 while [[ $wait_elapsed -lt 60 ]]; do sleep 2 wait_elapsed=$((wait_elapsed + 2)) local task_status task_status=$(pve_api GET "/nodes/${PVE_NODE}/tasks/${encoded_upid}/status" 2>/dev/null | \ python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status',''))" 2>/dev/null) || task_status="" if [[ "$task_status" == "stopped" ]]; then break fi done fi ;; esac } # Start a VM pve_start_vm() { local vmid="$1" case "$PVE_METHOD" in ssh) pve_run "qm start ${vmid}" ;; api) pve_api POST "/nodes/${PVE_NODE}/qemu/${vmid}/status/start" >/dev/null ;; esac } # Stop a VM pve_stop_vm() { local vmid="$1" case "$PVE_METHOD" in ssh) pve_run "qm stop ${vmid}" ;; api) pve_api POST "/nodes/${PVE_NODE}/qemu/${vmid}/status/stop" >/dev/null ;; esac } # Destroy a VM pve_destroy_vm() { local vmid="$1" case "$PVE_METHOD" in ssh) pve_run "qm destroy ${vmid} --purge" ;; api) pve_api DELETE "/nodes/${PVE_NODE}/qemu/${vmid}?purge=1" >/dev/null ;; esac } # Set VM configuration pve_set_vm() { local vmid="$1" shift local opts="$*" case "$PVE_METHOD" in ssh) pve_run "qm set ${vmid} ${opts}" ;; api) # Convert CLI-style opts (--key value) to JSON. # JSON avoids form-encoding ambiguity with values containing = and ; # Duplicate keys (e.g. --delete ide2 --delete ide3) are merged with commas. # Shell escapes (e.g. \;) are stripped since the API doesn't need them. local json_payload json_payload=$(echo "$opts" | python3 -c " import json, sys tokens = sys.stdin.read().split() obj = {} i = 0 while i < len(tokens): t = tokens[i] if t.startswith('--'): key = t.lstrip('-') if i + 1 < len(tokens) and not tokens[i+1].startswith('--'): val = tokens[i+1].replace(chr(92), '') i += 2 else: val = '' i += 1 if key in obj and obj[key]: obj[key] = obj[key] + ',' + val else: obj[key] = val else: i += 1 print(json.dumps(obj)) " 2>/dev/null) pve_api PUT "/nodes/${PVE_NODE}/qemu/${vmid}/config" "$json_payload" json >/dev/null ;; esac } # Get VM status pve_vm_status() { local vmid="$1" case "$PVE_METHOD" in ssh) pve_run "qm status ${vmid}" | awk '{print $2}' ;; api) pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" | \ python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status','unknown'))" 2>/dev/null || echo "unknown" ;; esac } # Get VM uptime (seconds); returns 0 if stopped or unknown pve_vm_uptime() { local vmid="$1" case "$PVE_METHOD" in ssh) pve_run "qm status ${vmid} --verbose" | awk '/^uptime:/{print $2}' ;; api) pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" | \ python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('uptime',0))" 2>/dev/null || echo "0" ;; esac } # Get cumulative write bytes for scsi0 (from QEMU blockstat). # Returns 0 if unavailable. Only works with api method. pve_vm_disk_writes() { local vmid="$1" case "$PVE_METHOD" in ssh) echo "0" ;; api) pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" | \ python3 -c " import json,sys d = json.load(sys.stdin).get('data',{}) bs = d.get('blockstat',{}).get('scsi0',{}) print(bs.get('wr_bytes',0)) " 2>/dev/null || echo "0" ;; esac } # Check if a VMID exists pve_vm_exists() { local vmid="$1" if [[ "$DRY_RUN" == true ]]; then return 1 fi case "$PVE_METHOD" in ssh) pve_run "qm status ${vmid}" &>/dev/null ;; api) pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" &>/dev/null ;; esac } # Get VM's IP address (best-effort via QEMU guest agent) pve_vm_ip() { local vmid="$1" # Method 1: Try QEMU guest agent (works if agent is installed) local ip="" case "$PVE_METHOD" in ssh) ip=$(pve_run "qm guest cmd ${vmid} network-get-interfaces 2>/dev/null" | \ python3 -c " import json, sys try: data = json.load(sys.stdin) for iface in data: if iface.get('name') == 'lo': continue for addr in iface.get('ip-addresses', []): if addr.get('ip-address-type') == 'ipv4': print(addr['ip-address']) sys.exit(0) except: pass " 2>/dev/null) || true ;; api) ip=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/agent/network-get-interfaces" 2>/dev/null | \ python3 -c " import json, sys try: data = json.load(sys.stdin).get('data', {}).get('result', []) for iface in data: if iface.get('name') == 'lo': continue for addr in iface.get('ip-addresses', []): if addr.get('ip-address-type') == 'ipv4': print(addr['ip-address']) sys.exit(0) except: pass " 2>/dev/null) || true ;; esac if [[ -n "$ip" ]]; then echo "$ip" return 0 fi # Method 2: ARP-based lookup using VM's MAC address. # Works without guest agent (e.g. IncusOS which is immutable). # Get MAC from VM config, flush stale ARP, ping sweep, look up. local mac="" case "$PVE_METHOD" in ssh) mac=$(pve_run "qm config ${vmid}" 2>/dev/null | \ awk -F'=' '/^net0:/{print $1}' | grep -oiE '[0-9a-f:]{17}') || true ;; api) mac=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null | \ python3 -c " import json, sys, re try: cfg = json.load(sys.stdin).get('data', {}) net0 = cfg.get('net0', '') m = re.search(r'([0-9A-Fa-f:]{17})', net0) if m: print(m.group(1).lower()) except: pass " 2>/dev/null) || true ;; esac if [[ -z "$mac" ]]; then return 1 fi # Populate ARP table to find this MAC's IP address. # Strategy: first check existing ARP, then try broadcast ping, then # fall back to a parallel ping sweep of the local subnet. ip neigh flush nud stale 2>/dev/null || true local bridge_subnet bridge_subnet=$(ip -4 addr show | awk '/inet / && !/127\.0\.0/ {print $2; exit}') # Try broadcast ping first (fast, but many hosts ignore it) if [[ -n "$bridge_subnet" ]]; then local broadcast broadcast=$(python3 -c "import ipaddress; print(ipaddress.ip_interface('${bridge_subnet}').network.broadcast_address)" 2>/dev/null) || true if [[ -n "$broadcast" ]]; then ping -c 2 -W 1 -b "$broadcast" &>/dev/null || true sleep 1 fi fi # Check ARP table ip=$(ip neigh show | awk -v mac="$mac" 'tolower($0) ~ tolower(mac) && (/REACHABLE/ || /DELAY/ || /STALE/) {print $1; exit}') || true if [[ -n "$ip" ]] && ping -c 1 -W 1 "$ip" &>/dev/null; then echo "$ip" return 0 fi # Broadcast didn't work — parallel ping sweep of the subnet if [[ -n "$bridge_subnet" ]]; then local net_prefix net_prefix=$(python3 -c " import ipaddress net = ipaddress.ip_interface('${bridge_subnet}').network for h in net.hosts(): print(h) " 2>/dev/null) if [[ -n "$net_prefix" ]]; then # Ping all hosts in parallel (background, 1s timeout each) while IFS= read -r host_ip; do ping -c 1 -W 1 "$host_ip" &>/dev/null & done <<< "$net_prefix" wait sleep 1 # Re-check ARP table ip=$(ip neigh show | awk -v mac="$mac" 'tolower($0) ~ tolower(mac) && (/REACHABLE/ || /DELAY/) {print $1; exit}') || true if [[ -n "$ip" ]] && ping -c 1 -W 1 "$ip" &>/dev/null; then echo "$ip" return 0 fi fi fi return 1 } # --------------------------------------------------------------------------- # VM management marker # --------------------------------------------------------------------------- # Build the description string for a managed VM build_vm_description() { local config_basename config_basename=$(basename "$CONFIG_FILE") local timestamp timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ") local marker="${MANAGED_MARKER} config=${config_basename} created=${timestamp} by=${SCRIPT_NAME}/${VERSION}" if [[ -n "$PVE_POOL" ]]; then marker+=" pool=${PVE_POOL}" fi echo "$marker" } # Check if a VM has our management marker in its description pve_vm_is_managed() { local vmid="$1" if [[ "$DRY_RUN" == true ]]; then return 1 fi local config_output="" case "$PVE_METHOD" in ssh) config_output=$(pve_run "qm config ${vmid}" 2>/dev/null) || return 1 ;; api) config_output=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null) || return 1 ;; esac echo "$config_output" | grep -qF "$MANAGED_MARKER" } # Check if a VM still has install media (ide2) attached. # Returns 0 if install media is present, 1 if not (already installed). pve_vm_has_install_media() { local vmid="$1" if [[ "$DRY_RUN" == true ]]; then return 1 fi local config_output="" case "$PVE_METHOD" in ssh) config_output=$(pve_run "qm config ${vmid}" 2>/dev/null) || return 1 ;; api) config_output=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null) || return 1 ;; esac echo "$config_output" | grep -q "ide2" } # List all VMs as "vmid name" pairs (one per line) pve_list_vms() { if [[ "$DRY_RUN" == true ]]; then echo "" return 0 fi # When a resource pool is configured, list only pool members (type=qemu) if [[ -n "$PVE_POOL" ]]; then case "$PVE_METHOD" in ssh) pve_run "pvesh get /pools/${PVE_POOL} --output-format json 2>/dev/null" | \ python3 -c " import json, sys data = json.load(sys.stdin) for m in data.get('members', []): if m.get('type') == 'qemu': print('{} {}'.format(m.get('vmid', ''), m.get('name', ''))) " 2>/dev/null || true ;; api) pve_api GET "/pools/${PVE_POOL}" 2>/dev/null | \ python3 -c " import json, sys data = json.load(sys.stdin).get('data', {}) for m in data.get('members', []): if m.get('type') == 'qemu': print('{} {}'.format(m.get('vmid', ''), m.get('name', ''))) " 2>/dev/null || true ;; esac return 0 fi case "$PVE_METHOD" in ssh) pve_run "qm list 2>/dev/null" | awk 'NR>1 {print $1, $2}' || true ;; api) pve_api GET "/nodes/${PVE_NODE}/qemu" 2>/dev/null | \ python3 -c " import json, sys data = json.load(sys.stdin) for vm in data.get('data', []): print('{} {}'.format(vm.get('vmid', ''), vm.get('name', ''))) " 2>/dev/null || true ;; esac } # --------------------------------------------------------------------------- # Pre-flight checks # --------------------------------------------------------------------------- preflight_checks() { step "Running pre-flight checks" local errors=0 local warnings=0 # 1. Check that storage pools exist on Proxmox if [[ "$DRY_RUN" != true ]]; then case "$PVE_METHOD" in ssh) if ! pve_run "pvesm status 2>/dev/null" | awk 'NR>1 {print \$1}' | grep -qw "$PVE_STORAGE"; then error "Storage pool '${PVE_STORAGE}' not found on Proxmox" error "Available pools: $(pve_run "pvesm status 2>/dev/null" | awk 'NR>1 {print \$1}' | tr '\n' ' ')" error "Fix: update 'proxmox.storage' in your config file" errors=$((errors + 1)) fi if ! pve_run "pvesm status 2>/dev/null" | awk 'NR>1 {print \$1}' | grep -qw "$PVE_ISO_STORAGE"; then error "ISO storage '${PVE_ISO_STORAGE}' not found on Proxmox" error "Available pools: $(pve_run "pvesm status 2>/dev/null" | awk 'NR>1 {print \$1}' | tr '\n' ' ')" error "Fix: update 'proxmox.iso_storage' in your config file" errors=$((errors + 1)) fi ;; api) local storage_json storage_json=$(pve_api GET "/nodes/${PVE_NODE}/storage" 2>/dev/null) || storage_json="" local storage_list storage_list=$(echo "$storage_json" | python3 -c " import json, sys data = json.load(sys.stdin) for s in data.get('data', []): print(s.get('storage', '')) " 2>/dev/null) || storage_list="" if ! echo "$storage_list" | grep -qw "$PVE_STORAGE"; then error "Storage pool '${PVE_STORAGE}' not found on Proxmox" error "Available pools: $(echo "$storage_list" | tr '\n' ' ')" error "Fix: update 'proxmox.storage' in your config file" errors=$((errors + 1)) fi if ! echo "$storage_list" | grep -qw "$PVE_ISO_STORAGE"; then error "ISO storage '${PVE_ISO_STORAGE}' not found on Proxmox" error "Available pools: $(echo "$storage_list" | tr '\n' ' ')" error "Fix: update 'proxmox.iso_storage' in your config file" errors=$((errors + 1)) else # Check that ISO storage accepts 'iso' content type local iso_content iso_content=$(echo "$storage_json" | python3 -c " import json, sys data = json.load(sys.stdin) for s in data.get('data', []): if s.get('storage') == '${PVE_ISO_STORAGE}': print(s.get('content', '')) break " 2>/dev/null) || iso_content="" if [[ -n "$iso_content" ]] && ! echo "$iso_content" | grep -q "iso"; then error "Storage '${PVE_ISO_STORAGE}' does not accept ISO content (content types: ${iso_content})" error "Fix: enable 'iso' content on '${PVE_ISO_STORAGE}' in Proxmox, or use a different iso_storage" errors=$((errors + 1)) fi fi ;; esac # 2. Network bridge check (SSH only, warn-level) if [[ "$PVE_METHOD" == "ssh" ]]; then if ! pve_run "ip link show ${PVE_BRIDGE} 2>/dev/null" &>/dev/null; then warn "Network bridge '${PVE_BRIDGE}' not found on Proxmox" warn "VMs may fail to get network connectivity" warn "Fix: update 'proxmox.bridge' in your config file" warnings=$((warnings + 1)) fi fi # 3. ISO availability: check if already on Proxmox or CDN reachable if [[ -z "$LOCAL_ISO" ]]; then local any_iso_present=false local iso_list="" case "$PVE_METHOD" in ssh) iso_list=$(pve_run "pvesm list ${PVE_ISO_STORAGE} --content iso 2>/dev/null" 2>/dev/null) || iso_list="" ;; api) iso_list=$(pve_api GET "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content?content=iso" 2>/dev/null) || iso_list="" ;; esac if echo "$iso_list" | grep -q "IncusOS_"; then any_iso_present=true fi if [[ "$any_iso_present" != true ]]; then # No ISO on Proxmox -- check if CDN is reachable if ! curl -fsSL --connect-timeout 5 -o /dev/null "$CDN_INDEX" 2>/dev/null; then error "No IncusOS ISO found on Proxmox and CDN is unreachable: ${CDN_INDEX}" error "Fix: provide a local ISO with --iso FILE, or check internet connectivity" errors=$((errors + 1)) fi fi fi fi # 4. Resource pool check (if configured) if [[ -n "$PVE_POOL" ]] && [[ "$DRY_RUN" != true ]]; then local pool_ok=false case "$PVE_METHOD" in ssh) if pve_run "pvesh get /pools/${PVE_POOL} --output-format json 2>/dev/null" &>/dev/null; then pool_ok=true fi ;; api) if pve_api GET "/pools/${PVE_POOL}" &>/dev/null; then pool_ok=true fi ;; esac if [[ "$pool_ok" != true ]]; then error "Resource pool '${PVE_POOL}' not found on Proxmox" error "Create it with these commands on the Proxmox host:" error " pveum pool add ${PVE_POOL} --comment \"IncusOS Lab VMs\"" error " pveum pool modify ${PVE_POOL} --storage ${PVE_STORAGE},${PVE_ISO_STORAGE}" if [[ "$PVE_METHOD" == "api" ]] && [[ -n "$PVE_API_TOKEN_ID" ]]; then local api_user api_user=$(echo "$PVE_API_TOKEN_ID" | cut -d'!' -f1) error " pveum aclmod /pool/${PVE_POOL} -user ${api_user} -role IncusOSDeploy" fi errors=$((errors + 1)) else detail "Resource pool '${PVE_POOL}' exists" fi fi # 5. genisoimage for SEED_DATA ISO 9660 generation if ! command -v genisoimage &>/dev/null; then error "genisoimage is required for SEED_DATA generation but not found" error "Install with: sudo apt install genisoimage" errors=$((errors + 1)) fi if [[ $errors -gt 0 ]]; then error "${errors} pre-flight check(s) failed" exit 1 fi if [[ $warnings -gt 0 ]]; then success "Pre-flight checks passed with ${warnings} warning(s)" else success "Pre-flight checks passed" fi } # --------------------------------------------------------------------------- # Connectivity check # --------------------------------------------------------------------------- check_connectivity() { step "Checking Proxmox connectivity (${PVE_METHOD})" case "$PVE_METHOD" in ssh) if ! ssh -o BatchMode=yes -o ConnectTimeout=10 \ "${PVE_SSH_USER}@${PVE_HOST}" "pvesh get /version --output-format json" &>/dev/null; then error "Cannot connect to Proxmox host via SSH" error "Ensure key-based SSH auth is set up:" error " ssh-copy-id ${PVE_SSH_USER}@${PVE_HOST}" error "Test with: ssh ${PVE_SSH_USER}@${PVE_HOST} pvesh get /version" exit 1 fi local pve_version pve_version=$(ssh -o BatchMode=yes "${PVE_SSH_USER}@${PVE_HOST}" \ "pvesh get /version --output-format json" 2>/dev/null | \ python3 -c "import json,sys; print(json.load(sys.stdin).get('version','unknown'))" 2>/dev/null) || pve_version="unknown" success "Connected to ${PVE_HOST} (Proxmox VE ${pve_version})" ;; api) local url="https://${PVE_HOST}:8006/api2/json/version" local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET:-}" local response if ! response=$(curl -fsSk -H "$auth" "$url" 2>/dev/null); then error "Cannot connect to Proxmox API at https://${PVE_HOST}:8006" error "Check that:" error " 1. The host is reachable" error " 2. The API token ID is correct: ${PVE_API_TOKEN_ID}" error " 3. PROXMOX_TOKEN_SECRET is set" exit 1 fi local pve_version pve_version=$(echo "$response" | \ python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('version','unknown'))" 2>/dev/null) || pve_version="unknown" success "Connected to ${PVE_HOST} (Proxmox VE ${pve_version})" ;; esac } # --------------------------------------------------------------------------- # VMID allocation # --------------------------------------------------------------------------- allocate_vmids() { step "Allocating VMIDs" # Get existing VMs as "vmid name" pairs local vm_list vm_list=$(pve_list_vms) # Build lookup: existing IDs and name-to-VMID map local existing_ids="" declare -A existing_name_to_vmid if [[ -n "$vm_list" ]]; then while IFS=' ' read -r eid ename; do [[ -z "$eid" ]] && continue existing_ids="${existing_ids} ${eid}" if [[ -n "$ename" ]]; then existing_name_to_vmid["$ename"]="$eid" fi done <<< "$vm_list" fi local next_id="$DEF_START_VMID" local count=${#VM_NAMES[@]} local errors=0 local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" # Skip if filtered if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi # Check for name collision with existing Proxmox VMs if [[ -n "${existing_name_to_vmid[$name]+x}" ]]; then local collision_vmid="${existing_name_to_vmid[$name]}" if pve_vm_is_managed "$collision_vmid"; then warn "VM '${name}' already exists at VMID ${collision_vmid} (managed by ${SCRIPT_NAME})" warn "Use --cleanup to redeploy, or remove it manually" # Assign the existing VMID so cleanup/skip works correctly VM_VMIDS[$i]="$collision_vmid" i=$((i + 1)) continue else error "VM name '${name}' already exists at VMID ${collision_vmid} but is NOT managed by ${SCRIPT_NAME}" error "This VM was not created by this tool and will not be modified" error "Fix: rename the VM in your config file, or manually remove VMID ${collision_vmid}:" error " ssh ${PVE_SSH_USER}@${PVE_HOST} qm destroy ${collision_vmid} --purge" errors=$((errors + 1)) i=$((i + 1)) continue fi fi if [[ -n "${VM_VMIDS[$i]}" ]]; then # Explicit VMID detail "VM '${name}': VMID ${VM_VMIDS[$i]} (explicit)" else # Auto-assign, skipping existing while echo "$existing_ids" | grep -qw "$next_id" 2>/dev/null; do next_id=$((next_id + 1)) done VM_VMIDS[$i]="$next_id" detail "VM '${name}': VMID ${next_id} (auto-assigned)" next_id=$((next_id + 1)) fi i=$((i + 1)) done if [[ $errors -gt 0 ]]; then error "${errors} name collision(s) with unmanaged VMs -- cannot continue" exit 1 fi success "VMIDs allocated" } # --------------------------------------------------------------------------- # Certificate detection (reused pattern from sibling scripts) # --------------------------------------------------------------------------- CERT_FILE_RESOLVED="" detect_client_cert() { if [[ "$NO_CERT" == true ]]; then info "Skipping certificate injection (--no-cert)" return fi if [[ -n "$CERT_FILE" ]]; then if [[ ! -f "$CERT_FILE" ]]; then error "Certificate file not found: ${CERT_FILE}" exit 1 fi if ! grep -q "BEGIN CERTIFICATE" "$CERT_FILE"; then error "File does not appear to be a PEM certificate: ${CERT_FILE}" exit 1 fi CERT_FILE_RESOLVED="$CERT_FILE" success "Certificate: ${CERT_FILE_RESOLVED}" return fi step "Auto-detecting client certificate" local cert_paths=( "${HOME}/.config/incus/client.crt" "${HOME}/snap/incus/common/config/client.crt" "${HOME}/.config/lxc/client.crt" ) for path in "${cert_paths[@]}"; do if [[ -f "$path" ]] && grep -q "BEGIN CERTIFICATE" "$path"; then CERT_FILE_RESOLVED="$path" success "Certificate: ${CERT_FILE_RESOLVED}" return fi done # Try CLI fallback if command -v incus &>/dev/null; then local tmpfile="${WORKDIR}/client.crt" if incus remote get-client-certificate > "$tmpfile" 2>/dev/null; then CERT_FILE_RESOLVED="$tmpfile" success "Certificate obtained via Incus CLI" return fi fi # Check if any VM is ops-center (which requires a cert) local needs_cert=false for app in "${VM_APPS[@]}"; do if [[ "$app" == "operations-center" ]]; then needs_cert=true break fi done if [[ "$needs_cert" == true ]]; then error "No client certificate found" error "Operations Center requires at least one trusted certificate for access" error "" error "Options:" error " --cert FILE Provide a certificate file directly" error " Generate one: incus remote list (creates ~/.config/incus/client.crt)" error " --no-cert Skip certificate injection (you will be locked out!)" exit 1 fi warn "No client certificate found -- VMs will require manual trust setup" warn "Use --cert FILE to provide a certificate, or --no-cert to suppress this warning" } # --------------------------------------------------------------------------- # CDN operations (reused from incusos-iso) # --------------------------------------------------------------------------- LATEST_VERSION="" fetch_latest_version() { local channel="${DEF_CHANNEL:-stable}" step "Fetching latest IncusOS version (channel: ${channel})" local index_file="${WORKDIR}/cdn-index.json" if ! curl -fsSL "$CDN_INDEX" -o "$index_file" 2>/dev/null; then error "Failed to fetch CDN index from ${CDN_INDEX}" error "Check your internet connection and try again" exit 1 fi if command -v jq &>/dev/null; then LATEST_VERSION=$(jq -r \ --arg ch "$channel" \ '[.updates[] | select(.channels[] == $ch)] | sort_by(.version) | last | .version // empty' \ "$index_file") else LATEST_VERSION=$(python3 -c " import json data = json.load(open('${index_file}')) versions = [u['version'] for u in data.get('updates', []) if '${channel}' in u.get('channels', [])] if versions: print(sorted(versions)[-1]) " 2>/dev/null) fi if [[ -z "$LATEST_VERSION" ]]; then error "No version found for channel '${channel}'" exit 1 fi success "Latest IncusOS version: ${LATEST_VERSION}" } download_iso() { local version="$1" local output_path="$2" local iso_filename iso_filename=$(basename "$output_path") # Check persistent cache first local cached_iso="${CACHEDIR}/${iso_filename}" if [[ -f "$cached_iso" ]]; then success "ISO found in local cache: ${cached_iso}" cp "$cached_iso" "$output_path" return 0 fi local url="${CDN_BASE}/${version}/x86_64/IncusOS_${version}.iso.gz" local gz_path="${cached_iso}.gz" step "Downloading IncusOS ISO" detail "URL: ${url}" detail "Cache directory: ${CACHEDIR}" if ! curl -fL --progress-bar "$url" -o "$gz_path"; then error "Failed to download ISO from ${url}" rm -f "$gz_path" exit 1 fi step "Decompressing ISO" if ! gzip -d "$gz_path"; then local avail avail=$(df -h --output=avail "$(dirname "$gz_path")" 2>/dev/null | tail -1 | tr -d ' ') || avail="unknown" error "Failed to decompress ISO (${avail} available on disk)" error "The decompressed ISO requires several GiB of disk space" error "Free up space, or set TMPDIR to a larger filesystem:" error " TMPDIR=/path/with/space ${SCRIPT_NAME} ${CONFIG_FILE}" rm -f "$gz_path" exit 1 fi # Copy from cache to workdir cp "$cached_iso" "$output_path" success "ISO ready: ${output_path} (cached for future runs)" } # --------------------------------------------------------------------------- # Deployment plan summary # --------------------------------------------------------------------------- print_plan() { local count=${#VM_NAMES[@]} echo "" echo -e "${BOLD}Deployment Plan${RESET}" echo -e " Proxmox host: ${CYAN}${PVE_HOST}${RESET} (${PVE_METHOD})" echo -e " Node: ${PVE_NODE}" echo -e " Storage: ${PVE_STORAGE} (disk), ${PVE_ISO_STORAGE} (iso)" echo -e " Bridge: ${PVE_BRIDGE}" if [[ -n "$PVE_POOL" ]]; then echo -e " Pool: ${CYAN}${PVE_POOL}${RESET}" fi if [[ -n "$CERT_FILE_RESOLVED" ]]; then echo -e " Certificate: ${GREEN}${CERT_FILE_RESOLVED}${RESET}" elif [[ "$NO_CERT" == true ]]; then echo -e " Certificate: ${DIM}none (--no-cert)${RESET}" else echo -e " Certificate: ${YELLOW}not found${RESET}" fi echo "" echo -e " ${BOLD}VMs (${count}):${RESET}" local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" local skip="" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then skip=" ${DIM}(skipped)${RESET}" fi echo -e " ${CYAN}${name}${RESET}${skip}" echo -e " VMID: ${VM_VMIDS[$i]} App: ${VM_APPS[$i]} Defaults: ${VM_APPLY_DEFAULTS[$i]}" echo -e " CPU: ${VM_CORES[$i]} cores RAM: ${VM_MEMORY[$i]} MiB Disk: ${VM_DISK[$i]} GiB" if [[ -n "${VM_CLUSTERS[$i]}" ]]; then echo -e " Cluster: ${VM_CLUSTERS[$i]} (${VM_CLUSTER_ROLES[$i]})" fi i=$((i + 1)) done echo "" } # --------------------------------------------------------------------------- # Confirmation prompt # --------------------------------------------------------------------------- confirm() { local message="$1" if [[ "$YES" == true ]] || [[ "$DRY_RUN" == true ]]; then return 0 fi echo -n -e "${BOLD}${message} [y/N] ${RESET}" local reply read -r reply if [[ ! "$reply" =~ ^[Yy]$ ]]; then info "Aborted" exit 0 fi } # --------------------------------------------------------------------------- # Phase 1: Validate # --------------------------------------------------------------------------- phase_validate() { step "Phase 1: Validate" echo "" parse_config load_config validate_config check_connectivity allocate_vmids preflight_checks detect_client_cert print_plan } # --------------------------------------------------------------------------- # Phase 2: Prepare (download ISO + generate seeds) # --------------------------------------------------------------------------- ISO_FILENAME="" phase_prepare() { step "Phase 2: Prepare" echo "" # Determine ISO if [[ -n "$LOCAL_ISO" ]]; then ISO_FILENAME=$(basename "$LOCAL_ISO") info "Using local ISO: ${LOCAL_ISO}" if [[ "$DRY_RUN" != true ]]; then cp "$LOCAL_ISO" "${WORKDIR}/${ISO_FILENAME}" fi elif [[ "$DRY_RUN" == true ]]; then ISO_FILENAME="IncusOS_latest.iso" detail "[dry-run] Would fetch latest version from CDN and download ISO" else # Check if ISO already exists on Proxmox fetch_latest_version ISO_FILENAME="IncusOS_${LATEST_VERSION}.iso" if pve_iso_exists "$ISO_FILENAME"; then success "ISO already on Proxmox: ${ISO_FILENAME} (reusing from previous deployment)" else download_iso "$LATEST_VERSION" "${WORKDIR}/${ISO_FILENAME}" fi fi # Generate per-VM seed images step "Generating seed images" local count=${#VM_NAMES[@]} local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" # Skip if filtered if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local app="${VM_APPS[$i]}" local apply_defaults="${VM_APPLY_DEFAULTS[$i]}" local seed_output="${WORKDIR}/seed-${name}.iso" detail "Generating seed for '${name}' (${app}, defaults=${apply_defaults})" # Build incusos-seed command local seed_cmd=("$SEED_TOOL") seed_cmd+=(--format iso) seed_cmd+=(--app "$app") seed_cmd+=(--output "$seed_output") seed_cmd+=(--force-install) seed_cmd+=(--force-reboot) seed_cmd+=(--hostname "$name") seed_cmd+=(--quiet) if [[ "$apply_defaults" == "true" ]]; then seed_cmd+=(--defaults) fi if [[ -n "$CERT_FILE_RESOLVED" ]]; then seed_cmd+=(--cert "$CERT_FILE_RESOLVED") elif [[ "$NO_CERT" == true ]]; then seed_cmd+=(--no-cert) fi if [[ "$DRY_RUN" == true ]]; then detail "[dry-run] ${seed_cmd[*]}" else if ! "${seed_cmd[@]}"; then error "Failed to generate seed for VM '${name}'" exit 1 fi fi i=$((i + 1)) done success "Seed images generated" } # --------------------------------------------------------------------------- # Phase 3: Upload # --------------------------------------------------------------------------- phase_upload() { step "Phase 3: Upload" echo "" # Upload ISO (if we downloaded it) local iso_path="${WORKDIR}/${ISO_FILENAME}" if [[ -f "$iso_path" ]]; then if pve_iso_exists "$ISO_FILENAME"; then success "ISO already on Proxmox: ${ISO_FILENAME}" else step "Uploading ISO: ${ISO_FILENAME}" if ! pve_upload "$iso_path" "$ISO_FILENAME"; then error "Failed to upload ISO to Proxmox" exit 1 fi success "ISO uploaded: ${ISO_FILENAME}" fi else detail "ISO already on Proxmox, skipping upload" fi # Upload seed images local count=${#VM_NAMES[@]} local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local seed_file="seed-${name}.iso" local seed_path="${WORKDIR}/${seed_file}" if [[ -f "$seed_path" ]] || [[ "$DRY_RUN" == true ]]; then step "Uploading seed: ${seed_file}" if ! pve_upload "$seed_path" "$seed_file"; then error "Failed to upload seed for VM '${name}'" exit 1 fi success "Seed uploaded: ${seed_file}" fi i=$((i + 1)) done success "All files uploaded" } # --------------------------------------------------------------------------- # Phase 4: Create VMs # --------------------------------------------------------------------------- phase_create() { step "Phase 4: Create VMs" echo "" local count=${#VM_NAMES[@]} local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local vmid="${VM_VMIDS[$i]}" local cores="${VM_CORES[$i]}" local memory="${VM_MEMORY[$i]}" local disk="${VM_DISK[$i]}" local seed_file="seed-${name}.iso" step "Creating VM '${name}' (VMID ${vmid})" # Check if VM already exists if pve_vm_exists "$vmid"; then if pve_vm_is_managed "$vmid"; then warn "VM ${vmid} ('${name}') already exists (managed) -- skipping" warn "Use --cleanup to redeploy" else warn "VM ${vmid} ('${name}') already exists (NOT managed by ${SCRIPT_NAME}) -- skipping" warn "Choose a different VMID or remove it manually:" warn " ssh ${PVE_SSH_USER}@${PVE_HOST} qm destroy ${vmid} --purge" fi i=$((i + 1)) continue fi detail "cores=${cores} memory=${memory}MiB disk=${disk}GiB" detail "bios=ovmf machine=q35 cpu=host scsihw=virtio-scsi-pci" detail "efidisk: pre-enrolled-keys=0, tpm: v2.0, balloon: 0" detail "iso=${ISO_FILENAME} seed=${seed_file}" if ! pve_create_vm "$vmid" "$name" "$cores" "$memory" "$disk" \ "$ISO_FILENAME" "$seed_file"; then error "Failed to create VM '${name}' (VMID ${vmid})" exit 1 fi success "VM '${name}' created (VMID ${vmid})" i=$((i + 1)) done success "All VMs created" } # --------------------------------------------------------------------------- # Phase 5: Install # --------------------------------------------------------------------------- phase_install() { step "Phase 5: Install" echo "" local count=${#VM_NAMES[@]} local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local vmid="${VM_VMIDS[$i]}" if [[ "$DRY_RUN" == true ]]; then step "Starting VM '${name}' (VMID ${vmid})" detail "[dry-run] Would start VM, swap boot order, wait for install, clean up" i=$((i + 1)) continue fi # Idempotency: check current VM state before acting if pve_vm_exists "$vmid"; then local cur_status cur_status=$(pve_vm_status "$vmid") || cur_status="unknown" local has_media=true pve_vm_has_install_media "$vmid" || has_media=false if [[ "$cur_status" == "running" ]] && [[ "$has_media" == false ]]; then # Already installed and running — detect IP and skip success "VM '${name}' (VMID ${vmid}) is already installed and running" local ip ip=$(pve_vm_ip "$vmid") || ip="" if [[ -n "$ip" ]]; then detail "IP: ${ip}" fi i=$((i + 1)) continue fi if [[ "$cur_status" == "stopped" ]] && [[ "$has_media" == false ]]; then # Installed but stopped — start from disk step "VM '${name}' is installed but stopped — starting from disk" if pve_start_vm "$vmid"; then sleep 30 local ip ip=$(pve_vm_ip "$vmid") || ip="" if [[ -n "$ip" ]]; then success "VM '${name}' started at ${ip}" else success "VM '${name}' started from disk" fi else error "Failed to start VM '${name}' from disk" fi i=$((i + 1)) continue fi if [[ "$cur_status" == "running" ]] && [[ "$has_media" == true ]]; then # Mid-install or stuck — fall through to blockstat monitoring info "VM '${name}' is running with install media — monitoring installation" fi # stopped + has_media → needs install, fall through fi step "Starting VM '${name}' (VMID ${vmid})" # 1. Start VM (boots from ISO, reads SEED_DATA, installs) if ! pve_start_vm "$vmid"; then error "Failed to start VM '${name}' (VMID ${vmid})" i=$((i + 1)) continue fi success "VM '${name}' started" # 2. Wait for installation to complete by monitoring disk writes # # Install flow: boot ISO -> read SEED_DATA -> install to disk -> reboot # (force_reboot in seed). A guest-level reboot does NOT reset the QEMU # VM uptime, so we cannot detect reboot that way. Instead we monitor # scsi0 write bytes via blockstat: # - During install: wr_bytes increases steadily # - After install + reboot: wr_bytes stops (VM is idle at post-reboot # screen or "install media detected" error) # # When wr_bytes has been stable for STABLE_COUNT consecutive polls, # installation is complete. We then stop the VM, remove install media, # and start cleanly from disk. step "Waiting for '${name}' to complete installation" detail "Monitoring disk write activity (polling every 5s, timeout: 10 minutes)" local timeout=600 local interval=5 local elapsed=0 local install_done=false local prev_wr_bytes=0 local writes_started=false local stable_count=0 local STABLE_THRESHOLD=3 # 3 consecutive polls with no change = 15s idle while [[ $elapsed -lt $timeout ]]; do sleep "$interval" elapsed=$((elapsed + interval)) local status status=$(pve_vm_status "$vmid") || status="unknown" case "$status" in stopped) detail "VM stopped [${elapsed}s]" install_done=true break ;; running) local wr_bytes wr_bytes=$(pve_vm_disk_writes "$vmid") || wr_bytes=0 if [[ $wr_bytes -gt $prev_wr_bytes ]]; then # Disk writes are happening -- install in progress writes_started=true stable_count=0 local wr_mb=$(( wr_bytes / 1048576 )) # Show progress every 15s if [[ $((elapsed % 15)) -eq 0 ]]; then detail "Installing... (${wr_mb} MiB written) [${elapsed}s]" fi elif [[ "$writes_started" == true ]]; then # Writes have stopped after being active stable_count=$((stable_count + 1)) if [[ $stable_count -ge $STABLE_THRESHOLD ]]; then local wr_mb=$(( wr_bytes / 1048576 )) detail "Disk activity stopped (${wr_mb} MiB written total) [${elapsed}s]" install_done=true break fi fi prev_wr_bytes=$wr_bytes ;; *) detail "VM status: ${status} [${elapsed}s]" ;; esac done if [[ "$install_done" != true ]]; then warn "VM '${name}' did not complete installation within ${timeout}s" warn "Check the VM console on Proxmox for details" i=$((i + 1)) continue fi # 3. Stop the VM (it's either at "remove install media" prompt or # post-reboot "install media detected" error -- either way idle). local cur_status cur_status=$(pve_vm_status "$vmid") || cur_status="unknown" if [[ "$cur_status" == "running" ]]; then detail "Stopping VM to remove install media..." pve_stop_vm "$vmid" || true sleep 5 fi # 4. Remove install media and set clean boot order step "Removing install media from '${name}'" pve_set_vm "$vmid" "--delete ide2 --delete ide3 --boot order=scsi0" || true success "Install media detached, boot order set to disk-only" # 5. Start VM from disk step "Starting '${name}' from disk" if ! pve_start_vm "$vmid"; then error "Failed to start VM '${name}' from disk" i=$((i + 1)) continue fi # 5. Wait for services to come up detail "Waiting for services to start..." sleep 30 # 6. Try to detect IP local ip ip=$(pve_vm_ip "$vmid") || ip="" echo "" if [[ -n "$ip" ]]; then success "VM '${name}' installed and running at ${ip}" if [[ -n "$CERT_FILE_RESOLVED" ]]; then detail "Add remote: incus remote add ${name} ${ip}" fi else success "VM '${name}' installed and running" detail "Check Proxmox console for IP address" fi echo "" i=$((i + 1)) done success "Installation phase complete" } # --------------------------------------------------------------------------- # Status check # --------------------------------------------------------------------------- phase_status() { step "Deployment Status" echo "" local count=${#VM_NAMES[@]} local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local vmid="${VM_VMIDS[$i]}" local app="${VM_APPS[$i]}" echo -e " ${BOLD}${name}${RESET} (VMID ${vmid}, app: ${app})" # Proxmox state if [[ "$DRY_RUN" == true ]]; then echo -e " Proxmox: ${DIM}[dry-run]${RESET}" i=$((i + 1)) echo "" continue fi if ! pve_vm_exists "$vmid"; then echo -e " Proxmox: ${RED}not found${RESET}" i=$((i + 1)) echo "" continue fi local managed=false pve_vm_is_managed "$vmid" && managed=true local status status=$(pve_vm_status "$vmid") || status="unknown" local has_media=false pve_vm_has_install_media "$vmid" && has_media=true local install_state="installed" if [[ "$has_media" == true ]]; then install_state="not installed (media attached)" fi local status_color="$GREEN" if [[ "$status" != "running" ]]; then status_color="$YELLOW" fi echo -e " Proxmox: exists managed: ${managed} status: ${status_color}${status}${RESET} ${install_state}" # Network and service checks (only for running, installed VMs) if [[ "$status" == "running" ]] && [[ "$has_media" == false ]]; then local ip ip=$(pve_vm_ip "$vmid") || ip="" if [[ -n "$ip" ]]; then local port_status="${RED}closed${RESET}" if curl -sk --connect-timeout 5 "https://${ip}:8443" &>/dev/null; then port_status="${GREEN}open${RESET}" fi echo -e " Network: ${ip} port 8443: ${port_status}" # Incus remote check if command -v incus &>/dev/null; then local remote_name="" remote_name=$(incus remote list --format csv 2>/dev/null | \ awk -F',' -v addr="$ip" '$2 ~ addr {print $1; exit}') || true if [[ -n "$remote_name" ]]; then echo -e " Remote: ${GREEN}incus remote '${remote_name}' configured${RESET}" else echo -e " Remote: ${YELLOW}no incus remote for ${ip}${RESET}" fi fi else echo -e " Network: ${YELLOW}IP not detected${RESET}" fi fi i=$((i + 1)) echo "" done } # --------------------------------------------------------------------------- # Post-deployment checks # --------------------------------------------------------------------------- phase_post_deploy() { step "Post-Deployment Checks" echo "" local count=${#VM_NAMES[@]} local all_ok=true local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local vmid="${VM_VMIDS[$i]}" local app="${VM_APPS[$i]}" if [[ "$DRY_RUN" == true ]]; then detail "[dry-run] Would check '${name}' (VMID ${vmid})" i=$((i + 1)) continue fi # Only check running, installed VMs if ! pve_vm_exists "$vmid"; then i=$((i + 1)) continue fi local status status=$(pve_vm_status "$vmid") || status="unknown" if [[ "$status" != "running" ]]; then i=$((i + 1)) continue fi local has_media=false pve_vm_has_install_media "$vmid" && has_media=true if [[ "$has_media" == true ]]; then i=$((i + 1)) continue fi local ip ip=$(pve_vm_ip "$vmid") || ip="" if [[ -z "$ip" ]]; then warn "VM '${name}': could not detect IP address" all_ok=false i=$((i + 1)) continue fi # Port 8443 check if curl -sk --connect-timeout 5 "https://${ip}:8443" &>/dev/null; then success "VM '${name}' (${ip}): port 8443 reachable" else warn "VM '${name}' (${ip}): port 8443 not reachable (may still be starting)" all_ok=false i=$((i + 1)) continue fi # App-specific checks if [[ "$app" == "incus" ]] && command -v incus &>/dev/null; then local remote_name="" remote_name=$(incus remote list --format csv 2>/dev/null | \ awk -F',' -v addr="$ip" '$2 ~ addr {print $1; exit}') || true if [[ -n "$remote_name" ]]; then if incus info "${remote_name}:" &>/dev/null; then success "VM '${name}': incus remote '${remote_name}' connected" else warn "VM '${name}': incus remote '${remote_name}' exists but connection failed" all_ok=false fi else detail "VM '${name}': no incus remote configured" detail " Add with: incus remote add ${name} ${ip} --accept-certificate" fi elif [[ "$app" == "operations-center" ]]; then success "VM '${name}': Operations Center at https://${ip}:8443" detail " Import PKCS#12 cert in browser for web UI access:" detail " openssl pkcs12 -export -inkey ~/.config/incus/client.key -in ~/.config/incus/client.crt -out client.pfx" fi i=$((i + 1)) done if [[ "$all_ok" == true ]] && [[ "$DRY_RUN" != true ]]; then echo "" success "All post-deployment checks passed" fi } # --------------------------------------------------------------------------- # Cleanup mode # --------------------------------------------------------------------------- do_cleanup() { step "Cleanup: destroying VMs defined in config" if [[ "$FORCE_CLEANUP" == true ]]; then warn "Force cleanup enabled -- will destroy VMs even without management marker" fi echo "" local count=${#VM_NAMES[@]} local destroyed=0 local not_found=0 local skipped_unmanaged=0 local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local vmid="${VM_VMIDS[$i]}" # 1. Check if VM exists if ! pve_vm_exists "$vmid"; then detail "VM '${name}' (VMID ${vmid}): does not exist" not_found=$((not_found + 1)) i=$((i + 1)) continue fi # 2. Check if VM is managed by us if [[ "$DRY_RUN" != true ]]; then if ! pve_vm_is_managed "$vmid"; then if [[ "$FORCE_CLEANUP" == true ]]; then warn "VM '${name}' (VMID ${vmid}): NOT managed by ${SCRIPT_NAME} -- destroying anyway (--force-cleanup)" else warn "VM '${name}' (VMID ${vmid}): NOT managed by ${SCRIPT_NAME} -- refusing to destroy" warn " This VM was not created by this tool. To destroy it manually:" warn " ssh ${PVE_SSH_USER}@${PVE_HOST} qm stop ${vmid} && ssh ${PVE_SSH_USER}@${PVE_HOST} qm destroy ${vmid} --purge" warn " Or use --force-cleanup to override this safety check" skipped_unmanaged=$((skipped_unmanaged + 1)) i=$((i + 1)) continue fi fi fi # 3. Destroy the VM step "Destroying VM '${name}' (VMID ${vmid})" if [[ "$DRY_RUN" == true ]]; then detail "[dry-run] Would stop and destroy VM ${vmid}" destroyed=$((destroyed + 1)) i=$((i + 1)) continue fi # Stop if running local status status=$(pve_vm_status "$vmid") || status="unknown" if [[ "$status" == "running" ]]; then detail "Stopping VM..." pve_stop_vm "$vmid" || true sleep 5 fi # Destroy if pve_destroy_vm "$vmid"; then success "VM '${name}' (VMID ${vmid}) destroyed" destroyed=$((destroyed + 1)) else error "Failed to destroy VM '${name}' (VMID ${vmid})" fi i=$((i + 1)) done # Summary echo "" local summary="Cleanup complete: ${destroyed} destroyed" if [[ $not_found -gt 0 ]]; then summary+=", ${not_found} not found" fi if [[ $skipped_unmanaged -gt 0 ]]; then summary+=", ${skipped_unmanaged} skipped (unmanaged)" fi success "$summary" } # Deep cleanup: remove ISOs, seeds, remotes, and local cache cleanup_storage() { step "Deep cleanup: removing ISOs and seeds from Proxmox storage" local count=${#VM_NAMES[@]} local i=0 # Remove per-VM seed ISOs while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local seed_file="seed-${name}.iso" if [[ "$DRY_RUN" == true ]]; then detail "[dry-run] Would delete ${PVE_ISO_STORAGE}:iso/${seed_file}" else case "$PVE_METHOD" in ssh) pve_run "pvesm free ${PVE_ISO_STORAGE}:iso/${seed_file} 2>/dev/null" &>/dev/null || true ;; api) pve_api DELETE "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content/${PVE_ISO_STORAGE}:iso/${seed_file}" &>/dev/null || true ;; esac detail "Deleted seed: ${seed_file}" fi i=$((i + 1)) done # Remove IncusOS ISOs from Proxmox storage local iso_list="" if [[ "$DRY_RUN" != true ]]; then case "$PVE_METHOD" in ssh) iso_list=$(pve_run "pvesm list ${PVE_ISO_STORAGE} --content iso 2>/dev/null" 2>/dev/null | grep "IncusOS_" | awk '{print $1}') || iso_list="" ;; api) iso_list=$(pve_api GET "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content?content=iso" 2>/dev/null | \ python3 -c " import json, sys data = json.load(sys.stdin) for item in data.get('data', []): volid = item.get('volid', '') if 'IncusOS_' in volid: print(volid) " 2>/dev/null) || iso_list="" ;; esac fi if [[ -n "$iso_list" ]]; then while IFS= read -r volid; do [[ -z "$volid" ]] && continue if [[ "$DRY_RUN" == true ]]; then detail "[dry-run] Would delete ${volid}" else case "$PVE_METHOD" in ssh) pve_run "pvesm free ${volid} 2>/dev/null" &>/dev/null || true ;; api) local encoded_volid encoded_volid=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${volid}', safe=''))" 2>/dev/null) pve_api DELETE "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content/${encoded_volid}" &>/dev/null || true ;; esac detail "Deleted ISO: ${volid}" fi done <<< "$iso_list" fi success "Storage cleanup complete" } cleanup_remotes() { step "Deep cleanup: removing incus remotes" if ! command -v incus &>/dev/null; then detail "incus client not installed, skipping remote cleanup" return fi local count=${#VM_NAMES[@]} local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi # Check if a remote with this name exists if incus remote list --format csv 2>/dev/null | grep -q "^${name},"; then if [[ "$DRY_RUN" == true ]]; then detail "[dry-run] Would remove incus remote '${name}'" else incus remote remove "$name" 2>/dev/null || true detail "Removed incus remote: ${name}" fi fi i=$((i + 1)) done success "Remote cleanup complete" } cleanup_local_cache() { step "Deep cleanup: clearing local ISO cache" local cache_base="${XDG_CACHE_HOME:-${HOME}/.cache}/${SCRIPT_NAME}/iso-cache" if [[ -d "$cache_base" ]]; then if [[ "$DRY_RUN" == true ]]; then detail "[dry-run] Would clear ${cache_base}" else rm -rf "$cache_base" detail "Cleared cache: ${cache_base}" fi else detail "No local cache to clear" fi success "Local cache cleanup complete" } # Pool-wide cleanup: destroy ALL managed VMs in the resource pool do_cleanup_all() { step "Pool-wide cleanup: destroying all managed VMs" echo "" if [[ -z "$PVE_POOL" ]]; then error "Pool-wide cleanup requires a resource pool (proxmox.pool) in the config" error "Without a pool, there is no safe way to scope which VMs to destroy" exit 1 fi # Get all VMs in the pool local vm_list vm_list=$(pve_list_vms) if [[ -z "$vm_list" ]]; then info "No VMs found in pool '${PVE_POOL}'" return fi # Filter for managed VMs local managed_vms=() local managed_names=() while IFS=' ' read -r eid ename; do [[ -z "$eid" ]] && continue if pve_vm_is_managed "$eid"; then managed_vms+=("$eid") managed_names+=("${ename:-unknown}") fi done <<< "$vm_list" if [[ ${#managed_vms[@]} -eq 0 ]]; then info "No managed VMs found in pool '${PVE_POOL}'" return fi # Show what will be destroyed echo -e " ${BOLD}Managed VMs in pool '${PVE_POOL}':${RESET}" local idx=0 while [[ $idx -lt ${#managed_vms[@]} ]]; do echo -e " VMID ${managed_vms[$idx]}: ${managed_names[$idx]}" idx=$((idx + 1)) done echo "" # Confirm confirm "Destroy all ${#managed_vms[@]} managed VM(s) in pool '${PVE_POOL}'?" # Destroy each local destroyed=0 idx=0 while [[ $idx -lt ${#managed_vms[@]} ]]; do local vmid="${managed_vms[$idx]}" local name="${managed_names[$idx]}" step "Destroying VM '${name}' (VMID ${vmid})" if [[ "$DRY_RUN" == true ]]; then detail "[dry-run] Would stop and destroy VM ${vmid}" destroyed=$((destroyed + 1)) idx=$((idx + 1)) continue fi local status status=$(pve_vm_status "$vmid") || status="unknown" if [[ "$status" == "running" ]]; then detail "Stopping VM..." pve_stop_vm "$vmid" || true sleep 5 fi if pve_destroy_vm "$vmid"; then success "VM '${name}' (VMID ${vmid}) destroyed" destroyed=$((destroyed + 1)) else error "Failed to destroy VM '${name}' (VMID ${vmid})" fi idx=$((idx + 1)) done echo "" success "Pool cleanup complete: ${destroyed} VM(s) destroyed" } # --------------------------------------------------------------------------- # Reconcile existing deployments # --------------------------------------------------------------------------- # Check if VMs from config already exist and offer interactive options. # Only called for PHASE=all, non-cleanup, non-dry-run runs. # Returns 0 to continue with fresh deploy, or exits. reconcile_existing() { local count=${#VM_NAMES[@]} local existing=0 local running=0 local installed=0 local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local vmid="${VM_VMIDS[$i]}" if pve_vm_exists "$vmid"; then existing=$((existing + 1)) local status status=$(pve_vm_status "$vmid") || status="unknown" if [[ "$status" == "running" ]]; then running=$((running + 1)) fi local has_media=false pve_vm_has_install_media "$vmid" && has_media=true if [[ "$has_media" == false ]]; then installed=$((installed + 1)) fi fi i=$((i + 1)) done # No existing VMs — fresh deploy, proceed normally if [[ $existing -eq 0 ]]; then return 0 fi # Count of VMs we're deploying (respecting filter) local target=0 for name in "${VM_NAMES[@]}"; do if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then continue fi target=$((target + 1)) done echo "" echo -e "${BOLD}Existing deployment detected${RESET}" echo -e " ${existing}/${target} VMs exist (${running} running, ${installed} installed)" echo "" if [[ "$existing" -lt "$target" ]]; then local incomplete=$((target - existing)) echo -e " ${YELLOW}Partial deployment:${RESET} ${incomplete} VM(s) still need to be created/installed" fi # With --yes, default to safe option (post-deploy checks) if [[ "$YES" == true ]]; then info "Existing VMs detected with --yes; running post-deployment checks" echo "" phase_status phase_post_deploy exit 0 fi echo -e " Options:" echo -e " ${BOLD}1)${RESET} Run status checks and post-deployment verification" echo -e " ${BOLD}2)${RESET} Continue deployment (skip existing, install incomplete VMs)" echo -e " ${BOLD}3)${RESET} Destroy and redeploy cleanly" echo -e " ${BOLD}4)${RESET} Abort" echo "" echo -n -e " ${BOLD}Choose [1/2/3/4]: ${RESET}" local choice read -r choice case "$choice" in 1) phase_status phase_post_deploy exit 0 ;; 2) info "Continuing deployment (existing VMs will be skipped)" return 0 ;; 3) warn "Destroying existing VMs before redeployment" do_cleanup echo "" return 0 ;; 4|*) info "Aborted" exit 0 ;; esac } # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- main() { setup_colors parse_args "$@" validate_args # Handle doctor mode early (before config parsing) if [[ "$DOCTOR" == true ]]; then run_doctor return fi if [[ "$QUIET" != true ]]; then echo -e "${BOLD}${SCRIPT_NAME}${RESET} v${VERSION}" echo "" fi setup_workdir if [[ "$DRY_RUN" != true ]]; then check_dependencies fi echo "" # Always run validate first phase_validate # Handle cleanup-all mode (pool-wide, not config-specific VMs) if [[ "$CLEANUP_ALL" == true ]]; then do_cleanup_all if [[ "$DEEP_CLEANUP" == true ]]; then echo "" cleanup_storage cleanup_remotes cleanup_local_cache fi return fi # Handle cleanup mode if [[ "$CLEANUP" == true ]]; then confirm "Destroy VMs listed above?" do_cleanup if [[ "$DEEP_CLEANUP" == true ]]; then echo "" cleanup_storage cleanup_remotes cleanup_local_cache fi return fi # Reconcile existing deployments (interactive menu on re-runs) if [[ "$PHASE" == "all" ]] && [[ "$DRY_RUN" != true ]]; then reconcile_existing fi # Confirm before proceeding if [[ "$PHASE" == "all" ]]; then confirm "Proceed with deployment?" fi # Run requested phases case "$PHASE" in validate) # Already done above ;; prepare) phase_prepare ;; upload) phase_prepare phase_upload ;; create) phase_prepare phase_upload phase_create ;; install) phase_prepare phase_upload phase_create phase_install ;; status) phase_status phase_post_deploy ;; all) phase_prepare echo "" phase_upload echo "" phase_create echo "" phase_install echo "" phase_post_deploy ;; esac echo "" if [[ "$DRY_RUN" == true ]]; then info "Dry run complete -- no changes were made" else success "Done" fi } main "$@"