#!/usr/bin/env bash # incusos-proxmox - Declarative IncusOS deployment on Proxmox VE # # Reads a YAML configuration file defining VMs and deploys them on a Proxmox # host with IncusOS-correct settings (UEFI, TPM 2.0, VirtIO-SCSI, etc.). # Generates per-VM seed images, uploads ISOs, creates VMs, and boots them # through automated installation. # # Usage: incusos-proxmox [OPTIONS] CONFIG_FILE # Run 'incusos-proxmox --help' for full usage information. set -euo pipefail readonly VERSION="0.1.0" readonly SCRIPT_NAME="incusos-proxmox" readonly CDN_INDEX="https://images.linuxcontainers.org/os/index.json" readonly CDN_BASE="https://images.linuxcontainers.org/os" readonly MIN_DISK_GIB=50 readonly MIN_MEMORY_MIB=4096 readonly MANAGED_MARKER="[incusos-lab:managed]" # --------------------------------------------------------------------------- # Color and formatting # --------------------------------------------------------------------------- setup_colors() { if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[0;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' BOLD='\033[1m' DIM='\033[2m' RESET='\033[0m' else RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' RESET='' fi } info() { [[ "$QUIET" == true ]] && return; echo -e "${BLUE}[info]${RESET} $*"; } success() { [[ "$QUIET" == true ]] && return; echo -e "${GREEN}[ok]${RESET} $*"; } warn() { echo -e "${YELLOW}[warn]${RESET} $*" >&2; } error() { echo -e "${RED}[error]${RESET} $*" >&2; } step() { [[ "$QUIET" == true ]] && return; echo -e "${CYAN}[step]${RESET} ${BOLD}$*${RESET}"; } detail() { [[ "$VERBOSE" != true ]] && return; echo -e "${DIM} $*${RESET}"; } # --------------------------------------------------------------------------- # Help # --------------------------------------------------------------------------- usage() { cat < pvesh get /version ${BOLD}API method:${RESET} Requires an API token. Set up minimal privileges: pveum role add IncusOSDeploy --privs "VM.Allocate VM.Config.Disk \\ VM.Config.CPU VM.Config.Memory VM.Config.Network VM.Config.CDROM \\ VM.Config.Options VM.Config.HWType VM.PowerMgmt VM.Audit \\ Datastore.AllocateSpace Datastore.AllocateTemplate \\ Datastore.Audit SDN.Use" pveum user add automation@pve pveum aclmod / -user automation@pve -role IncusOSDeploy pveum user token add automation@pve deploy --privsep 0 For resource pool isolation (recommended): pveum pool add IncusLab --comment "IncusOS Lab VMs" pveum pool modify IncusLab --storage local-lvm,local pveum aclmod /pool/IncusLab -user automation@pve -role IncusOSDeploy Set the token secret: export PROXMOX_TOKEN_SECRET="" And in your config file: proxmox: method: api api_token_id: automation@pve!deploy ${BOLD}PREREQUISITES${RESET} - python3 with PyYAML (python3-yaml / py3-yaml) - jq (for JSON field access; falls back to python3) - ssh + scp (for SSH method) - curl (for API method and CDN downloads) - incusos-seed sibling script (auto-detected) - Bash 4+ (associative arrays) EOF exit 0 } # --------------------------------------------------------------------------- # Argument parsing # --------------------------------------------------------------------------- # Defaults PHASE="all" METHOD="" HOST_OVERRIDE="" LOCAL_ISO="" CERT_FILE="" NO_CERT=false VM_FILTER="" YES=false DOCTOR=false CLEANUP=false CLEANUP_ALL=false DEEP_CLEANUP=false FORCE_CLEANUP=false LAB_UP=false LAB_DOWN=false RESOURCES=false LIST_LABS=false DRY_RUN=false QUIET=false VERBOSE=false PROXMOX_CONFIG="" CONFIG_FILE="" parse_args() { while [[ $# -gt 0 ]]; do case "$1" in -p|--phase) PHASE="${2:?'--phase requires a value'}" shift 2 ;; -m|--method) METHOD="${2:?'--method requires a value'}" shift 2 ;; -H|--host) HOST_OVERRIDE="${2:?'--host requires a value'}" shift 2 ;; -i|--iso) LOCAL_ISO="${2:?'--iso requires a file path'}" shift 2 ;; -c|--cert) CERT_FILE="${2:?'--cert requires a file path'}" shift 2 ;; --no-cert) NO_CERT=true shift ;; --vm) VM_FILTER="${2:?'--vm requires a name'}" shift 2 ;; -y|--yes) YES=true shift ;; --status) PHASE="status" shift ;; --doctor) DOCTOR=true shift ;; --cleanup) CLEANUP=true shift ;; --deep) DEEP_CLEANUP=true shift ;; --cleanup-all) CLEANUP_ALL=true shift ;; --force-cleanup) CLEANUP=true FORCE_CLEANUP=true shift ;; --lab-up) LAB_UP=true shift ;; --lab-down) LAB_DOWN=true shift ;; --resources) RESOURCES=true shift ;; --labs) LIST_LABS=true shift ;; --proxmox) PROXMOX_CONFIG="${2:?'--proxmox requires a file path'}" shift 2 ;; --dry-run) DRY_RUN=true shift ;; -v|--verbose) VERBOSE=true shift ;; -q|--quiet) QUIET=true shift ;; -h|--help) usage ;; -V|--version) echo "${SCRIPT_NAME} v${VERSION}" exit 0 ;; -*) error "Unknown option: $1" echo "Run '${SCRIPT_NAME} --help' for usage information." >&2 exit 1 ;; *) if [[ -z "$CONFIG_FILE" ]]; then CONFIG_FILE="$1" else error "Unexpected argument: $1 (config file already set to ${CONFIG_FILE})" echo "Run '${SCRIPT_NAME} --help' for usage information." >&2 exit 1 fi shift ;; esac done } # --------------------------------------------------------------------------- # Validation # --------------------------------------------------------------------------- validate_args() { # Universal checks (apply regardless of mode) if [[ "$VERBOSE" == true ]] && [[ "$QUIET" == true ]]; then error "Cannot use both --verbose and --quiet" exit 1 fi # --doctor can run without a config file if [[ "$DOCTOR" == true ]] && [[ -z "$CONFIG_FILE" ]]; then return 0 fi # --cleanup-all, --labs, --resources can run without a config file (uses proxmox.yaml) if [[ "$CLEANUP_ALL" == true ]] && [[ -z "$CONFIG_FILE" ]]; then return 0 fi if [[ "$LIST_LABS" == true ]] && [[ -z "$CONFIG_FILE" ]]; then return 0 fi if [[ "$RESOURCES" == true ]] && [[ -z "$CONFIG_FILE" ]]; then return 0 fi if [[ -z "$CONFIG_FILE" ]]; then error "No configuration file specified" echo "Usage: ${SCRIPT_NAME} [OPTIONS] CONFIG_FILE" >&2 echo "Run '${SCRIPT_NAME} --help' for usage information." >&2 exit 1 fi if [[ ! -f "$CONFIG_FILE" ]]; then error "Configuration file not found: ${CONFIG_FILE}" exit 1 fi # --deep only makes sense with --cleanup or --cleanup-all if [[ "$DEEP_CLEANUP" == true ]] && [[ "$CLEANUP" != true ]] && [[ "$CLEANUP_ALL" != true ]]; then error "--deep can only be used with --cleanup or --cleanup-all" exit 1 fi case "$PHASE" in validate|prepare|upload|create|install|status|all) ;; *) error "Invalid phase: ${PHASE}" error "Supported: validate, prepare, upload, create, install, status, all" exit 1 ;; esac if [[ -n "$METHOD" ]]; then case "$METHOD" in ssh|api) ;; *) error "Invalid method: ${METHOD}" error "Supported: ssh, api" exit 1 ;; esac fi if [[ -n "$CERT_FILE" ]] && [[ "$NO_CERT" == true ]]; then error "Cannot use both --cert and --no-cert" exit 1 fi if [[ -n "$CERT_FILE" ]] && [[ ! -f "$CERT_FILE" ]]; then error "Certificate file not found: ${CERT_FILE}" exit 1 fi if [[ -n "$LOCAL_ISO" ]] && [[ ! -f "$LOCAL_ISO" ]]; then error "ISO file not found: ${LOCAL_ISO}" exit 1 fi } # --------------------------------------------------------------------------- # Dependency checks # --------------------------------------------------------------------------- SEED_TOOL="" check_dependencies() { step "Checking dependencies" local missing=false # python3 if ! command -v python3 &>/dev/null; then error "python3 is required but not found" error "Install Python 3 for your platform (https://www.python.org/)" missing=true else detail "python3: $(command -v python3)" fi # python3-yaml (optional -- basic YAML fallback built-in) if command -v python3 &>/dev/null; then if python3 -c "import yaml" 2>/dev/null; then detail "PyYAML: available" else detail "PyYAML: not found (using built-in basic parser)" fi fi # jq (optional, python3 fallback) if command -v jq &>/dev/null; then detail "jq: $(command -v jq)" else detail "jq: not found (using python3 fallback)" fi # curl if ! command -v curl &>/dev/null; then error "curl is required but not found" missing=true else detail "curl: $(command -v curl)" fi # ssh/scp (for SSH method) if [[ "${PVE_METHOD:-ssh}" == "ssh" ]]; then if ! command -v ssh &>/dev/null; then error "ssh is required for SSH method but not found" missing=true fi if ! command -v scp &>/dev/null; then error "scp is required for SSH method but not found" missing=true fi fi # incusos-seed sibling script local script_dir script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SEED_TOOL="${script_dir}/incusos-seed" if [[ ! -x "$SEED_TOOL" ]]; then error "incusos-seed not found at: ${SEED_TOOL}" error "The incusos-seed script must be in the same directory as ${SCRIPT_NAME}" missing=true else detail "incusos-seed: ${SEED_TOOL}" fi # bash version if [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then error "Bash 4+ is required (found ${BASH_VERSION})" missing=true else detail "bash: ${BASH_VERSION}" fi if [[ "$missing" == true ]]; then error "Missing dependencies -- cannot continue" exit 1 fi success "All dependencies satisfied" } # --------------------------------------------------------------------------- # Doctor mode # --------------------------------------------------------------------------- run_doctor() { echo -e "${BOLD}${SCRIPT_NAME} doctor${RESET}" echo "" local script_dir script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # -- Platform section -- echo -e " ${BOLD}Platform:${RESET}" echo -e " OS: $(uname -s) $(uname -m)" echo -e " Bash: ${BASH_VERSION}" echo "" # -- Tools section -- echo -e " ${BOLD}Tools:${RESET}" # incus client if command -v incus &>/dev/null; then local incus_ver incus_ver=$(incus version 2>/dev/null | sed -n 's/.*Client version: \([0-9.]*\).*/\1/p') [[ -z "$incus_ver" ]] && incus_ver="unknown" echo -e " incus client: ${GREEN}${incus_ver} ✓${RESET}" else echo -e " incus client: ${YELLOW}not installed${RESET}" fi # flasher-tool if command -v flasher-tool &>/dev/null; then local ft_ver ft_ver=$(flasher-tool --version 2>/dev/null | head -1 | sed 's/^flasher-tool version //' || echo "installed") if [[ -z "$ft_ver" ]] || [[ "$ft_ver" == *"unknown"* ]]; then ft_ver="dev (installed)" fi echo -e " flasher-tool: ${GREEN}${ft_ver} ✓${RESET}" else echo -e " flasher-tool: ${YELLOW}not installed${RESET}" fi # operations-center CLI if command -v operations-center &>/dev/null; then local oc_ver oc_ver=$(operations-center version 2>/dev/null | head -1 || echo "installed") echo -e " operations-center CLI: ${GREEN}${oc_ver} ✓${RESET}" else echo -e " operations-center CLI: ${DIM}not installed (optional)${RESET}" fi # python3 if command -v python3 &>/dev/null; then local py_ver py_ver=$(python3 --version 2>/dev/null | awk '{print $2}') echo -e " python3: ${GREEN}${py_ver} ✓${RESET}" else echo -e " python3: ${RED}not installed (required)${RESET}" fi # curl if command -v curl &>/dev/null; then local curl_ver curl_ver=$(curl --version 2>/dev/null | head -1 | awk '{print $2}') echo -e " curl: ${GREEN}${curl_ver} ✓${RESET}" else echo -e " curl: ${RED}not installed (required)${RESET}" fi # ISO creation tool (genisoimage on Linux, mkisofs on macOS) if command -v genisoimage &>/dev/null; then echo -e " genisoimage: ${GREEN}installed ✓${RESET}" elif command -v mkisofs &>/dev/null; then echo -e " mkisofs: ${GREEN}installed ✓${RESET} (cdrtools)" else case "$(uname -s)" in Darwin) echo -e " mkisofs: ${RED}not installed (brew install cdrtools)${RESET}" ;; *) echo -e " genisoimage: ${RED}not installed (required for seed ISOs)${RESET}" ;; esac fi # jq if command -v jq &>/dev/null; then local jq_ver jq_ver=$(jq --version 2>/dev/null | sed 's/^jq-//') echo -e " jq: ${GREEN}${jq_ver} ✓${RESET}" else echo -e " jq: ${YELLOW}not installed (python3 fallback used)${RESET}" fi # incusos-seed sibling if [[ -x "${script_dir}/incusos-seed" ]]; then echo -e " incusos-seed: ${GREEN}found ✓${RESET}" else echo -e " incusos-seed: ${RED}not found at ${script_dir}/incusos-seed${RESET}" fi # -- IncusOS section -- echo "" echo -e " ${BOLD}IncusOS:${RESET}" local cdn_latest="" local cdn_index cdn_index=$(mktemp) if curl -fsSL --connect-timeout 5 "$CDN_INDEX" -o "$cdn_index" 2>/dev/null; then if command -v jq &>/dev/null; then cdn_latest=$(jq -r '[.updates[] | select(.channels[] == "stable")] | sort_by(.version) | last | .version // empty' "$cdn_index" 2>/dev/null) else cdn_latest=$(python3 -c " import json data = json.load(open('${cdn_index}')) versions = [u['version'] for u in data.get('updates', []) if 'stable' in u.get('channels', [])] if versions: print(sorted(versions)[-1]) " 2>/dev/null) fi fi rm -f "$cdn_index" if [[ -n "$cdn_latest" ]]; then echo -e " Latest stable: ${cdn_latest}" else echo -e " Latest stable: ${YELLOW}could not reach CDN${RESET}" fi # Check local cache local cache_base="${XDG_CACHE_HOME:-${HOME}/.cache}/${SCRIPT_NAME}/iso-cache" if [[ -d "$cache_base" ]]; then local cached_iso cached_iso=$(ls -1 "${cache_base}"/IncusOS_*.iso 2>/dev/null | sort | tail -1) if [[ -n "$cached_iso" ]]; then local cached_name cached_name=$(basename "$cached_iso" .iso | sed 's/^IncusOS_//') if [[ -n "$cdn_latest" ]] && [[ "$cached_name" == "$cdn_latest" ]]; then echo -e " Cached ISO: ${GREEN}${cached_name} (up to date)${RESET}" else echo -e " Cached ISO: ${YELLOW}${cached_name}${RESET}" fi else echo -e " Cached ISO: ${DIM}none${RESET}" fi else echo -e " Cached ISO: ${DIM}none${RESET}" fi # -- Config section -- echo "" echo -e " ${BOLD}Config:${RESET}" if find_proxmox_config; then echo -e " proxmox.yaml: ${GREEN}${PROXMOX_CONFIG_RESOLVED} ✓${RESET}" else echo -e " proxmox.yaml: ${DIM}not found (optional)${RESET}" fi if [[ -n "$CONFIG_FILE" ]] && [[ -f "$CONFIG_FILE" ]]; then echo -e " Lab config: ${CONFIG_FILE}" fi # -- Proxmox section (if config file or proxmox.yaml provides connection info) -- echo "" if [[ -n "$CONFIG_FILE" ]] && [[ -f "$CONFIG_FILE" ]]; then echo -e " ${BOLD}Proxmox:${RESET}" # Minimal config parsing for doctor mode local doctor_workdir doctor_workdir=$(mktemp -d "${TMPDIR:-/tmp}/${SCRIPT_NAME}-doctor-XXXXXX") WORKDIR="$doctor_workdir" CONFIG_JSON="${doctor_workdir}/config.json" if python3 -c " import json, sys try: import yaml with open(sys.argv[1]) as f: data = yaml.safe_load(f) except ImportError: # Minimal inline parse -- just need proxmox section data = {} with open(sys.argv[1]) as f: import re lines = f.readlines() processed = [] for raw in lines: s = raw.rstrip() if not s or s.lstrip().startswith('#'): continue indent = len(s) - len(s.lstrip()) content = s.lstrip() processed.append((indent, content)) result = {} section = None for indent, content in processed: m = re.match(r'^([^:]+?):\s*(.*?)$', content) if m: key = m.group(1).strip() val = m.group(2).strip() if not val: section = key result[section] = {} elif section: if val.startswith('\"') or val.startswith(\"'\"): val = val[1:-1] result[section][key] = val data = result json.dump(data, sys.stdout, indent=2) " "$CONFIG_FILE" > "$CONFIG_JSON" 2>/dev/null; then local doc_host doc_method doc_token_id doc_pool doc_host=$(json_get "$CONFIG_JSON" ".proxmox.host" "") doc_method=$(json_get "$CONFIG_JSON" ".proxmox.method" "ssh") doc_token_id=$(json_get "$CONFIG_JSON" ".proxmox.api_token_id" "") doc_pool=$(json_get "$CONFIG_JSON" ".proxmox.pool" "") if [[ -z "$doc_host" ]]; then echo -e " Host: ${RED}not set in config${RESET}" else echo -e " Host: ${doc_host} (${doc_method})" # Connectivity check local pve_ok=false if [[ "$doc_method" == "api" ]]; then if [[ -n "$doc_token_id" ]] && [[ -n "${PROXMOX_TOKEN_SECRET:-}" ]]; then local auth="Authorization: PVEAPIToken=${doc_token_id}=${PROXMOX_TOKEN_SECRET}" local pve_ver pve_ver=$(curl -fsSk --connect-timeout 5 -H "$auth" \ "https://${doc_host}:8006/api2/json/version" 2>/dev/null | \ python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('version',''))" 2>/dev/null) || pve_ver="" if [[ -n "$pve_ver" ]]; then echo -e " Connectivity: ${GREEN}connected (PVE ${pve_ver}) ✓${RESET}" pve_ok=true else echo -e " Connectivity: ${RED}API connection failed${RESET}" fi else echo -e " Connectivity: ${YELLOW}API token not configured or PROXMOX_TOKEN_SECRET not set${RESET}" fi else local ssh_user ssh_user=$(json_get "$CONFIG_JSON" ".proxmox.ssh_user" "root") if ssh -o BatchMode=yes -o ConnectTimeout=5 "${ssh_user}@${doc_host}" "true" &>/dev/null; then echo -e " Connectivity: ${GREEN}SSH connected ✓${RESET}" pve_ok=true else echo -e " Connectivity: ${RED}SSH connection failed${RESET}" fi fi # Pool check (API only, if connected) if [[ -n "$doc_pool" ]]; then if [[ "$pve_ok" == true ]] && [[ "$doc_method" == "api" ]]; then local auth="Authorization: PVEAPIToken=${doc_token_id}=${PROXMOX_TOKEN_SECRET}" if curl -fsSk --connect-timeout 5 -H "$auth" \ "https://${doc_host}:8006/api2/json/pools/${doc_pool}" &>/dev/null; then echo -e " Pool: ${GREEN}${doc_pool} exists ✓${RESET}" else echo -e " Pool: ${RED}${doc_pool} not found${RESET}" fi else echo -e " Pool: ${doc_pool} (not verified)" fi fi fi else echo -e " ${RED}Failed to parse config file${RESET}" fi rm -rf "$doctor_workdir" elif [[ -n "$PROXMOX_CONFIG_RESOLVED" ]]; then # Use proxmox.yaml for connectivity check (no lab config provided) echo -e " ${BOLD}Proxmox:${RESET} ${DIM}(from proxmox.yaml)${RESET}" local doctor_workdir doctor_workdir=$(mktemp -d "${TMPDIR:-/tmp}/${SCRIPT_NAME}-doctor-XXXXXX") WORKDIR="$doctor_workdir" load_proxmox_config "$PROXMOX_CONFIG_RESOLVED" [[ -z "$PVE_NODE" ]] && PVE_NODE="pve" [[ -z "$PVE_METHOD" ]] && PVE_METHOD="ssh" [[ -z "$PVE_SSH_USER" ]] && PVE_SSH_USER="root" [[ -n "$METHOD" ]] && PVE_METHOD="$METHOD" [[ -n "$HOST_OVERRIDE" ]] && PVE_HOST="$HOST_OVERRIDE" if [[ -z "$PVE_HOST" ]]; then echo -e " Host: ${RED}not set in proxmox.yaml${RESET}" else echo -e " Host: ${PVE_HOST} (${PVE_METHOD})" local pve_ok=false if [[ "$PVE_METHOD" == "api" ]]; then if [[ -n "$PVE_API_TOKEN_ID" ]] && [[ -n "${PROXMOX_TOKEN_SECRET:-}" ]]; then local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}" local pve_ver pve_ver=$(curl -fsSk --connect-timeout 5 -H "$auth" \ "https://${PVE_HOST}:8006/api2/json/version" 2>/dev/null | \ python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('version',''))" 2>/dev/null) || pve_ver="" if [[ -n "$pve_ver" ]]; then echo -e " Connectivity: ${GREEN}connected (PVE ${pve_ver}) ✓${RESET}" pve_ok=true else echo -e " Connectivity: ${RED}API connection failed${RESET}" fi else echo -e " Connectivity: ${YELLOW}API token not configured or PROXMOX_TOKEN_SECRET not set${RESET}" fi else if ssh -o BatchMode=yes -o ConnectTimeout=5 "${PVE_SSH_USER}@${PVE_HOST}" "true" &>/dev/null; then echo -e " Connectivity: ${GREEN}SSH connected ✓${RESET}" pve_ok=true else echo -e " Connectivity: ${RED}SSH connection failed${RESET}" fi fi if [[ -n "$PVE_POOL" ]]; then if [[ "$pve_ok" == true ]] && [[ "$PVE_METHOD" == "api" ]]; then local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}" if curl -fsSk --connect-timeout 5 -H "$auth" \ "https://${PVE_HOST}:8006/api2/json/pools/${PVE_POOL}" &>/dev/null; then echo -e " Pool: ${GREEN}${PVE_POOL} exists ✓${RESET}" else echo -e " Pool: ${RED}${PVE_POOL} not found${RESET}" fi else echo -e " Pool: ${PVE_POOL} (not verified)" fi fi fi rm -rf "$doctor_workdir" else echo -e " ${BOLD}Proxmox:${RESET} ${DIM}(no config file or proxmox.yaml found)${RESET}" fi echo "" } # --------------------------------------------------------------------------- # Temporary directory management # --------------------------------------------------------------------------- WORKDIR="" CACHEDIR="" setup_workdir() { # Use persistent storage for the work directory -- the ISO download + # decompression can exceed 4 GiB, which overflows tmpfs on most systems. # Honour TMPDIR if explicitly set; otherwise use ~/.cache on real disk. local base="${TMPDIR:-${XDG_CACHE_HOME:-${HOME}/.cache}/${SCRIPT_NAME}}" mkdir -p "$base" # Persistent cache for ISO downloads (survives across runs) CACHEDIR="${base}/iso-cache" mkdir -p "$CACHEDIR" WORKDIR=$(mktemp -d "${base}/${SCRIPT_NAME}-XXXXXX") trap 'rm -rf "$WORKDIR"' EXIT } # --------------------------------------------------------------------------- # YAML parsing via python3 # --------------------------------------------------------------------------- # Convert YAML config to JSON, stored in a file for repeated access CONFIG_JSON="" parse_config() { step "Parsing configuration: ${CONFIG_FILE}" CONFIG_JSON="${WORKDIR}/config.json" # Try PyYAML first, fall back to a basic pure-Python parser if ! python3 -c " import json, sys, re def _parse_value(v): v = v.strip() # Strip inline comments (but not inside quotes) if not v.startswith(('\"', \"'\")): m = re.match(r'^(.*?)\s+#.*$', v) if m: v = m.group(1).rstrip() if not v: return None if (v.startswith('\"') and v.endswith('\"')) or (v.startswith(\"'\") and v.endswith(\"'\")): return v[1:-1] if v == 'true': return True if v == 'false': return False if v == 'null' or v == '~': return None try: return int(v) except ValueError: pass try: return float(v) except ValueError: pass return v def parse_yaml_basic(path): \"\"\"Minimal YAML parser for simple config files. Handles: mappings, lists of mappings, lists of scalars, inline comments. Does NOT handle: anchors, multi-line strings, flow style, merge keys.\"\"\" with open(path) as f: lines = f.readlines() # Pre-process: strip comments and blank lines, track indent + content processed = [] for raw in lines: s = raw.rstrip() if not s or s.lstrip().startswith('#'): continue indent = len(s) - len(s.lstrip()) content = s.lstrip() processed.append((indent, content)) def parse_block(idx, base_indent): \"\"\"Parse a block at the given indentation level, return (result, next_idx).\"\"\" if idx >= len(processed): return None, idx indent, content = processed[idx] # Detect if this block is a list or mapping if content.startswith('- '): return parse_list(idx, indent) else: return parse_mapping(idx, indent) def parse_mapping(idx, base_indent): result = {} while idx < len(processed): indent, content = processed[idx] if indent < base_indent: break if indent > base_indent: break if content.startswith('- '): break # Must be key: value m = re.match(r'^([^:]+?):\s*(.*?)$', content) if not m: idx += 1 continue key = m.group(1).strip() val_str = m.group(2).strip() # Strip inline comment from value if val_str and not val_str.startswith(('\"', \"'\")): cm = re.match(r'^(.*?)\s+#.*$', val_str) if cm: val_str = cm.group(1).rstrip() if val_str and not val_str.startswith('#'): result[key] = _parse_value(val_str) idx += 1 else: # Value is a nested block idx += 1 if idx < len(processed) and processed[idx][0] > base_indent: child_indent = processed[idx][0] child, idx = parse_block(idx, child_indent) result[key] = child if child is not None else {} else: result[key] = {} return result, idx def parse_list(idx, base_indent): result = [] while idx < len(processed): indent, content = processed[idx] if indent < base_indent: break if indent > base_indent: break if not content.startswith('- '): break item_str = content[2:].strip() # Check if it's a mapping item: '- key: value' m = re.match(r'^([^:]+?):\s*(.*?)$', item_str) if m: # It's a mapping -- parse this item and continuation lines obj = {} key = m.group(1).strip() val_str = m.group(2).strip() if val_str and not val_str.startswith('#'): if not val_str.startswith(('\"', \"'\")): cm = re.match(r'^(.*?)\s+#.*$', val_str) if cm: val_str = cm.group(1).rstrip() obj[key] = _parse_value(val_str) else: idx += 1 if idx < len(processed) and processed[idx][0] > indent: child_indent = processed[idx][0] child, idx = parse_block(idx, child_indent) obj[key] = child if child is not None else {} else: obj[key] = {} # Parse remaining keys at item indent + 2 item_indent = indent + 2 while idx < len(processed) and processed[idx][0] == item_indent: ic = processed[idx][1] if ic.startswith('- '): break im = re.match(r'^([^:]+?):\s*(.*?)$', ic) if im: ik = im.group(1).strip() iv = im.group(2).strip() if iv and not iv.startswith('#'): if not iv.startswith(('\"', \"'\")): icm = re.match(r'^(.*?)\s+#.*$', iv) if icm: iv = icm.group(1).rstrip() obj[ik] = _parse_value(iv) idx += 1 else: idx += 1 if idx < len(processed) and processed[idx][0] > item_indent: child_indent = processed[idx][0] child, idx = parse_block(idx, child_indent) obj[ik] = child if child is not None else {} else: obj[ik] = {} else: idx += 1 result.append(obj) continue idx += 1 # Parse remaining keys of this list item item_indent = indent + 2 while idx < len(processed) and processed[idx][0] >= item_indent: if processed[idx][0] == item_indent: ic = processed[idx][1] if ic.startswith('- '): break im = re.match(r'^([^:]+?):\s*(.*?)$', ic) if im: ik = im.group(1).strip() iv = im.group(2).strip() if iv and not iv.startswith('#'): if not iv.startswith(('\"', \"'\")): icm = re.match(r'^(.*?)\s+#.*$', iv) if icm: iv = icm.group(1).rstrip() obj[ik] = _parse_value(iv) idx += 1 else: idx += 1 if idx < len(processed) and processed[idx][0] > item_indent: child_indent = processed[idx][0] child, idx = parse_block(idx, child_indent) obj[ik] = child if child is not None else {} else: obj[ik] = {} else: idx += 1 else: # Deeper nesting handled by recursive calls above break result.append(obj) else: # Scalar list item result.append(_parse_value(item_str)) idx += 1 return result, idx result, _ = parse_block(0, 0) return result if result is not None else {} try: import yaml with open(sys.argv[1]) as f: data = yaml.safe_load(f) except ImportError: data = parse_yaml_basic(sys.argv[1]) json.dump(data, sys.stdout, indent=2) " "$CONFIG_FILE" > "$CONFIG_JSON" 2>/dev/null; then error "Failed to parse YAML configuration: ${CONFIG_FILE}" error "Check that the file is valid YAML" error "For complex YAML features, install PyYAML: pip3 install pyyaml" exit 1 fi success "Configuration parsed" } # JSON field accessor -- uses jq if available, falls back to python3 json_get() { local file="$1" local query="$2" local default="${3:-}" local result if command -v jq &>/dev/null; then result=$(jq -r "$query // empty" "$file" 2>/dev/null) || result="" else result=$(python3 -c " import json, sys data = json.load(open(sys.argv[1])) path = sys.argv[2] # Simple jq-like path traversal for common patterns if path.startswith('.'): path = path[1:] parts = path.replace('[]','').split('.') obj = data for p in parts: if p == '': continue if isinstance(obj, dict): obj = obj.get(p) elif isinstance(obj, list) and p.isdigit(): obj = obj[int(p)] else: obj = None if obj is None: break if obj is None: print('') elif isinstance(obj, bool): print('true' if obj else 'false') elif isinstance(obj, (dict, list)): json.dump(obj, sys.stdout) else: print(obj) " "$file" "$query" 2>/dev/null) || result="" fi if [[ -z "$result" ]] && [[ -n "$default" ]]; then echo "$default" else echo "$result" fi } # Get the number of VMs in config vm_count() { if command -v jq &>/dev/null; then jq -r '.vms | length' "$CONFIG_JSON" 2>/dev/null || echo "0" else python3 -c " import json data = json.load(open('$CONFIG_JSON')) print(len(data.get('vms', []))) " 2>/dev/null || echo "0" fi } # Get a field from a specific VM by index vm_get() { local index="$1" local field="$2" local default="${3:-}" local result if command -v jq &>/dev/null; then result=$(jq -r ".vms[${index}].${field} // empty" "$CONFIG_JSON" 2>/dev/null) || result="" else result=$(python3 -c " import json data = json.load(open('$CONFIG_JSON')) vm = data.get('vms', [])[${index}] val = vm.get('${field}') if val is None: print('') elif isinstance(val, bool): print('true' if val else 'false') else: print(val) " 2>/dev/null) || result="" fi if [[ -z "$result" ]] && [[ -n "$default" ]]; then echo "$default" else echo "$result" fi } # --------------------------------------------------------------------------- # Configuration loading # --------------------------------------------------------------------------- # Proxmox connection settings PVE_HOST="" PVE_NODE="" PVE_STORAGE="" PVE_ISO_STORAGE="" PVE_BRIDGE="" PVE_METHOD="" PVE_SSH_USER="" PVE_API_TOKEN_ID="" PVE_POOL="" # Default VM settings DEF_CORES="" DEF_MEMORY="" DEF_DISK="" DEF_CHANNEL="" DEF_START_VMID="" # VM arrays (indexed by position) declare -a VM_NAMES=() declare -a VM_APPS=() declare -a VM_APPLY_DEFAULTS=() declare -a VM_CORES=() declare -a VM_MEMORY=() declare -a VM_DISK=() declare -a VM_VMIDS=() declare -a VM_CLUSTERS=() declare -a VM_CLUSTER_ROLES=() # Path to the resolved proxmox connection config file (set by find_proxmox_config) PROXMOX_CONFIG_RESOLVED="" # Auto-discover proxmox.yaml: --proxmox flag > script directory > cwd find_proxmox_config() { if [[ -n "$PROXMOX_CONFIG" ]]; then if [[ ! -f "$PROXMOX_CONFIG" ]]; then error "Proxmox config file not found: ${PROXMOX_CONFIG}" exit 1 fi PROXMOX_CONFIG_RESOLVED="$PROXMOX_CONFIG" detail "Using proxmox config: ${PROXMOX_CONFIG_RESOLVED} (--proxmox)" return 0 fi local script_dir script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" if [[ -f "${script_dir}/proxmox.yaml" ]]; then PROXMOX_CONFIG_RESOLVED="${script_dir}/proxmox.yaml" detail "Using proxmox config: ${PROXMOX_CONFIG_RESOLVED} (auto-discovered)" return 0 fi if [[ -f "proxmox.yaml" ]]; then PROXMOX_CONFIG_RESOLVED="$(pwd)/proxmox.yaml" detail "Using proxmox config: ${PROXMOX_CONFIG_RESOLVED} (auto-discovered)" return 0 fi # No proxmox.yaml found -- caller decides if this is an error return 1 } # Parse proxmox.yaml (flat key: value structure) and set PVE_* globals as base values load_proxmox_config() { local pxconfig="$1" local px_json px_json=$(mktemp "${WORKDIR}/proxmox-config-XXXXXX.json") if ! python3 -c " import json, sys, re result = {} with open(sys.argv[1]) as f: for line in f: s = line.strip() if not s or s.startswith('#'): continue m = re.match(r'^([^:]+?):\s*(.*?)$', s) if m: key = m.group(1).strip() val = m.group(2).strip() # Strip inline comments if val and not val.startswith(('\"', \"'\")): cm = re.match(r'^(.*?)\s+#.*$', val) if cm: val = cm.group(1).rstrip() if (val.startswith('\"') and val.endswith('\"')) or (val.startswith(\"'\") and val.endswith(\"'\")): val = val[1:-1] result[key] = val json.dump(result, sys.stdout, indent=2) " "$pxconfig" > "$px_json" 2>/dev/null; then error "Failed to parse proxmox config: ${pxconfig}" exit 1 fi # Set base values from proxmox.yaml (lab config overlay and CLI flags applied later) local val val=$(json_get "$px_json" ".host" ""); [[ -n "$val" ]] && PVE_HOST="$val" val=$(json_get "$px_json" ".node" ""); [[ -n "$val" ]] && PVE_NODE="$val" val=$(json_get "$px_json" ".storage" ""); [[ -n "$val" ]] && PVE_STORAGE="$val" val=$(json_get "$px_json" ".iso_storage" ""); [[ -n "$val" ]] && PVE_ISO_STORAGE="$val" val=$(json_get "$px_json" ".bridge" ""); [[ -n "$val" ]] && PVE_BRIDGE="$val" val=$(json_get "$px_json" ".ssh_user" ""); [[ -n "$val" ]] && PVE_SSH_USER="$val" val=$(json_get "$px_json" ".api_token_id" ""); [[ -n "$val" ]] && PVE_API_TOKEN_ID="$val" val=$(json_get "$px_json" ".pool" ""); [[ -n "$val" ]] && PVE_POOL="$val" val=$(json_get "$px_json" ".method" ""); [[ -n "$val" ]] && PVE_METHOD="$val" rm -f "$px_json" } load_config() { step "Loading configuration values" # Merge priority: proxmox.yaml (base) → lab config proxmox: (overlay) → CLI flags # Step 1: proxmox.yaml base values were already loaded by the caller via # load_proxmox_config(). PVE_* globals may already be set. # Step 2: overlay from lab config proxmox: section (if present) local val val=$(json_get "$CONFIG_JSON" ".proxmox.host" ""); [[ -n "$val" ]] && PVE_HOST="$val" val=$(json_get "$CONFIG_JSON" ".proxmox.node" ""); [[ -n "$val" ]] && PVE_NODE="$val" val=$(json_get "$CONFIG_JSON" ".proxmox.storage" ""); [[ -n "$val" ]] && PVE_STORAGE="$val" val=$(json_get "$CONFIG_JSON" ".proxmox.iso_storage" ""); [[ -n "$val" ]] && PVE_ISO_STORAGE="$val" val=$(json_get "$CONFIG_JSON" ".proxmox.bridge" ""); [[ -n "$val" ]] && PVE_BRIDGE="$val" val=$(json_get "$CONFIG_JSON" ".proxmox.ssh_user" ""); [[ -n "$val" ]] && PVE_SSH_USER="$val" val=$(json_get "$CONFIG_JSON" ".proxmox.api_token_id" ""); [[ -n "$val" ]] && PVE_API_TOKEN_ID="$val" val=$(json_get "$CONFIG_JSON" ".proxmox.pool" ""); [[ -n "$val" ]] && PVE_POOL="$val" val=$(json_get "$CONFIG_JSON" ".proxmox.method" ""); [[ -n "$val" ]] && PVE_METHOD="$val" # Step 3: apply defaults for anything still unset [[ -z "$PVE_NODE" ]] && PVE_NODE="pve" [[ -z "$PVE_STORAGE" ]] && PVE_STORAGE="local-lvm" [[ -z "$PVE_ISO_STORAGE" ]] && PVE_ISO_STORAGE="local" [[ -z "$PVE_BRIDGE" ]] && PVE_BRIDGE="vmbr0" [[ -z "$PVE_SSH_USER" ]] && PVE_SSH_USER="root" [[ -z "$PVE_METHOD" ]] && PVE_METHOD="ssh" # Step 4: CLI flags override everything if [[ -n "$METHOD" ]]; then PVE_METHOD="$METHOD" fi if [[ -n "$HOST_OVERRIDE" ]]; then PVE_HOST="$HOST_OVERRIDE" fi # Default VM settings DEF_CORES=$(json_get "$CONFIG_JSON" ".defaults.cores" "4") DEF_MEMORY=$(json_get "$CONFIG_JSON" ".defaults.memory" "8192") DEF_DISK=$(json_get "$CONFIG_JSON" ".defaults.disk" "64") DEF_CHANNEL=$(json_get "$CONFIG_JSON" ".defaults.channel" "stable") DEF_START_VMID=$(json_get "$CONFIG_JSON" ".defaults.start_vmid" "200") # Load VMs local count count=$(vm_count) local i=0 while [[ $i -lt $count ]]; do VM_NAMES+=("$(vm_get "$i" "name" "")") VM_APPS+=("$(vm_get "$i" "app" "incus")") VM_APPLY_DEFAULTS+=("$(vm_get "$i" "apply_defaults" "false")") VM_CORES+=("$(vm_get "$i" "cores" "$DEF_CORES")") VM_MEMORY+=("$(vm_get "$i" "memory" "$DEF_MEMORY")") VM_DISK+=("$(vm_get "$i" "disk" "$DEF_DISK")") VM_VMIDS+=("$(vm_get "$i" "vmid" "")") VM_CLUSTERS+=("$(vm_get "$i" "cluster" "")") VM_CLUSTER_ROLES+=("$(vm_get "$i" "cluster_role" "")") i=$((i + 1)) done success "Loaded ${count} VM definition(s)" } # --------------------------------------------------------------------------- # Configuration validation # --------------------------------------------------------------------------- validate_config() { step "Validating configuration" local errors=0 # Required: Proxmox host if [[ -z "$PVE_HOST" ]]; then error "proxmox.host is required" errors=$((errors + 1)) fi # Method validation case "$PVE_METHOD" in ssh|api) ;; *) error "Invalid proxmox.method: ${PVE_METHOD} (must be ssh or api)" errors=$((errors + 1)) ;; esac # API method requires token if [[ "$PVE_METHOD" == "api" ]]; then if [[ -z "$PVE_API_TOKEN_ID" ]]; then error "proxmox.api_token_id is required for API method" error "Example: automation@pve!deploy" errors=$((errors + 1)) fi if [[ -z "${PROXMOX_TOKEN_SECRET:-}" ]]; then error "PROXMOX_TOKEN_SECRET environment variable is required for API method" error "Set it with: export PROXMOX_TOKEN_SECRET=\"\"" errors=$((errors + 1)) fi fi # Must have at least one VM local count=${#VM_NAMES[@]} if [[ $count -eq 0 ]]; then error "No VMs defined in configuration" errors=$((errors + 1)) fi # Validate each VM local seen_names="" local cluster_inits="" local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" local app="${VM_APPS[$i]}" local memory="${VM_MEMORY[$i]}" local disk="${VM_DISK[$i]}" local cluster="${VM_CLUSTERS[$i]}" local role="${VM_CLUSTER_ROLES[$i]}" # Name required if [[ -z "$name" ]]; then error "VM #$((i + 1)): name is required" errors=$((errors + 1)) i=$((i + 1)) continue fi # Unique names if echo "$seen_names" | grep -qw "$name"; then error "VM '${name}': duplicate VM name" errors=$((errors + 1)) fi seen_names="${seen_names} ${name}" # Valid app case "$app" in incus|operations-center) ;; *) error "VM '${name}': invalid app '${app}' (must be incus or operations-center)" errors=$((errors + 1)) ;; esac # Minimum memory if [[ "$memory" -lt "$MIN_MEMORY_MIB" ]] 2>/dev/null; then error "VM '${name}': memory ${memory} MiB is below minimum ${MIN_MEMORY_MIB} MiB" errors=$((errors + 1)) fi # Minimum disk if [[ "$disk" -lt "$MIN_DISK_GIB" ]] 2>/dev/null; then error "VM '${name}': disk ${disk} GiB is below minimum ${MIN_DISK_GIB} GiB" errors=$((errors + 1)) fi # Cluster role validation if [[ -n "$role" ]] && [[ -z "$cluster" ]]; then error "VM '${name}': cluster_role '${role}' set but no cluster name specified" errors=$((errors + 1)) fi if [[ -n "$role" ]]; then case "$role" in init|join) ;; *) error "VM '${name}': invalid cluster_role '${role}' (must be init or join)" errors=$((errors + 1)) ;; esac fi # Track cluster init nodes if [[ -n "$cluster" ]] && [[ "$role" == "init" ]]; then if echo "$cluster_inits" | grep -qw "$cluster"; then error "Cluster '${cluster}': multiple init nodes defined (only one allowed)" errors=$((errors + 1)) fi cluster_inits="${cluster_inits} ${cluster}" fi i=$((i + 1)) done # VM filter validation if [[ -n "$VM_FILTER" ]]; then local found=false for name in "${VM_NAMES[@]}"; do if [[ "$name" == "$VM_FILTER" ]]; then found=true break fi done if [[ "$found" != true ]]; then error "VM '${VM_FILTER}' not found in configuration" error "Available VMs: ${VM_NAMES[*]}" errors=$((errors + 1)) fi fi if [[ $errors -gt 0 ]]; then error "${errors} validation error(s) found" exit 1 fi success "Configuration valid" } # --------------------------------------------------------------------------- # Proxmox transport layer # --------------------------------------------------------------------------- # Execute a command on the Proxmox host pve_run() { local cmd="$1" if [[ "$DRY_RUN" == true ]]; then info "[dry-run] ${PVE_METHOD}: ${cmd}" return 0 fi case "$PVE_METHOD" in ssh) ssh -o BatchMode=yes -o ConnectTimeout=10 \ "${PVE_SSH_USER}@${PVE_HOST}" "$cmd" ;; api) error "API execution of arbitrary commands is not supported" error "Use specific API functions instead" return 1 ;; esac } # Execute a Proxmox API call (API method) # Usage: pve_api METHOD ENDPOINT [DATA] [json] # Pass "json" as 4th arg to send data as application/json instead of form-encoded. pve_api() { local method="$1" local endpoint="$2" local data="${3:-}" local content_type="${4:-form}" local url="https://${PVE_HOST}:8006/api2/json${endpoint}" local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET:-}" if [[ "$DRY_RUN" == true ]]; then info "[dry-run] API ${method} ${endpoint}" if [[ -n "$data" ]]; then detail "[dry-run] data: ${data}" fi return 0 fi local curl_args=(-fsSk -H "$auth") if [[ "$content_type" == "json" ]]; then curl_args+=(-H "Content-Type: application/json") fi case "$method" in GET) curl "${curl_args[@]}" "$url" ;; POST) if [[ -n "$data" ]]; then curl "${curl_args[@]}" -X POST -d "$data" "$url" else curl "${curl_args[@]}" -X POST "$url" fi ;; PUT) if [[ -n "$data" ]]; then curl "${curl_args[@]}" -X PUT -d "$data" "$url" else curl "${curl_args[@]}" -X PUT "$url" fi ;; DELETE) curl "${curl_args[@]}" -X DELETE "$url" ;; esac } # Upload a file to Proxmox ISO storage pve_upload() { local local_path="$1" local remote_filename="$2" if [[ "$DRY_RUN" == true ]]; then info "[dry-run] upload: ${local_path} -> ${PVE_ISO_STORAGE}:iso/${remote_filename}" return 0 fi case "$PVE_METHOD" in ssh) local iso_dir # Query the actual path of the ISO storage iso_dir=$(pve_run "pvesm path ${PVE_ISO_STORAGE}:iso/test 2>/dev/null" | sed 's|/test$||') || true if [[ -z "$iso_dir" ]]; then # Common default paths iso_dir="/var/lib/vz/template/iso" fi scp -o BatchMode=yes -o ConnectTimeout=10 \ "$local_path" "${PVE_SSH_USER}@${PVE_HOST}:${iso_dir}/${remote_filename}" ;; api) local upload_response upload_rc upload_response=$(curl -sSk \ -H "Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET:-}" \ -X POST \ -F "content=iso" \ -F "filename=@${local_path};filename=${remote_filename}" \ "https://${PVE_HOST}:8006/api2/json/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/upload" \ -w "\n%{http_code}" 2>&1) || true local http_code http_code=$(echo "$upload_response" | tail -1) local body body=$(echo "$upload_response" | sed '$d') if [[ "$http_code" -ge 200 ]] && [[ "$http_code" -lt 300 ]] 2>/dev/null; then return 0 else error "Upload failed (HTTP ${http_code})" # Extract error message from JSON response local api_error api_error=$(echo "$body" | python3 -c " import json, sys try: data = json.load(sys.stdin) errors = data.get('errors', {}) if errors: for k, v in errors.items(): print(f' {k}: {v}') elif data.get('message'): print(f' {data[\"message\"]}') else: print(json.dumps(data, indent=2)) except: print(sys.stdin.read() if hasattr(sys.stdin, 'read') else '') " 2>/dev/null) || api_error="$body" if [[ -n "$api_error" ]]; then error "Proxmox API response:" echo "$api_error" | while IFS= read -r line; do error " ${line}" done fi return 1 fi ;; esac } # Check if an ISO exists on Proxmox storage pve_iso_exists() { local filename="$1" if [[ "$DRY_RUN" == true ]]; then return 1 fi case "$PVE_METHOD" in ssh) pve_run "pvesm list ${PVE_ISO_STORAGE} --content iso 2>/dev/null" | grep -q "$filename" ;; api) pve_api GET "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content?content=iso" | \ grep -q "$filename" ;; esac } # Get list of existing VMIDs pve_list_vmids() { if [[ "$DRY_RUN" == true ]]; then echo "" return 0 fi case "$PVE_METHOD" in ssh) pve_run "qm list 2>/dev/null" | awk 'NR>1 {print $1}' || true ;; api) pve_api GET "/nodes/${PVE_NODE}/qemu" 2>/dev/null | \ python3 -c " import json, sys data = json.load(sys.stdin) for vm in data.get('data', []): print(vm.get('vmid', '')) " 2>/dev/null || true ;; esac } # Create a VM via qm create pve_create_vm() { local vmid="$1" local name="$2" local cores="$3" local memory="$4" local disk_gib="$5" local iso_filename="$6" local seed_filename="$7" local description description=$(build_vm_description) local cmd="qm create ${vmid}" cmd+=" --name ${name}" cmd+=" --description '${description}'" cmd+=" --ostype l26" cmd+=" --bios ovmf" cmd+=" --machine q35" cmd+=" --efidisk0 ${PVE_STORAGE}:0,efitype=4m,pre-enrolled-keys=0" cmd+=" --tpmstate0 ${PVE_STORAGE}:1,version=v2.0" cmd+=" --cpu host" cmd+=" --cores ${cores}" cmd+=" --memory ${memory}" cmd+=" --balloon 0" cmd+=" --scsihw virtio-scsi-pci" cmd+=" --scsi0 ${PVE_STORAGE}:${disk_gib},iothread=1" cmd+=" --ide2 ${PVE_ISO_STORAGE}:iso/${iso_filename},media=cdrom" cmd+=" --ide3 ${PVE_ISO_STORAGE}:iso/${seed_filename},media=cdrom" cmd+=" --net0 virtio,bridge=${PVE_BRIDGE}" cmd+=" --boot order=ide2\\;scsi0" cmd+=" --agent 1" if [[ -n "$PVE_POOL" ]]; then cmd+=" --pool ${PVE_POOL}" fi if [[ "$DRY_RUN" == true ]]; then info "[dry-run] ${cmd}" return 0 fi case "$PVE_METHOD" in ssh) pve_run "$cmd" ;; api) # Build JSON payload -- the Proxmox API requires JSON for VM # creation because comma-separated disk/net specs are misinterpreted # when sent as form-encoded data. local json_payload json_payload=$(python3 -c " import json payload = { 'vmid': ${vmid}, 'name': '${name}', 'description': '''${description}''', 'ostype': 'l26', 'bios': 'ovmf', 'machine': 'q35', 'efidisk0': '${PVE_STORAGE}:0,efitype=4m,pre-enrolled-keys=0', 'tpmstate0': '${PVE_STORAGE}:1,version=v2.0', 'cpu': 'host', 'cores': ${cores}, 'memory': ${memory}, 'balloon': 0, 'scsihw': 'virtio-scsi-pci', 'scsi0': '${PVE_STORAGE}:${disk_gib},iothread=1', 'ide2': '${PVE_ISO_STORAGE}:iso/${iso_filename},media=cdrom', 'ide3': '${PVE_ISO_STORAGE}:iso/${seed_filename},media=cdrom', 'net0': 'virtio,bridge=${PVE_BRIDGE}', 'boot': 'order=ide2;scsi0', 'agent': 1, } pool = '${PVE_POOL}' if pool: payload['pool'] = pool print(json.dumps(payload)) ") local create_response create_response=$(pve_api POST "/nodes/${PVE_NODE}/qemu" "$json_payload" json) # VM creation is async -- the API returns a UPID. Wait for the # task to finish before returning, otherwise the caller may try # to start or configure the VM while it's still being created. local upid upid=$(echo "$create_response" | python3 -c "import json,sys; print(json.load(sys.stdin).get('data',''))" 2>/dev/null) || upid="" if [[ -n "$upid" ]]; then local encoded_upid encoded_upid=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${upid}', safe=''))" 2>/dev/null) local wait_elapsed=0 while [[ $wait_elapsed -lt 60 ]]; do sleep 2 wait_elapsed=$((wait_elapsed + 2)) local task_status task_status=$(pve_api GET "/nodes/${PVE_NODE}/tasks/${encoded_upid}/status" 2>/dev/null | \ python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status',''))" 2>/dev/null) || task_status="" if [[ "$task_status" == "stopped" ]]; then break fi done fi ;; esac } # Start a VM pve_start_vm() { local vmid="$1" case "$PVE_METHOD" in ssh) pve_run "qm start ${vmid}" ;; api) pve_api POST "/nodes/${PVE_NODE}/qemu/${vmid}/status/start" >/dev/null ;; esac } # Stop a VM pve_stop_vm() { local vmid="$1" case "$PVE_METHOD" in ssh) pve_run "qm stop ${vmid}" ;; api) pve_api POST "/nodes/${PVE_NODE}/qemu/${vmid}/status/stop" >/dev/null ;; esac } # Destroy a VM pve_destroy_vm() { local vmid="$1" case "$PVE_METHOD" in ssh) pve_run "qm destroy ${vmid} --purge" ;; api) pve_api DELETE "/nodes/${PVE_NODE}/qemu/${vmid}?purge=1" >/dev/null ;; esac } # Set VM configuration pve_set_vm() { local vmid="$1" shift local opts="$*" case "$PVE_METHOD" in ssh) pve_run "qm set ${vmid} ${opts}" ;; api) # Convert CLI-style opts (--key value) to JSON. # JSON avoids form-encoding ambiguity with values containing = and ; # Duplicate keys (e.g. --delete ide2 --delete ide3) are merged with commas. # Shell escapes (e.g. \;) are stripped since the API doesn't need them. local json_payload json_payload=$(echo "$opts" | python3 -c " import json, sys tokens = sys.stdin.read().split() obj = {} i = 0 while i < len(tokens): t = tokens[i] if t.startswith('--'): key = t.lstrip('-') if i + 1 < len(tokens) and not tokens[i+1].startswith('--'): val = tokens[i+1].replace(chr(92), '') i += 2 else: val = '' i += 1 if key in obj and obj[key]: obj[key] = obj[key] + ',' + val else: obj[key] = val else: i += 1 print(json.dumps(obj)) " 2>/dev/null) pve_api PUT "/nodes/${PVE_NODE}/qemu/${vmid}/config" "$json_payload" json >/dev/null ;; esac } # Get VM status pve_vm_status() { local vmid="$1" case "$PVE_METHOD" in ssh) pve_run "qm status ${vmid}" | awk '{print $2}' ;; api) pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" | \ python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status','unknown'))" 2>/dev/null || echo "unknown" ;; esac } # Get VM uptime (seconds); returns 0 if stopped or unknown pve_vm_uptime() { local vmid="$1" case "$PVE_METHOD" in ssh) pve_run "qm status ${vmid} --verbose" | awk '/^uptime:/{print $2}' ;; api) pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" | \ python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('uptime',0))" 2>/dev/null || echo "0" ;; esac } # Get cumulative write bytes for scsi0 (from QEMU blockstat). # Returns 0 if unavailable. Only works with api method. pve_vm_disk_writes() { local vmid="$1" case "$PVE_METHOD" in ssh) echo "0" ;; api) pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" | \ python3 -c " import json,sys d = json.load(sys.stdin).get('data',{}) bs = d.get('blockstat',{}).get('scsi0',{}) print(bs.get('wr_bytes',0)) " 2>/dev/null || echo "0" ;; esac } # Check if a VMID exists pve_vm_exists() { local vmid="$1" if [[ "$DRY_RUN" == true ]]; then return 1 fi case "$PVE_METHOD" in ssh) pve_run "qm status ${vmid}" &>/dev/null ;; api) pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" &>/dev/null ;; esac } # Get VM's IP address (best-effort via QEMU guest agent) pve_vm_ip() { local vmid="$1" # Method 1: Try QEMU guest agent (works if agent is installed) local ip="" case "$PVE_METHOD" in ssh) ip=$(pve_run "qm guest cmd ${vmid} network-get-interfaces 2>/dev/null" | \ python3 -c " import json, sys try: data = json.load(sys.stdin) for iface in data: if iface.get('name') == 'lo': continue for addr in iface.get('ip-addresses', []): if addr.get('ip-address-type') == 'ipv4': print(addr['ip-address']) sys.exit(0) except: pass " 2>/dev/null) || true ;; api) ip=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/agent/network-get-interfaces" 2>/dev/null | \ python3 -c " import json, sys try: data = json.load(sys.stdin).get('data', {}).get('result', []) for iface in data: if iface.get('name') == 'lo': continue for addr in iface.get('ip-addresses', []): if addr.get('ip-address-type') == 'ipv4': print(addr['ip-address']) sys.exit(0) except: pass " 2>/dev/null) || true ;; esac if [[ -n "$ip" ]]; then echo "$ip" return 0 fi # Method 2: ARP-based lookup using VM's MAC address. # Works without guest agent (e.g. IncusOS which is immutable). # Get MAC from VM config, flush stale ARP, ping sweep, look up. local mac="" case "$PVE_METHOD" in ssh) mac=$(pve_run "qm config ${vmid}" 2>/dev/null | \ awk -F'=' '/^net0:/{print $1}' | grep -oiE '[0-9a-f:]{17}') || true ;; api) mac=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null | \ python3 -c " import json, sys, re try: cfg = json.load(sys.stdin).get('data', {}) net0 = cfg.get('net0', '') m = re.search(r'([0-9A-Fa-f:]{17})', net0) if m: print(m.group(1).lower()) except: pass " 2>/dev/null) || true ;; esac if [[ -z "$mac" ]]; then return 1 fi # Populate ARP table to find this MAC's IP address. # Strategy: first check existing ARP, then try broadcast ping, then # fall back to a parallel ping sweep of the local subnet. ip neigh flush nud stale 2>/dev/null || true local bridge_subnet bridge_subnet=$(ip -4 addr show | awk '/inet / && !/127\.0\.0/ {print $2; exit}') # Try broadcast ping first (fast, but many hosts ignore it) if [[ -n "$bridge_subnet" ]]; then local broadcast broadcast=$(python3 -c "import ipaddress; print(ipaddress.ip_interface('${bridge_subnet}').network.broadcast_address)" 2>/dev/null) || true if [[ -n "$broadcast" ]]; then ping -c 2 -W 1 -b "$broadcast" &>/dev/null || true sleep 1 fi fi # Check ARP table ip=$(ip neigh show | awk -v mac="$mac" 'tolower($0) ~ tolower(mac) && (/REACHABLE/ || /DELAY/ || /STALE/) {print $1; exit}') || true if [[ -n "$ip" ]] && ping -c 1 -W 1 "$ip" &>/dev/null; then echo "$ip" return 0 fi # Broadcast didn't work — parallel ping sweep of the subnet if [[ -n "$bridge_subnet" ]]; then local net_prefix net_prefix=$(python3 -c " import ipaddress net = ipaddress.ip_interface('${bridge_subnet}').network for h in net.hosts(): print(h) " 2>/dev/null) if [[ -n "$net_prefix" ]]; then # Ping all hosts in parallel (background, 1s timeout each) while IFS= read -r host_ip; do ping -c 1 -W 1 "$host_ip" &>/dev/null & done <<< "$net_prefix" wait sleep 1 # Re-check ARP table ip=$(ip neigh show | awk -v mac="$mac" 'tolower($0) ~ tolower(mac) && (/REACHABLE/ || /DELAY/) {print $1; exit}') || true if [[ -n "$ip" ]] && ping -c 1 -W 1 "$ip" &>/dev/null; then echo "$ip" return 0 fi fi fi return 1 } # --------------------------------------------------------------------------- # VM management marker # --------------------------------------------------------------------------- # Build the description string for a managed VM build_vm_description() { local config_basename config_basename=$(basename "$CONFIG_FILE") local timestamp timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ") local marker="${MANAGED_MARKER} config=${config_basename} created=${timestamp} by=${SCRIPT_NAME}/${VERSION}" if [[ -n "$PVE_POOL" ]]; then marker+=" pool=${PVE_POOL}" fi echo "$marker" } # Check if a VM has our management marker in its description pve_vm_is_managed() { local vmid="$1" if [[ "$DRY_RUN" == true ]]; then return 1 fi local config_output="" case "$PVE_METHOD" in ssh) config_output=$(pve_run "qm config ${vmid}" 2>/dev/null) || return 1 ;; api) config_output=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null) || return 1 ;; esac echo "$config_output" | grep -qF "$MANAGED_MARKER" } # Check if a VM still has install media (ide2) attached. # Returns 0 if install media is present, 1 if not (already installed). pve_vm_has_install_media() { local vmid="$1" if [[ "$DRY_RUN" == true ]]; then return 1 fi local config_output="" case "$PVE_METHOD" in ssh) config_output=$(pve_run "qm config ${vmid}" 2>/dev/null) || return 1 ;; api) config_output=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null) || return 1 ;; esac echo "$config_output" | grep -q "ide2" } # List all VMs as "vmid name" pairs (one per line) pve_list_vms() { if [[ "$DRY_RUN" == true ]]; then echo "" return 0 fi # When a resource pool is configured, list only pool members (type=qemu) if [[ -n "$PVE_POOL" ]]; then case "$PVE_METHOD" in ssh) pve_run "pvesh get /pools/${PVE_POOL} --output-format json 2>/dev/null" | \ python3 -c " import json, sys data = json.load(sys.stdin) for m in data.get('members', []): if m.get('type') == 'qemu': print('{} {}'.format(m.get('vmid', ''), m.get('name', ''))) " 2>/dev/null || true ;; api) pve_api GET "/pools/${PVE_POOL}" 2>/dev/null | \ python3 -c " import json, sys data = json.load(sys.stdin).get('data', {}) for m in data.get('members', []): if m.get('type') == 'qemu': print('{} {}'.format(m.get('vmid', ''), m.get('name', ''))) " 2>/dev/null || true ;; esac return 0 fi case "$PVE_METHOD" in ssh) pve_run "qm list 2>/dev/null" | awk 'NR>1 {print $1, $2}' || true ;; api) pve_api GET "/nodes/${PVE_NODE}/qemu" 2>/dev/null | \ python3 -c " import json, sys data = json.load(sys.stdin) for vm in data.get('data', []): print('{} {}'.format(vm.get('vmid', ''), vm.get('name', ''))) " 2>/dev/null || true ;; esac } # --------------------------------------------------------------------------- # Pre-flight checks # --------------------------------------------------------------------------- preflight_checks() { step "Running pre-flight checks" local errors=0 local warnings=0 # 1. Check that storage pools exist on Proxmox if [[ "$DRY_RUN" != true ]]; then case "$PVE_METHOD" in ssh) if ! pve_run "pvesm status 2>/dev/null" | awk 'NR>1 {print \$1}' | grep -qw "$PVE_STORAGE"; then error "Storage pool '${PVE_STORAGE}' not found on Proxmox" error "Available pools: $(pve_run "pvesm status 2>/dev/null" | awk 'NR>1 {print \$1}' | tr '\n' ' ')" error "Fix: update 'proxmox.storage' in your config file" errors=$((errors + 1)) fi if ! pve_run "pvesm status 2>/dev/null" | awk 'NR>1 {print \$1}' | grep -qw "$PVE_ISO_STORAGE"; then error "ISO storage '${PVE_ISO_STORAGE}' not found on Proxmox" error "Available pools: $(pve_run "pvesm status 2>/dev/null" | awk 'NR>1 {print \$1}' | tr '\n' ' ')" error "Fix: update 'proxmox.iso_storage' in your config file" errors=$((errors + 1)) fi ;; api) local storage_json storage_json=$(pve_api GET "/nodes/${PVE_NODE}/storage" 2>/dev/null) || storage_json="" local storage_list storage_list=$(echo "$storage_json" | python3 -c " import json, sys data = json.load(sys.stdin) for s in data.get('data', []): print(s.get('storage', '')) " 2>/dev/null) || storage_list="" if ! echo "$storage_list" | grep -qw "$PVE_STORAGE"; then error "Storage pool '${PVE_STORAGE}' not found on Proxmox" error "Available pools: $(echo "$storage_list" | tr '\n' ' ')" error "Fix: update 'proxmox.storage' in your config file" errors=$((errors + 1)) fi if ! echo "$storage_list" | grep -qw "$PVE_ISO_STORAGE"; then error "ISO storage '${PVE_ISO_STORAGE}' not found on Proxmox" error "Available pools: $(echo "$storage_list" | tr '\n' ' ')" error "Fix: update 'proxmox.iso_storage' in your config file" errors=$((errors + 1)) else # Check that ISO storage accepts 'iso' content type local iso_content iso_content=$(echo "$storage_json" | python3 -c " import json, sys data = json.load(sys.stdin) for s in data.get('data', []): if s.get('storage') == '${PVE_ISO_STORAGE}': print(s.get('content', '')) break " 2>/dev/null) || iso_content="" if [[ -n "$iso_content" ]] && ! echo "$iso_content" | grep -q "iso"; then error "Storage '${PVE_ISO_STORAGE}' does not accept ISO content (content types: ${iso_content})" error "Fix: enable 'iso' content on '${PVE_ISO_STORAGE}' in Proxmox, or use a different iso_storage" errors=$((errors + 1)) fi fi ;; esac # 2. Network bridge check (SSH only, warn-level) if [[ "$PVE_METHOD" == "ssh" ]]; then if ! pve_run "ip link show ${PVE_BRIDGE} 2>/dev/null" &>/dev/null; then warn "Network bridge '${PVE_BRIDGE}' not found on Proxmox" warn "VMs may fail to get network connectivity" warn "Fix: update 'proxmox.bridge' in your config file" warnings=$((warnings + 1)) fi fi # 3. ISO availability: check if already on Proxmox or CDN reachable if [[ -z "$LOCAL_ISO" ]]; then local any_iso_present=false local iso_list="" case "$PVE_METHOD" in ssh) iso_list=$(pve_run "pvesm list ${PVE_ISO_STORAGE} --content iso 2>/dev/null" 2>/dev/null) || iso_list="" ;; api) iso_list=$(pve_api GET "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content?content=iso" 2>/dev/null) || iso_list="" ;; esac if echo "$iso_list" | grep -q "IncusOS_"; then any_iso_present=true fi if [[ "$any_iso_present" != true ]]; then # No ISO on Proxmox -- check if CDN is reachable if ! curl -fsSL --connect-timeout 5 -o /dev/null "$CDN_INDEX" 2>/dev/null; then error "No IncusOS ISO found on Proxmox and CDN is unreachable: ${CDN_INDEX}" error "Fix: provide a local ISO with --iso FILE, or check internet connectivity" errors=$((errors + 1)) fi fi fi fi # 4. Resource pool check (if configured) if [[ -n "$PVE_POOL" ]] && [[ "$DRY_RUN" != true ]]; then local pool_ok=false case "$PVE_METHOD" in ssh) if pve_run "pvesh get /pools/${PVE_POOL} --output-format json 2>/dev/null" &>/dev/null; then pool_ok=true fi ;; api) if pve_api GET "/pools/${PVE_POOL}" &>/dev/null; then pool_ok=true fi ;; esac if [[ "$pool_ok" != true ]]; then error "Resource pool '${PVE_POOL}' not found on Proxmox" error "Create it with these commands on the Proxmox host:" error " pveum pool add ${PVE_POOL} --comment \"IncusOS Lab VMs\"" error " pveum pool modify ${PVE_POOL} --storage ${PVE_STORAGE},${PVE_ISO_STORAGE}" if [[ "$PVE_METHOD" == "api" ]] && [[ -n "$PVE_API_TOKEN_ID" ]]; then local api_user api_user=$(echo "$PVE_API_TOKEN_ID" | cut -d'!' -f1) error " pveum aclmod /pool/${PVE_POOL} -user ${api_user} -role IncusOSDeploy" fi errors=$((errors + 1)) else detail "Resource pool '${PVE_POOL}' exists" fi fi # 5. ISO creation tool for SEED_DATA ISO 9660 generation if ! command -v genisoimage &>/dev/null && ! command -v mkisofs &>/dev/null; then error "An ISO creation tool is required for SEED_DATA generation but not found" case "$(uname -s)" in Darwin) error "Install with: brew install cdrtools" ;; *) error "Install with: sudo apt install genisoimage" ;; esac errors=$((errors + 1)) fi if [[ $errors -gt 0 ]]; then error "${errors} pre-flight check(s) failed" exit 1 fi if [[ $warnings -gt 0 ]]; then success "Pre-flight checks passed with ${warnings} warning(s)" else success "Pre-flight checks passed" fi } # --------------------------------------------------------------------------- # Connectivity check # --------------------------------------------------------------------------- check_connectivity() { step "Checking Proxmox connectivity (${PVE_METHOD})" case "$PVE_METHOD" in ssh) if ! ssh -o BatchMode=yes -o ConnectTimeout=10 \ "${PVE_SSH_USER}@${PVE_HOST}" "pvesh get /version --output-format json" &>/dev/null; then error "Cannot connect to Proxmox host via SSH" error "Ensure key-based SSH auth is set up:" error " ssh-copy-id ${PVE_SSH_USER}@${PVE_HOST}" error "Test with: ssh ${PVE_SSH_USER}@${PVE_HOST} pvesh get /version" exit 1 fi local pve_version pve_version=$(ssh -o BatchMode=yes "${PVE_SSH_USER}@${PVE_HOST}" \ "pvesh get /version --output-format json" 2>/dev/null | \ python3 -c "import json,sys; print(json.load(sys.stdin).get('version','unknown'))" 2>/dev/null) || pve_version="unknown" success "Connected to ${PVE_HOST} (Proxmox VE ${pve_version})" ;; api) local url="https://${PVE_HOST}:8006/api2/json/version" local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET:-}" local response if ! response=$(curl -fsSk -H "$auth" "$url" 2>/dev/null); then error "Cannot connect to Proxmox API at https://${PVE_HOST}:8006" error "Check that:" error " 1. The host is reachable" error " 2. The API token ID is correct: ${PVE_API_TOKEN_ID}" error " 3. PROXMOX_TOKEN_SECRET is set" exit 1 fi local pve_version pve_version=$(echo "$response" | \ python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('version','unknown'))" 2>/dev/null) || pve_version="unknown" success "Connected to ${PVE_HOST} (Proxmox VE ${pve_version})" ;; esac } # --------------------------------------------------------------------------- # VMID allocation # --------------------------------------------------------------------------- allocate_vmids() { step "Allocating VMIDs" # Get existing VMs as "vmid name" pairs local vm_list vm_list=$(pve_list_vms) # Build lookup: existing IDs and name-to-VMID map local existing_ids="" declare -A existing_name_to_vmid if [[ -n "$vm_list" ]]; then while IFS=' ' read -r eid ename; do [[ -z "$eid" ]] && continue existing_ids="${existing_ids} ${eid}" if [[ -n "$ename" ]]; then existing_name_to_vmid["$ename"]="$eid" fi done <<< "$vm_list" fi local next_id="$DEF_START_VMID" local count=${#VM_NAMES[@]} local errors=0 local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" # Skip if filtered if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi # Check for name collision with existing Proxmox VMs if [[ -n "${existing_name_to_vmid[$name]+x}" ]]; then local collision_vmid="${existing_name_to_vmid[$name]}" if pve_vm_is_managed "$collision_vmid"; then warn "VM '${name}' already exists at VMID ${collision_vmid} (managed by ${SCRIPT_NAME})" warn "Use --cleanup to redeploy, or remove it manually" # Assign the existing VMID so cleanup/skip works correctly VM_VMIDS[$i]="$collision_vmid" i=$((i + 1)) continue else error "VM name '${name}' already exists at VMID ${collision_vmid} but is NOT managed by ${SCRIPT_NAME}" error "This VM was not created by this tool and will not be modified" error "Fix: rename the VM in your config file, or manually remove VMID ${collision_vmid}:" error " ssh ${PVE_SSH_USER}@${PVE_HOST} qm destroy ${collision_vmid} --purge" errors=$((errors + 1)) i=$((i + 1)) continue fi fi if [[ -n "${VM_VMIDS[$i]}" ]]; then # Explicit VMID detail "VM '${name}': VMID ${VM_VMIDS[$i]} (explicit)" else # Auto-assign, skipping existing while echo "$existing_ids" | grep -qw "$next_id" 2>/dev/null; do next_id=$((next_id + 1)) done VM_VMIDS[$i]="$next_id" detail "VM '${name}': VMID ${next_id} (auto-assigned)" next_id=$((next_id + 1)) fi i=$((i + 1)) done if [[ $errors -gt 0 ]]; then error "${errors} name collision(s) with unmanaged VMs -- cannot continue" exit 1 fi success "VMIDs allocated" } # --------------------------------------------------------------------------- # Certificate detection (reused pattern from sibling scripts) # --------------------------------------------------------------------------- CERT_FILE_RESOLVED="" detect_client_cert() { if [[ "$NO_CERT" == true ]]; then info "Skipping certificate injection (--no-cert)" return fi if [[ -n "$CERT_FILE" ]]; then if [[ ! -f "$CERT_FILE" ]]; then error "Certificate file not found: ${CERT_FILE}" exit 1 fi if ! grep -q "BEGIN CERTIFICATE" "$CERT_FILE"; then error "File does not appear to be a PEM certificate: ${CERT_FILE}" exit 1 fi CERT_FILE_RESOLVED="$CERT_FILE" success "Certificate: ${CERT_FILE_RESOLVED}" return fi step "Auto-detecting client certificate" local cert_paths=( "${HOME}/.config/incus/client.crt" "${HOME}/snap/incus/common/config/client.crt" "${HOME}/.config/lxc/client.crt" ) for path in "${cert_paths[@]}"; do if [[ -f "$path" ]] && grep -q "BEGIN CERTIFICATE" "$path"; then CERT_FILE_RESOLVED="$path" success "Certificate: ${CERT_FILE_RESOLVED}" return fi done # Try CLI fallback if command -v incus &>/dev/null; then local tmpfile="${WORKDIR}/client.crt" if incus remote get-client-certificate > "$tmpfile" 2>/dev/null; then CERT_FILE_RESOLVED="$tmpfile" success "Certificate obtained via Incus CLI" return fi fi # Check if any VM is ops-center (which requires a cert) local needs_cert=false for app in "${VM_APPS[@]}"; do if [[ "$app" == "operations-center" ]]; then needs_cert=true break fi done if [[ "$needs_cert" == true ]]; then error "No client certificate found" error "Operations Center requires at least one trusted certificate for access" error "" error "Options:" error " --cert FILE Provide a certificate file directly" error " Generate one: incus remote list (creates ~/.config/incus/client.crt)" error " --no-cert Skip certificate injection (you will be locked out!)" exit 1 fi warn "No client certificate found -- VMs will require manual trust setup" warn "Use --cert FILE to provide a certificate, or --no-cert to suppress this warning" } # --------------------------------------------------------------------------- # CDN operations (reused from incusos-iso) # --------------------------------------------------------------------------- LATEST_VERSION="" fetch_latest_version() { local channel="${DEF_CHANNEL:-stable}" step "Fetching latest IncusOS version (channel: ${channel})" local index_file="${WORKDIR}/cdn-index.json" if ! curl -fsSL "$CDN_INDEX" -o "$index_file" 2>/dev/null; then error "Failed to fetch CDN index from ${CDN_INDEX}" error "Check your internet connection and try again" exit 1 fi if command -v jq &>/dev/null; then LATEST_VERSION=$(jq -r \ --arg ch "$channel" \ '[.updates[] | select(.channels[] == $ch)] | sort_by(.version) | last | .version // empty' \ "$index_file") else LATEST_VERSION=$(python3 -c " import json data = json.load(open('${index_file}')) versions = [u['version'] for u in data.get('updates', []) if '${channel}' in u.get('channels', [])] if versions: print(sorted(versions)[-1]) " 2>/dev/null) fi if [[ -z "$LATEST_VERSION" ]]; then error "No version found for channel '${channel}'" exit 1 fi success "Latest IncusOS version: ${LATEST_VERSION}" } download_iso() { local version="$1" local output_path="$2" local iso_filename iso_filename=$(basename "$output_path") # Check persistent cache first local cached_iso="${CACHEDIR}/${iso_filename}" if [[ -f "$cached_iso" ]]; then success "ISO found in local cache: ${cached_iso}" cp "$cached_iso" "$output_path" return 0 fi local url="${CDN_BASE}/${version}/x86_64/IncusOS_${version}.iso.gz" local gz_path="${cached_iso}.gz" step "Downloading IncusOS ISO" detail "URL: ${url}" detail "Cache directory: ${CACHEDIR}" if ! curl -fL --progress-bar "$url" -o "$gz_path"; then error "Failed to download ISO from ${url}" rm -f "$gz_path" exit 1 fi step "Decompressing ISO" if ! gzip -d "$gz_path"; then local avail avail=$(df -h --output=avail "$(dirname "$gz_path")" 2>/dev/null | tail -1 | tr -d ' ') || avail="unknown" error "Failed to decompress ISO (${avail} available on disk)" error "The decompressed ISO requires several GiB of disk space" error "Free up space, or set TMPDIR to a larger filesystem:" error " TMPDIR=/path/with/space ${SCRIPT_NAME} ${CONFIG_FILE}" rm -f "$gz_path" exit 1 fi # Copy from cache to workdir cp "$cached_iso" "$output_path" success "ISO ready: ${output_path} (cached for future runs)" } # --------------------------------------------------------------------------- # Deployment plan summary # --------------------------------------------------------------------------- print_plan() { local count=${#VM_NAMES[@]} echo "" echo -e "${BOLD}Deployment Plan${RESET}" echo -e " Proxmox host: ${CYAN}${PVE_HOST}${RESET} (${PVE_METHOD})" echo -e " Node: ${PVE_NODE}" echo -e " Storage: ${PVE_STORAGE} (disk), ${PVE_ISO_STORAGE} (iso)" echo -e " Bridge: ${PVE_BRIDGE}" if [[ -n "$PVE_POOL" ]]; then echo -e " Pool: ${CYAN}${PVE_POOL}${RESET}" fi if [[ -n "$CERT_FILE_RESOLVED" ]]; then echo -e " Certificate: ${GREEN}${CERT_FILE_RESOLVED}${RESET}" elif [[ "$NO_CERT" == true ]]; then echo -e " Certificate: ${DIM}none (--no-cert)${RESET}" else echo -e " Certificate: ${YELLOW}not found${RESET}" fi echo "" echo -e " ${BOLD}VMs (${count}):${RESET}" local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" local skip="" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then skip=" ${DIM}(skipped)${RESET}" fi echo -e " ${CYAN}${name}${RESET}${skip}" echo -e " VMID: ${VM_VMIDS[$i]} App: ${VM_APPS[$i]} Defaults: ${VM_APPLY_DEFAULTS[$i]}" echo -e " CPU: ${VM_CORES[$i]} cores RAM: ${VM_MEMORY[$i]} MiB Disk: ${VM_DISK[$i]} GiB" if [[ -n "${VM_CLUSTERS[$i]}" ]]; then echo -e " Cluster: ${VM_CLUSTERS[$i]} (${VM_CLUSTER_ROLES[$i]})" fi i=$((i + 1)) done echo "" } # --------------------------------------------------------------------------- # Confirmation prompt # --------------------------------------------------------------------------- confirm() { local message="$1" if [[ "$YES" == true ]] || [[ "$DRY_RUN" == true ]]; then return 0 fi echo -n -e "${BOLD}${message} [y/N] ${RESET}" local reply read -r reply if [[ ! "$reply" =~ ^[Yy]$ ]]; then info "Aborted" exit 0 fi } # --------------------------------------------------------------------------- # Phase 1: Validate # --------------------------------------------------------------------------- phase_validate() { step "Phase 1: Validate" echo "" # Load proxmox.yaml base values (before lab config overlay) if find_proxmox_config; then load_proxmox_config "$PROXMOX_CONFIG_RESOLVED" fi parse_config load_config validate_config check_connectivity allocate_vmids preflight_checks detect_client_cert print_plan } # --------------------------------------------------------------------------- # Phase 2: Prepare (download ISO + generate seeds) # --------------------------------------------------------------------------- ISO_FILENAME="" phase_prepare() { step "Phase 2: Prepare" echo "" # Determine ISO if [[ -n "$LOCAL_ISO" ]]; then ISO_FILENAME=$(basename "$LOCAL_ISO") info "Using local ISO: ${LOCAL_ISO}" if [[ "$DRY_RUN" != true ]]; then cp "$LOCAL_ISO" "${WORKDIR}/${ISO_FILENAME}" fi elif [[ "$DRY_RUN" == true ]]; then ISO_FILENAME="IncusOS_latest.iso" info "[dry-run] Would fetch latest version from CDN and download ISO" else # Check if ISO already exists on Proxmox fetch_latest_version ISO_FILENAME="IncusOS_${LATEST_VERSION}.iso" if pve_iso_exists "$ISO_FILENAME"; then success "ISO already on Proxmox: ${ISO_FILENAME} (reusing from previous deployment)" else download_iso "$LATEST_VERSION" "${WORKDIR}/${ISO_FILENAME}" fi fi # Generate per-VM seed images step "Generating seed images" local count=${#VM_NAMES[@]} local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" # Skip if filtered if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local app="${VM_APPS[$i]}" local apply_defaults="${VM_APPLY_DEFAULTS[$i]}" local seed_output="${WORKDIR}/seed-${name}.iso" detail "Generating seed for '${name}' (${app}, defaults=${apply_defaults})" # Build incusos-seed command local seed_cmd=("$SEED_TOOL") seed_cmd+=(--format iso) seed_cmd+=(--app "$app") seed_cmd+=(--output "$seed_output") seed_cmd+=(--force-install) seed_cmd+=(--force-reboot) seed_cmd+=(--hostname "$name") seed_cmd+=(--quiet) if [[ "$apply_defaults" == "true" ]]; then seed_cmd+=(--defaults) fi if [[ -n "$CERT_FILE_RESOLVED" ]]; then seed_cmd+=(--cert "$CERT_FILE_RESOLVED") elif [[ "$NO_CERT" == true ]]; then seed_cmd+=(--no-cert) fi if [[ "$DRY_RUN" == true ]]; then info "[dry-run] ${seed_cmd[*]}" else if ! "${seed_cmd[@]}"; then error "Failed to generate seed for VM '${name}'" exit 1 fi fi i=$((i + 1)) done success "Seed images generated" } # --------------------------------------------------------------------------- # Phase 3: Upload # --------------------------------------------------------------------------- phase_upload() { step "Phase 3: Upload" echo "" # Upload ISO (if we downloaded it) local iso_path="${WORKDIR}/${ISO_FILENAME}" if [[ -f "$iso_path" ]]; then if pve_iso_exists "$ISO_FILENAME"; then success "ISO already on Proxmox: ${ISO_FILENAME}" else step "Uploading ISO: ${ISO_FILENAME}" if ! pve_upload "$iso_path" "$ISO_FILENAME"; then error "Failed to upload ISO to Proxmox" exit 1 fi success "ISO uploaded: ${ISO_FILENAME}" fi else detail "ISO already on Proxmox, skipping upload" fi # Upload seed images local count=${#VM_NAMES[@]} local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local seed_file="seed-${name}.iso" local seed_path="${WORKDIR}/${seed_file}" if [[ -f "$seed_path" ]] || [[ "$DRY_RUN" == true ]]; then step "Uploading seed: ${seed_file}" if ! pve_upload "$seed_path" "$seed_file"; then error "Failed to upload seed for VM '${name}'" exit 1 fi success "Seed uploaded: ${seed_file}" fi i=$((i + 1)) done success "All files uploaded" } # --------------------------------------------------------------------------- # Phase 4: Create VMs # --------------------------------------------------------------------------- phase_create() { step "Phase 4: Create VMs" echo "" local count=${#VM_NAMES[@]} local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local vmid="${VM_VMIDS[$i]}" local cores="${VM_CORES[$i]}" local memory="${VM_MEMORY[$i]}" local disk="${VM_DISK[$i]}" local seed_file="seed-${name}.iso" step "Creating VM '${name}' (VMID ${vmid})" # Check if VM already exists if pve_vm_exists "$vmid"; then if pve_vm_is_managed "$vmid"; then warn "VM ${vmid} ('${name}') already exists (managed) -- skipping" warn "Use --cleanup to redeploy" else warn "VM ${vmid} ('${name}') already exists (NOT managed by ${SCRIPT_NAME}) -- skipping" warn "Choose a different VMID or remove it manually:" warn " ssh ${PVE_SSH_USER}@${PVE_HOST} qm destroy ${vmid} --purge" fi i=$((i + 1)) continue fi detail "cores=${cores} memory=${memory}MiB disk=${disk}GiB" detail "bios=ovmf machine=q35 cpu=host scsihw=virtio-scsi-pci" detail "efidisk: pre-enrolled-keys=0, tpm: v2.0, balloon: 0" detail "iso=${ISO_FILENAME} seed=${seed_file}" if ! pve_create_vm "$vmid" "$name" "$cores" "$memory" "$disk" \ "$ISO_FILENAME" "$seed_file"; then error "Failed to create VM '${name}' (VMID ${vmid})" exit 1 fi success "VM '${name}' created (VMID ${vmid})" i=$((i + 1)) done success "All VMs created" } # --------------------------------------------------------------------------- # Remote registration # --------------------------------------------------------------------------- # Register an incus remote for a deployed VM. # Handles stale remotes: if a remote exists but points to a different IP, # it is removed and re-added with the correct IP (DHCP may reassign addresses # between deploys). # Usage: register_remote register_remote() { local name="$1" local ip="$2" # Only register if incus client is available and a cert was injected if ! command -v incus &>/dev/null; then return fi if [[ -z "$CERT_FILE_RESOLVED" ]]; then return fi # Check if remote already exists local existing_url="" existing_url=$(incus remote list --format csv 2>/dev/null | \ awk -F',' -v name="$name" '$1 == name {print $2; exit}') || true if [[ -n "$existing_url" ]]; then # Remote exists -- check if the IP matches if echo "$existing_url" | grep -q "${ip}"; then detail "Remote '${name}' already configured at ${ip}" return fi # Stale remote -- IP changed (DHCP reassignment between deploys) if [[ "$DRY_RUN" == true ]]; then info "[dry-run] Would update stale remote '${name}': ${existing_url} -> ${ip}" return fi warn "Remote '${name}' points to ${existing_url} but VM is at ${ip} -- updating" # Switch away first -- can't remove the current/default remote local current_remote="" current_remote=$(incus remote get-default 2>/dev/null) || true if [[ "$current_remote" == "$name" ]]; then incus remote switch local 2>/dev/null || true fi incus remote remove "$name" 2>/dev/null || true fi if [[ "$DRY_RUN" == true ]]; then info "[dry-run] Would add incus remote '${name}' at ${ip}" return fi # Wait for port 8443 to become reachable (up to 30s) local attempt=0 while [[ $attempt -lt 6 ]]; do if curl -sk --connect-timeout 3 "https://${ip}:8443" &>/dev/null; then break fi sleep 5 attempt=$((attempt + 1)) done if incus remote add "$name" "$ip" --accept-certificate --quiet 2>/dev/null; then success "Remote '${name}' added (${ip})" else warn "Could not add remote '${name}' at ${ip} -- add manually:" warn " incus remote add ${name} ${ip} --accept-certificate" fi } # --------------------------------------------------------------------------- # Phase 5: Install # --------------------------------------------------------------------------- phase_install() { step "Phase 5: Install" echo "" local count=${#VM_NAMES[@]} local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local vmid="${VM_VMIDS[$i]}" if [[ "$DRY_RUN" == true ]]; then step "Starting VM '${name}' (VMID ${vmid})" info "[dry-run] Would start VM, swap boot order, wait for install, clean up" i=$((i + 1)) continue fi # Idempotency: check current VM state before acting if pve_vm_exists "$vmid"; then local cur_status cur_status=$(pve_vm_status "$vmid") || cur_status="unknown" local has_media=true pve_vm_has_install_media "$vmid" || has_media=false if [[ "$cur_status" == "running" ]] && [[ "$has_media" == false ]]; then # Already installed and running — detect IP, register remote, skip success "VM '${name}' (VMID ${vmid}) is already installed and running" local ip ip=$(pve_vm_ip "$vmid") || ip="" if [[ -n "$ip" ]]; then detail "IP: ${ip}" register_remote "$name" "$ip" fi i=$((i + 1)) continue fi if [[ "$cur_status" == "stopped" ]] && [[ "$has_media" == false ]]; then # Installed but stopped — start from disk step "VM '${name}' is installed but stopped — starting from disk" if pve_start_vm "$vmid"; then sleep 30 local ip ip=$(pve_vm_ip "$vmid") || ip="" if [[ -n "$ip" ]]; then success "VM '${name}' started at ${ip}" register_remote "$name" "$ip" else success "VM '${name}' started from disk" fi else error "Failed to start VM '${name}' from disk" fi i=$((i + 1)) continue fi if [[ "$cur_status" == "running" ]] && [[ "$has_media" == true ]]; then # Mid-install or stuck — fall through to blockstat monitoring info "VM '${name}' is running with install media — monitoring installation" fi # stopped + has_media → needs install, fall through fi step "Starting VM '${name}' (VMID ${vmid})" # 1. Start VM (boots from ISO, reads SEED_DATA, installs) if ! pve_start_vm "$vmid"; then error "Failed to start VM '${name}' (VMID ${vmid})" i=$((i + 1)) continue fi success "VM '${name}' started" # 2. Wait for installation to complete by monitoring disk writes # # Install flow: boot ISO -> read SEED_DATA -> install to disk -> reboot # (force_reboot in seed). A guest-level reboot does NOT reset the QEMU # VM uptime, so we cannot detect reboot that way. Instead we monitor # scsi0 write bytes via blockstat: # - During install: wr_bytes increases steadily # - After install + reboot: wr_bytes stops (VM is idle at post-reboot # screen or "install media detected" error) # # When wr_bytes has been stable for STABLE_COUNT consecutive polls, # installation is complete. We then stop the VM, remove install media, # and start cleanly from disk. step "Waiting for '${name}' to complete installation" info "Monitoring disk write activity (polling every 5s, timeout: 10 minutes)" local timeout=600 local interval=5 local elapsed=0 local install_done=false local prev_wr_bytes=0 local writes_started=false local stable_count=0 local STABLE_THRESHOLD=3 # 3 consecutive polls with no change = 15s idle while [[ $elapsed -lt $timeout ]]; do sleep "$interval" elapsed=$((elapsed + interval)) local status status=$(pve_vm_status "$vmid") || status="unknown" case "$status" in stopped) detail "VM stopped [${elapsed}s]" install_done=true break ;; running) local wr_bytes wr_bytes=$(pve_vm_disk_writes "$vmid") || wr_bytes=0 if [[ $wr_bytes -gt $prev_wr_bytes ]]; then # Disk writes are happening -- install in progress writes_started=true stable_count=0 local wr_mb=$(( wr_bytes / 1048576 )) # Show progress every 15s if [[ $((elapsed % 15)) -eq 0 ]]; then info "Installing... (${wr_mb} MiB written) [${elapsed}s]" fi elif [[ "$writes_started" == true ]]; then # Writes have stopped after being active stable_count=$((stable_count + 1)) if [[ $stable_count -ge $STABLE_THRESHOLD ]]; then local wr_mb=$(( wr_bytes / 1048576 )) info "Disk activity stopped (${wr_mb} MiB written total) [${elapsed}s]" install_done=true break fi fi prev_wr_bytes=$wr_bytes ;; *) detail "VM status: ${status} [${elapsed}s]" ;; esac done if [[ "$install_done" != true ]]; then warn "VM '${name}' did not complete installation within ${timeout}s" warn "Check the VM console on Proxmox for details" i=$((i + 1)) continue fi # 3. Stop the VM (it's either at "remove install media" prompt or # post-reboot "install media detected" error -- either way idle). local cur_status cur_status=$(pve_vm_status "$vmid") || cur_status="unknown" if [[ "$cur_status" == "running" ]]; then info "Stopping VM to remove install media..." pve_stop_vm "$vmid" || true sleep 5 fi # 4. Remove install media and set clean boot order step "Removing install media from '${name}'" pve_set_vm "$vmid" "--delete ide2 --delete ide3 --boot order=scsi0" || true success "Install media detached, boot order set to disk-only" # 5. Start VM from disk step "Starting '${name}' from disk" if ! pve_start_vm "$vmid"; then error "Failed to start VM '${name}' from disk" i=$((i + 1)) continue fi # 5. Wait for services to come up info "Waiting for services to start..." sleep 30 # 6. Try to detect IP local ip ip=$(pve_vm_ip "$vmid") || ip="" echo "" if [[ -n "$ip" ]]; then success "VM '${name}' installed and running at ${ip}" register_remote "$name" "$ip" else success "VM '${name}' installed and running" info "Check Proxmox console for IP address" fi echo "" i=$((i + 1)) done success "Installation phase complete" } # --------------------------------------------------------------------------- # Lab lifecycle (--lab-up, --lab-down) # --------------------------------------------------------------------------- phase_lab_down() { step "Stopping lab VMs" echo "" local count=${#VM_NAMES[@]} local stopped=0 local skipped=0 local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" local vmid="${VM_VMIDS[$i]}" if [[ "$DRY_RUN" == true ]]; then info "Would stop ${name} (VMID ${vmid})" i=$((i + 1)) continue fi if ! pve_vm_exists "$vmid"; then warn "${name} (VMID ${vmid}): not found -- skipping" skipped=$((skipped + 1)) i=$((i + 1)) continue fi local status status=$(pve_vm_status "$vmid") || status="unknown" if [[ "$status" == "running" ]]; then info "Stopping ${name} (VMID ${vmid})..." if pve_stop_vm "$vmid"; then success "${name} stopped" stopped=$((stopped + 1)) else error "Failed to stop ${name}" fi else detail "${name}: already ${status}" skipped=$((skipped + 1)) fi i=$((i + 1)) done echo "" if [[ "$DRY_RUN" == true ]]; then info "Dry run -- no VMs were stopped" else success "Lab down: ${stopped} stopped, ${skipped} skipped" fi } phase_lab_up() { step "Starting lab VMs" echo "" local count=${#VM_NAMES[@]} local started=0 local skipped=0 local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" local vmid="${VM_VMIDS[$i]}" if [[ "$DRY_RUN" == true ]]; then info "Would start ${name} (VMID ${vmid})" i=$((i + 1)) continue fi if ! pve_vm_exists "$vmid"; then warn "${name} (VMID ${vmid}): not found -- skipping" skipped=$((skipped + 1)) i=$((i + 1)) continue fi local status status=$(pve_vm_status "$vmid") || status="unknown" if [[ "$status" == "stopped" ]]; then # Safety check: refuse to start VMs with install media still attached if pve_vm_has_install_media "$vmid"; then warn "${name}: install media still attached -- skipping (run full deploy instead)" skipped=$((skipped + 1)) i=$((i + 1)) continue fi info "Starting ${name} (VMID ${vmid})..." if pve_start_vm "$vmid"; then success "${name} started" started=$((started + 1)) else error "Failed to start ${name}" fi else detail "${name}: already ${status}" skipped=$((skipped + 1)) fi i=$((i + 1)) done if [[ "$DRY_RUN" == true ]]; then echo "" info "Dry run -- no VMs were started" return fi echo "" success "Lab up: ${started} started, ${skipped} skipped" if [[ $started -gt 0 ]]; then info "Waiting 30s for services to start..." sleep 30 echo "" phase_status fi } # --------------------------------------------------------------------------- # Resource and lab inventory (--resources, --labs) # --------------------------------------------------------------------------- do_resources() { echo -e "${BOLD}${SCRIPT_NAME} resources${RESET}" echo "" echo -e " ${BOLD}Proxmox Host:${RESET} ${PVE_HOST} (node: ${PVE_NODE})" echo "" # Host resources (API only -- SSH would need parsing pvesh output) if [[ "$PVE_METHOD" != "api" ]]; then warn "Resource details require API method (current: ${PVE_METHOD})" warn "Use --method api or set method: api in proxmox.yaml" return fi # Host memory and CPU local node_status node_status=$(pve_api GET "/nodes/${PVE_NODE}/status" 2>/dev/null) || node_status="" if [[ -n "$node_status" ]]; then python3 -c " import json, sys data = json.loads('''${node_status}''').get('data', {}) mem = data.get('memory', {}) cpu_count = data.get('cpuinfo', {}).get('cpus', '?') cpu_model = data.get('cpuinfo', {}).get('model', 'unknown') mem_used = mem.get('used', 0) / (1024**3) mem_total = mem.get('total', 0) / (1024**3) mem_free = mem_total - mem_used mem_pct = (mem_used / mem_total * 100) if mem_total > 0 else 0 print(f' RAM: {mem_used:.1f} GiB / {mem_total:.1f} GiB used ({mem_pct:.0f}%), {mem_free:.1f} GiB free') print(f' CPUs: {cpu_count} cores ({cpu_model})') uptime_s = data.get('uptime', 0) days = uptime_s // 86400 hours = (uptime_s % 86400) // 3600 print(f' Uptime: {days}d {hours}h') " 2>/dev/null || warn "Failed to parse node status" fi # Storage echo "" echo -e " ${BOLD}Storage:${RESET}" local storage_list storage_list=$(pve_api GET "/nodes/${PVE_NODE}/storage" 2>/dev/null) || storage_list="" if [[ -n "$storage_list" ]]; then python3 -c " import json, sys data = json.loads('''${storage_list}''').get('data', []) for s in sorted(data, key=lambda x: x.get('storage', '')): name = s.get('storage', '?') total = s.get('total', 0) / (1024**3) used = s.get('used', 0) / (1024**3) avail = s.get('avail', 0) / (1024**3) pct = (used / total * 100) if total > 0 else 0 stype = s.get('type', '?') content = s.get('content', '?') if total > 0: print(f' {name:<16} {stype:<12} {used:.1f}G / {total:.1f}G ({pct:.0f}%) -- {content}') else: print(f' {name:<16} {stype:<12} (no capacity info) -- {content}') " 2>/dev/null || warn "Failed to parse storage list" fi # Per-pool VM summary (if pool is configured) if [[ -n "$PVE_POOL" ]]; then echo "" echo -e " ${BOLD}Pool: ${PVE_POOL}${RESET}" local pool_data pool_data=$(pve_api GET "/pools/${PVE_POOL}" 2>/dev/null) || pool_data="" if [[ -n "$pool_data" ]]; then python3 -c " import json, sys data = json.loads('''${pool_data}''').get('data', {}) members = data.get('members', []) vms = [m for m in members if m.get('type') == 'qemu'] running = sum(1 for v in vms if v.get('status') == 'running') stopped = sum(1 for v in vms if v.get('status') == 'stopped') total_mem = sum(v.get('maxmem', 0) for v in vms) / (1024**3) total_disk = sum(v.get('maxdisk', 0) for v in vms) / (1024**3) print(f' VMs: {len(vms)} total ({running} running, {stopped} stopped)') print(f' Allocated: {total_mem:.1f} GiB RAM, {total_disk:.0f} GiB disk') " 2>/dev/null || warn "Failed to parse pool data" fi fi echo "" } do_list_labs() { echo -e "${BOLD}${SCRIPT_NAME} labs${RESET}" echo "" if [[ "$PVE_METHOD" != "api" ]]; then warn "Lab listing requires API method (current: ${PVE_METHOD})" warn "Use --method api or set method: api in proxmox.yaml" return fi # Get all managed VMs and group by config file local all_vms if [[ -n "$PVE_POOL" ]]; then all_vms=$(pve_api GET "/pools/${PVE_POOL}" 2>/dev/null) || all_vms="" else all_vms=$(pve_api GET "/nodes/${PVE_NODE}/qemu" 2>/dev/null) || all_vms="" fi if [[ -z "$all_vms" ]]; then info "No VMs found" return fi # Collect managed VMs and group by config= tag in description local vmids=() if [[ -n "$PVE_POOL" ]]; then # Pool response has members[] with type=qemu mapfile -t vmids < <(python3 -c " import json data = json.loads('''${all_vms}''').get('data', {}) for m in data.get('members', []): if m.get('type') == 'qemu': print(m.get('vmid', '')) " 2>/dev/null) else mapfile -t vmids < <(python3 -c " import json data = json.loads('''${all_vms}''').get('data', []) for v in data: print(v.get('vmid', '')) " 2>/dev/null) fi # For each VM, get config and check for managed marker declare -A lab_configs # config_name -> "vmid1:name1:status1 vmid2:name2:status2 ..." declare -A lab_ram # config_name -> total RAM in MiB declare -A lab_disk # config_name -> total disk in GiB local unmanaged=0 for vmid in "${vmids[@]}"; do [[ -z "$vmid" ]] && continue local config_output config_output=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null) || continue local desc name mem_mib disk_gib status read -r desc name mem_mib disk_gib < <(python3 -c " import json data = json.loads('''${config_output}''').get('data', {}) desc = data.get('description', '') name = data.get('name', 'unknown') mem = int(data.get('memory', 0)) # Rough disk from scsi0 size scsi0 = data.get('scsi0', '') disk_g = 0 for part in scsi0.split(','): if part.strip().startswith('size='): sz = part.strip().split('=')[1] if sz.endswith('G'): disk_g = int(sz[:-1]) print(f'{desc}|||{name}|||{mem}|||{disk_g}') " 2>/dev/null | tr '|||' ' ' || true) # Check for managed marker if [[ "$desc" != *"$MANAGED_MARKER"* ]]; then unmanaged=$((unmanaged + 1)) continue fi # Extract config= tag local config_tag="unknown" if [[ "$desc" =~ config=([^\ \]]+) ]]; then config_tag="${BASH_REMATCH[1]}" fi # Get VM status status=$(pve_vm_status "$vmid") || status="unknown" # Accumulate lab_configs["$config_tag"]="${lab_configs[$config_tag]:-} ${vmid}:${name}:${status}" lab_ram["$config_tag"]=$(( ${lab_ram[$config_tag]:-0} + ${mem_mib:-0} )) lab_disk["$config_tag"]=$(( ${lab_disk[$config_tag]:-0} + ${disk_gib:-0} )) done if [[ ${#lab_configs[@]} -eq 0 ]]; then info "No managed labs found" [[ $unmanaged -gt 0 ]] && detail "${unmanaged} unmanaged VMs in pool" return fi for config in $(echo "${!lab_configs[@]}" | tr ' ' '\n' | sort); do local vms="${lab_configs[$config]}" local ram_mib="${lab_ram[$config]}" local disk_gib="${lab_disk[$config]}" local vm_count=0 local running=0 local stopped=0 echo -e " ${BOLD}${config}${RESET}" for entry in $vms; do [[ -z "$entry" ]] && continue local v_vmid v_name v_status IFS=':' read -r v_vmid v_name v_status <<< "$entry" vm_count=$((vm_count + 1)) [[ "$v_status" == "running" ]] && running=$((running + 1)) [[ "$v_status" == "stopped" ]] && stopped=$((stopped + 1)) local status_color="${DIM}" [[ "$v_status" == "running" ]] && status_color="${GREEN}" [[ "$v_status" == "stopped" ]] && status_color="${YELLOW}" echo -e " ${v_name} (${v_vmid}): ${status_color}${v_status}${RESET}" done local lab_status="${GREEN}all running${RESET}" if [[ $running -eq 0 ]] && [[ $stopped -gt 0 ]]; then lab_status="${YELLOW}all stopped${RESET}" elif [[ $running -gt 0 ]] && [[ $stopped -gt 0 ]]; then lab_status="${YELLOW}mixed${RESET}" fi echo -e " Status: ${lab_status} | RAM: $((ram_mib / 1024))G | Disk: ${disk_gib}G" echo "" done [[ $unmanaged -gt 0 ]] && detail "${unmanaged} unmanaged VMs in pool" } # --------------------------------------------------------------------------- # Status check # --------------------------------------------------------------------------- phase_status() { step "Deployment Status" echo "" local count=${#VM_NAMES[@]} local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local vmid="${VM_VMIDS[$i]}" local app="${VM_APPS[$i]}" echo -e " ${BOLD}${name}${RESET} (VMID ${vmid}, app: ${app})" # Proxmox state if [[ "$DRY_RUN" == true ]]; then echo -e " Proxmox: ${DIM}[dry-run]${RESET}" i=$((i + 1)) echo "" continue fi if ! pve_vm_exists "$vmid"; then echo -e " Proxmox: ${RED}not found${RESET}" i=$((i + 1)) echo "" continue fi local managed=false pve_vm_is_managed "$vmid" && managed=true local status status=$(pve_vm_status "$vmid") || status="unknown" local has_media=false pve_vm_has_install_media "$vmid" && has_media=true local install_state="installed" if [[ "$has_media" == true ]]; then install_state="not installed (media attached)" fi local status_color="$GREEN" if [[ "$status" != "running" ]]; then status_color="$YELLOW" fi echo -e " Proxmox: exists managed: ${managed} status: ${status_color}${status}${RESET} ${install_state}" # Network and service checks (only for running, installed VMs) if [[ "$status" == "running" ]] && [[ "$has_media" == false ]]; then local ip ip=$(pve_vm_ip "$vmid") || ip="" if [[ -n "$ip" ]]; then local port_status="${RED}closed${RESET}" if curl -sk --connect-timeout 5 "https://${ip}:8443" &>/dev/null; then port_status="${GREEN}open${RESET}" fi echo -e " Network: ${ip} port 8443: ${port_status}" # Incus remote check + auto-register if command -v incus &>/dev/null; then local remote_name="" remote_name=$(incus remote list --format csv 2>/dev/null | \ awk -F',' -v addr="$ip" '$2 ~ addr {print $1; exit}') || true if [[ -n "$remote_name" ]]; then echo -e " Remote: ${GREEN}incus remote '${remote_name}' configured${RESET}" else register_remote "$name" "$ip" # Re-check after registration attempt remote_name=$(incus remote list --format csv 2>/dev/null | \ awk -F',' -v addr="$ip" '$2 ~ addr {print $1; exit}') || true if [[ -n "$remote_name" ]]; then echo -e " Remote: ${GREEN}incus remote '${remote_name}' configured${RESET}" else echo -e " Remote: ${YELLOW}no incus remote for ${ip}${RESET}" fi fi fi else echo -e " Network: ${YELLOW}IP not detected${RESET}" fi fi i=$((i + 1)) echo "" done } # --------------------------------------------------------------------------- # Post-deployment checks # --------------------------------------------------------------------------- phase_post_deploy() { step "Post-Deployment Checks" echo "" local count=${#VM_NAMES[@]} local all_ok=true local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local vmid="${VM_VMIDS[$i]}" local app="${VM_APPS[$i]}" if [[ "$DRY_RUN" == true ]]; then info "[dry-run] Would check '${name}' (VMID ${vmid})" i=$((i + 1)) continue fi # Only check running, installed VMs if ! pve_vm_exists "$vmid"; then i=$((i + 1)) continue fi local status status=$(pve_vm_status "$vmid") || status="unknown" if [[ "$status" != "running" ]]; then i=$((i + 1)) continue fi local has_media=false pve_vm_has_install_media "$vmid" && has_media=true if [[ "$has_media" == true ]]; then i=$((i + 1)) continue fi local ip ip=$(pve_vm_ip "$vmid") || ip="" if [[ -z "$ip" ]]; then warn "VM '${name}': could not detect IP address" all_ok=false i=$((i + 1)) continue fi # Port 8443 check if curl -sk --connect-timeout 5 "https://${ip}:8443" &>/dev/null; then success "VM '${name}' (${ip}): port 8443 reachable" else warn "VM '${name}' (${ip}): port 8443 not reachable (may still be starting)" all_ok=false i=$((i + 1)) continue fi # App-specific checks if [[ "$app" == "incus" ]] && command -v incus &>/dev/null; then local remote_name="" remote_name=$(incus remote list --format csv 2>/dev/null | \ awk -F',' -v addr="$ip" '$2 ~ addr {print $1; exit}') || true if [[ -n "$remote_name" ]]; then if incus info "${remote_name}:" &>/dev/null; then success "VM '${name}': incus remote '${remote_name}' connected" else warn "VM '${name}': incus remote '${remote_name}' exists but connection failed" all_ok=false fi else # Auto-register the remote register_remote "$name" "$ip" # Verify remote_name=$(incus remote list --format csv 2>/dev/null | \ awk -F',' -v addr="$ip" '$2 ~ addr {print $1; exit}') || true if [[ -n "$remote_name" ]]; then success "VM '${name}': incus remote '${remote_name}' connected" else info "VM '${name}': no incus remote configured" info " Add with: incus remote add ${name} ${ip} --accept-certificate" fi fi elif [[ "$app" == "operations-center" ]]; then success "VM '${name}': Operations Center at https://${ip}:8443" info " Import PKCS#12 cert in browser for web UI access:" info " openssl pkcs12 -export -inkey ~/.config/incus/client.key -in ~/.config/incus/client.crt -out client.pfx" fi i=$((i + 1)) done if [[ "$all_ok" == true ]] && [[ "$DRY_RUN" != true ]]; then echo "" success "All post-deployment checks passed" fi } # --------------------------------------------------------------------------- # Cleanup mode # --------------------------------------------------------------------------- do_cleanup() { step "Cleanup: destroying VMs defined in config" if [[ "$FORCE_CLEANUP" == true ]]; then warn "Force cleanup enabled -- will destroy VMs even without management marker" fi echo "" local count=${#VM_NAMES[@]} local destroyed=0 local not_found=0 local skipped_unmanaged=0 local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local vmid="${VM_VMIDS[$i]}" # 1. Check if VM exists if ! pve_vm_exists "$vmid"; then detail "VM '${name}' (VMID ${vmid}): does not exist" not_found=$((not_found + 1)) i=$((i + 1)) continue fi # 2. Check if VM is managed by us if [[ "$DRY_RUN" != true ]]; then if ! pve_vm_is_managed "$vmid"; then if [[ "$FORCE_CLEANUP" == true ]]; then warn "VM '${name}' (VMID ${vmid}): NOT managed by ${SCRIPT_NAME} -- destroying anyway (--force-cleanup)" else warn "VM '${name}' (VMID ${vmid}): NOT managed by ${SCRIPT_NAME} -- refusing to destroy" warn " This VM was not created by this tool. To destroy it manually:" warn " ssh ${PVE_SSH_USER}@${PVE_HOST} qm stop ${vmid} && ssh ${PVE_SSH_USER}@${PVE_HOST} qm destroy ${vmid} --purge" warn " Or use --force-cleanup to override this safety check" skipped_unmanaged=$((skipped_unmanaged + 1)) i=$((i + 1)) continue fi fi fi # 3. Destroy the VM step "Destroying VM '${name}' (VMID ${vmid})" if [[ "$DRY_RUN" == true ]]; then info "[dry-run] Would stop and destroy VM ${vmid}" destroyed=$((destroyed + 1)) i=$((i + 1)) continue fi # Stop if running local status status=$(pve_vm_status "$vmid") || status="unknown" if [[ "$status" == "running" ]]; then info "Stopping VM..." pve_stop_vm "$vmid" || true sleep 5 fi # Destroy if pve_destroy_vm "$vmid"; then success "VM '${name}' (VMID ${vmid}) destroyed" destroyed=$((destroyed + 1)) else error "Failed to destroy VM '${name}' (VMID ${vmid})" fi i=$((i + 1)) done # Summary echo "" local summary="Cleanup complete: ${destroyed} destroyed" if [[ $not_found -gt 0 ]]; then summary+=", ${not_found} not found" fi if [[ $skipped_unmanaged -gt 0 ]]; then summary+=", ${skipped_unmanaged} skipped (unmanaged)" fi success "$summary" } # Detect which IncusOS ISO was used by checking ide2 config on existing VMs. # Sets DEPLOYMENT_ISO to the filename (e.g. "IncusOS_25.03.iso") or "" if unknown. DEPLOYMENT_ISO="" detect_deployment_iso() { DEPLOYMENT_ISO="" if [[ "$DRY_RUN" == true ]]; then return fi local count=${#VM_NAMES[@]} local i=0 while [[ $i -lt $count ]]; do local vmid="${VM_VMIDS[$i]}" if pve_vm_exists "$vmid"; then local config="" case "$PVE_METHOD" in ssh) config=$(pve_run "qm config ${vmid} 2>/dev/null" 2>/dev/null) || config="" ;; api) config=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null) || config="" ;; esac if [[ -n "$config" ]]; then local iso_name # Extract IncusOS ISO name from ide2 line or JSON iso_name=$(echo "$config" | python3 -c " import sys, json, re text = sys.stdin.read() # Try JSON (API response) try: data = json.loads(text) ide2 = data.get('data', data).get('ide2', '') except (json.JSONDecodeError, AttributeError): ide2 = '' for line in text.splitlines(): if line.startswith('ide2:'): ide2 = line.split(':', 1)[1].strip() break m = re.search(r'(IncusOS_[^,]+\.iso)', ide2) if m: print(m.group(1)) " 2>/dev/null) || iso_name="" if [[ -n "$iso_name" ]]; then DEPLOYMENT_ISO="$iso_name" detail "Detected deployment ISO: ${DEPLOYMENT_ISO} (from VM ${vmid})" return fi fi fi i=$((i + 1)) done } # Deep cleanup: remove seeds and the specific IncusOS ISO used by this deployment cleanup_storage() { step "Deep cleanup: removing seeds and deployment ISO from Proxmox storage" local count=${#VM_NAMES[@]} local i=0 # Remove per-VM seed ISOs while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local seed_file="seed-${name}.iso" if [[ "$DRY_RUN" == true ]]; then info "[dry-run] Would delete ${PVE_ISO_STORAGE}:iso/${seed_file}" else case "$PVE_METHOD" in ssh) pve_run "pvesm free ${PVE_ISO_STORAGE}:iso/${seed_file} 2>/dev/null" &>/dev/null || true ;; api) pve_api DELETE "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content/${PVE_ISO_STORAGE}:iso/${seed_file}" &>/dev/null || true ;; esac info "Deleted seed: ${seed_file}" fi i=$((i + 1)) done # Remove only the specific IncusOS ISO used by this deployment if [[ -n "$DEPLOYMENT_ISO" ]]; then local volid="${PVE_ISO_STORAGE}:iso/${DEPLOYMENT_ISO}" if [[ "$DRY_RUN" == true ]]; then info "[dry-run] Would delete ${volid}" else case "$PVE_METHOD" in ssh) pve_run "pvesm free ${volid} 2>/dev/null" &>/dev/null || true ;; api) local encoded_volid encoded_volid=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${volid}', safe=''))" 2>/dev/null) pve_api DELETE "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content/${encoded_volid}" &>/dev/null || true ;; esac info "Deleted ISO: ${DEPLOYMENT_ISO}" fi else detail "No deployment ISO detected (VMs may already be destroyed)" fi success "Storage cleanup complete" } # Aggressive cleanup: remove ALL IncusOS ISOs and seed ISOs from storage cleanup_storage_all() { step "Deep cleanup: removing ALL IncusOS ISOs and seeds from Proxmox storage" local iso_list="" if [[ "$DRY_RUN" != true ]]; then case "$PVE_METHOD" in ssh) iso_list=$(pve_run "pvesm list ${PVE_ISO_STORAGE} --content iso 2>/dev/null" 2>/dev/null | grep -E "(IncusOS_|seed-)" | awk '{print $1}') || iso_list="" ;; api) iso_list=$(pve_api GET "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content?content=iso" 2>/dev/null | \ python3 -c " import json, sys data = json.load(sys.stdin) for item in data.get('data', []): volid = item.get('volid', '') if 'IncusOS_' in volid or 'seed-' in volid: print(volid) " 2>/dev/null) || iso_list="" ;; esac fi if [[ -n "$iso_list" ]]; then while IFS= read -r volid; do [[ -z "$volid" ]] && continue if [[ "$DRY_RUN" == true ]]; then info "[dry-run] Would delete ${volid}" else case "$PVE_METHOD" in ssh) pve_run "pvesm free ${volid} 2>/dev/null" &>/dev/null || true ;; api) local encoded_volid encoded_volid=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${volid}', safe=''))" 2>/dev/null) pve_api DELETE "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content/${encoded_volid}" &>/dev/null || true ;; esac info "Deleted: ${volid}" fi done <<< "$iso_list" else info "No IncusOS ISOs or seeds found in storage" fi success "Storage cleanup complete" } # Remove incus remotes. Uses VM_NAMES (config-based cleanup) or # DESTROYED_VM_NAMES (pool-wide cleanup) depending on what's populated. cleanup_remotes() { step "Deep cleanup: removing incus remotes" if ! command -v incus &>/dev/null; then detail "incus client not installed, skipping remote cleanup" return fi # Determine which name list to use local names_to_check=() if [[ ${#DESTROYED_VM_NAMES[@]} -gt 0 ]]; then # Pool-wide cleanup: use names of VMs actually destroyed names_to_check=("${DESTROYED_VM_NAMES[@]}") elif [[ ${#VM_NAMES[@]} -gt 0 ]]; then # Config-based cleanup local i=0 while [[ $i -lt ${#VM_NAMES[@]} ]]; do if [[ -n "$VM_FILTER" ]] && [[ "${VM_NAMES[$i]}" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi names_to_check+=("${VM_NAMES[$i]}") i=$((i + 1)) done fi if [[ ${#names_to_check[@]} -eq 0 ]]; then detail "No VM names to check for remote cleanup" success "Remote cleanup complete" return fi for name in "${names_to_check[@]}"; do # Check if a remote with this name exists if incus remote list --format csv 2>/dev/null | grep -q "^${name},"; then if [[ "$DRY_RUN" == true ]]; then info "[dry-run] Would remove incus remote '${name}'" else incus remote remove "$name" 2>/dev/null || true info "Removed incus remote: ${name}" fi fi done success "Remote cleanup complete" } cleanup_local_cache() { step "Deep cleanup: clearing local ISO cache" local cache_base="${XDG_CACHE_HOME:-${HOME}/.cache}/${SCRIPT_NAME}/iso-cache" if [[ -d "$cache_base" ]]; then if [[ "$DRY_RUN" == true ]]; then info "[dry-run] Would clear ${cache_base}" else rm -rf "$cache_base" info "Cleared cache: ${cache_base}" fi else detail "No local cache to clear" fi success "Local cache cleanup complete" } # Names of VMs destroyed during cleanup-all (used by cleanup_remotes) declare -a DESTROYED_VM_NAMES=() # Pool-wide cleanup: destroy ALL managed VMs in the resource pool do_cleanup_all() { step "Pool-wide cleanup: destroying all managed VMs" echo "" if [[ -z "$PVE_POOL" ]]; then error "Pool-wide cleanup requires a resource pool (proxmox.pool)" error "Set pool in proxmox.yaml or pass a config with proxmox.pool" error "Without a pool, there is no safe way to scope which VMs to destroy" exit 1 fi # Get all VMs in the pool local vm_list vm_list=$(pve_list_vms) if [[ -z "$vm_list" ]]; then info "No VMs found in pool '${PVE_POOL}'" return fi # Filter for managed VMs local managed_vms=() local managed_names=() while IFS=' ' read -r eid ename; do [[ -z "$eid" ]] && continue if pve_vm_is_managed "$eid"; then managed_vms+=("$eid") managed_names+=("${ename:-unknown}") fi done <<< "$vm_list" if [[ ${#managed_vms[@]} -eq 0 ]]; then info "No managed VMs found in pool '${PVE_POOL}'" return fi # Show what will be destroyed echo -e " ${BOLD}Managed VMs in pool '${PVE_POOL}':${RESET}" local idx=0 while [[ $idx -lt ${#managed_vms[@]} ]]; do echo -e " VMID ${managed_vms[$idx]}: ${managed_names[$idx]}" idx=$((idx + 1)) done echo "" # Confirm confirm "Destroy all ${#managed_vms[@]} managed VM(s) in pool '${PVE_POOL}'?" # Destroy each local destroyed=0 idx=0 while [[ $idx -lt ${#managed_vms[@]} ]]; do local vmid="${managed_vms[$idx]}" local name="${managed_names[$idx]}" step "Destroying VM '${name}' (VMID ${vmid})" if [[ "$DRY_RUN" == true ]]; then info "[dry-run] Would stop and destroy VM ${vmid}" destroyed=$((destroyed + 1)) DESTROYED_VM_NAMES+=("$name") idx=$((idx + 1)) continue fi local status status=$(pve_vm_status "$vmid") || status="unknown" if [[ "$status" == "running" ]]; then info "Stopping VM..." pve_stop_vm "$vmid" || true sleep 5 fi if pve_destroy_vm "$vmid"; then success "VM '${name}' (VMID ${vmid}) destroyed" destroyed=$((destroyed + 1)) DESTROYED_VM_NAMES+=("$name") else error "Failed to destroy VM '${name}' (VMID ${vmid})" fi idx=$((idx + 1)) done echo "" success "Pool cleanup complete: ${destroyed} VM(s) destroyed" } # --------------------------------------------------------------------------- # Reconcile existing deployments # --------------------------------------------------------------------------- # Check if VMs from config already exist and offer interactive options. # Only called for PHASE=all, non-cleanup, non-dry-run runs. # Returns 0 to continue with fresh deploy, or exits. reconcile_existing() { local count=${#VM_NAMES[@]} local existing=0 local running=0 local installed=0 local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local vmid="${VM_VMIDS[$i]}" if pve_vm_exists "$vmid"; then existing=$((existing + 1)) local status status=$(pve_vm_status "$vmid") || status="unknown" if [[ "$status" == "running" ]]; then running=$((running + 1)) fi local has_media=false pve_vm_has_install_media "$vmid" && has_media=true if [[ "$has_media" == false ]]; then installed=$((installed + 1)) fi fi i=$((i + 1)) done # No existing VMs — fresh deploy, proceed normally if [[ $existing -eq 0 ]]; then return 0 fi # Count of VMs we're deploying (respecting filter) local target=0 for name in "${VM_NAMES[@]}"; do if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then continue fi target=$((target + 1)) done echo "" echo -e "${BOLD}Existing deployment detected${RESET}" echo -e " ${existing}/${target} VMs exist (${running} running, ${installed} installed)" echo "" if [[ "$existing" -lt "$target" ]]; then local incomplete=$((target - existing)) echo -e " ${YELLOW}Partial deployment:${RESET} ${incomplete} VM(s) still need to be created/installed" fi # With --yes, default to safe option (post-deploy checks) if [[ "$YES" == true ]]; then info "Existing VMs detected with --yes; running post-deployment checks" echo "" phase_status phase_post_deploy exit 0 fi echo -e " Options:" echo -e " ${BOLD}1)${RESET} Run status checks and post-deployment verification" echo -e " ${BOLD}2)${RESET} Continue deployment (skip existing, install incomplete VMs)" echo -e " ${BOLD}3)${RESET} Destroy and redeploy cleanly" echo -e " ${BOLD}4)${RESET} Abort" echo "" echo -n -e " ${BOLD}Choose [1/2/3/4]: ${RESET}" local choice read -r choice case "$choice" in 1) phase_status phase_post_deploy exit 0 ;; 2) info "Continuing deployment (existing VMs will be skipped)" return 0 ;; 3) warn "Destroying existing VMs before redeployment" do_cleanup echo "" return 0 ;; 4|*) info "Aborted" exit 0 ;; esac } # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- main() { setup_colors parse_args "$@" validate_args # Handle doctor mode early (before config parsing) if [[ "$DOCTOR" == true ]]; then run_doctor return fi if [[ "$QUIET" != true ]]; then echo -e "${BOLD}${SCRIPT_NAME}${RESET} v${VERSION}" echo "" fi setup_workdir # Handle --labs and --resources without a config file (only needs proxmox.yaml) if { [[ "$LIST_LABS" == true ]] || [[ "$RESOURCES" == true ]]; } && [[ -z "$CONFIG_FILE" ]]; then if ! find_proxmox_config; then error "No proxmox.yaml found and no config file specified" error "Create proxmox.yaml or use --proxmox FILE" exit 1 fi load_proxmox_config "$PROXMOX_CONFIG_RESOLVED" [[ -z "$PVE_NODE" ]] && PVE_NODE="pve" [[ -z "$PVE_STORAGE" ]] && PVE_STORAGE="local-lvm" [[ -z "$PVE_ISO_STORAGE" ]] && PVE_ISO_STORAGE="local" [[ -z "$PVE_BRIDGE" ]] && PVE_BRIDGE="vmbr0" [[ -z "$PVE_SSH_USER" ]] && PVE_SSH_USER="root" [[ -z "$PVE_METHOD" ]] && PVE_METHOD="ssh" [[ -n "$METHOD" ]] && PVE_METHOD="$METHOD" [[ -n "$HOST_OVERRIDE" ]] && PVE_HOST="$HOST_OVERRIDE" if [[ -z "$PVE_HOST" ]]; then error "proxmox.yaml must contain 'host:'" exit 1 fi [[ "$RESOURCES" == true ]] && do_resources [[ "$LIST_LABS" == true ]] && do_list_labs return fi # Handle cleanup-all without a config file (only needs proxmox.yaml) if [[ "$CLEANUP_ALL" == true ]] && [[ -z "$CONFIG_FILE" ]]; then if ! find_proxmox_config; then error "No proxmox.yaml found and no config file specified" error "Create proxmox.yaml or use --proxmox FILE" exit 1 fi load_proxmox_config "$PROXMOX_CONFIG_RESOLVED" # Apply defaults for anything still unset [[ -z "$PVE_NODE" ]] && PVE_NODE="pve" [[ -z "$PVE_STORAGE" ]] && PVE_STORAGE="local-lvm" [[ -z "$PVE_ISO_STORAGE" ]] && PVE_ISO_STORAGE="local" [[ -z "$PVE_BRIDGE" ]] && PVE_BRIDGE="vmbr0" [[ -z "$PVE_SSH_USER" ]] && PVE_SSH_USER="root" [[ -z "$PVE_METHOD" ]] && PVE_METHOD="ssh" [[ -n "$METHOD" ]] && PVE_METHOD="$METHOD" [[ -n "$HOST_OVERRIDE" ]] && PVE_HOST="$HOST_OVERRIDE" if [[ -z "$PVE_HOST" ]]; then error "proxmox.yaml must contain 'host:'" exit 1 fi do_cleanup_all if [[ "$DEEP_CLEANUP" == true ]]; then echo "" cleanup_storage_all cleanup_remotes cleanup_local_cache fi return fi if [[ "$DRY_RUN" != true ]]; then check_dependencies fi echo "" # Always run validate first phase_validate # Handle cleanup-all mode (with config file for additional context) if [[ "$CLEANUP_ALL" == true ]]; then do_cleanup_all if [[ "$DEEP_CLEANUP" == true ]]; then echo "" cleanup_storage_all cleanup_remotes cleanup_local_cache fi return fi # Handle cleanup mode if [[ "$CLEANUP" == true ]]; then if [[ "$DEEP_CLEANUP" == true ]]; then # Detect which ISO was used before destroying VMs detect_deployment_iso fi confirm "Destroy VMs listed above?" do_cleanup if [[ "$DEEP_CLEANUP" == true ]]; then echo "" cleanup_storage cleanup_remotes cleanup_local_cache fi return fi # Handle lab lifecycle commands if [[ "$LAB_DOWN" == true ]]; then phase_lab_down return fi if [[ "$LAB_UP" == true ]]; then phase_lab_up return fi # Handle --resources and --labs with config file if [[ "$RESOURCES" == true ]] || [[ "$LIST_LABS" == true ]]; then [[ "$RESOURCES" == true ]] && do_resources [[ "$LIST_LABS" == true ]] && do_list_labs return fi # Reconcile existing deployments (interactive menu on re-runs) if [[ "$PHASE" == "all" ]] && [[ "$DRY_RUN" != true ]]; then reconcile_existing fi # Confirm before proceeding if [[ "$PHASE" == "all" ]]; then confirm "Proceed with deployment?" fi # Run requested phases case "$PHASE" in validate) # Already done above ;; prepare) phase_prepare ;; upload) phase_prepare phase_upload ;; create) phase_prepare phase_upload phase_create ;; install) phase_prepare phase_upload phase_create phase_install ;; status) phase_status phase_post_deploy ;; all) phase_prepare echo "" phase_upload echo "" phase_create echo "" phase_install echo "" phase_post_deploy ;; esac echo "" if [[ "$DRY_RUN" == true ]]; then info "Dry run complete -- no changes were made" else success "Done" fi } main "$@"