#!/usr/bin/env bash # incusos-proxmox - Declarative IncusOS deployment on Proxmox VE # # Reads a YAML configuration file defining VMs and deploys them on a Proxmox # host with IncusOS-correct settings (UEFI, TPM 2.0, VirtIO-SCSI, etc.). # Generates per-VM seed images, uploads ISOs, creates VMs, and boots them # through automated installation. # # Usage: incusos-proxmox [OPTIONS] CONFIG_FILE # Run 'incusos-proxmox --help' for full usage information. set -euo pipefail readonly VERSION="0.1.0" readonly SCRIPT_NAME="incusos-proxmox" readonly CDN_INDEX="https://images.linuxcontainers.org/os/index.json" readonly CDN_BASE="https://images.linuxcontainers.org/os" readonly MIN_DISK_GIB=50 readonly MIN_MEMORY_MIB=4096 readonly MANAGED_MARKER="[incusos-lab:managed]" # --------------------------------------------------------------------------- # Auto-load env file (secrets like PROXMOX_TOKEN_SECRET) # --------------------------------------------------------------------------- # Searches for an 'env' file: first next to the script, then walking up from # the current directory. Sources it if found and not already loaded. This is # a convenience -- users can still export variables manually or source the # file themselves. load_env_file() { # Skip if the secret is already set (user exported it, or env already sourced) if [[ -n "${PROXMOX_TOKEN_SECRET:-}" ]]; then return 0 fi local script_real script_real=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # Check: script directory's parent (repo root when script is in incusos/) if [[ -f "${script_real}/../env" ]]; then # shellcheck source=/dev/null source "${script_real}/../env" return 0 fi # Check: script directory itself if [[ -f "${script_real}/env" ]]; then # shellcheck source=/dev/null source "${script_real}/env" return 0 fi # Walk up from cwd local dir="$PWD" while [[ "$dir" != "/" ]]; do if [[ -f "${dir}/env" ]]; then # shellcheck source=/dev/null source "${dir}/env" return 0 fi dir=$(dirname "$dir") done # No env file found -- not an error; require_api_auth() will catch # missing secrets later if API method is used. return 0 } load_env_file # --------------------------------------------------------------------------- # Color and formatting # --------------------------------------------------------------------------- setup_colors() { if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[0;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' BOLD='\033[1m' DIM='\033[2m' RESET='\033[0m' else RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' RESET='' fi } info() { [[ "$QUIET" == true ]] && return; echo -e "${BLUE}[info]${RESET} $*"; } success() { [[ "$QUIET" == true ]] && return; echo -e "${GREEN}[ok]${RESET} $*"; } warn() { echo -e "${YELLOW}[warn]${RESET} $*" >&2; } error() { echo -e "${RED}[error]${RESET} $*" >&2; } step() { [[ "$QUIET" == true ]] && return; echo -e "${CYAN}[step]${RESET} ${BOLD}$*${RESET}"; } detail() { [[ "$VERBOSE" != true ]] && return; echo -e "${DIM} $*${RESET}"; } # --------------------------------------------------------------------------- # Help # --------------------------------------------------------------------------- usage() { cat < pvesh get /version ${BOLD}API method:${RESET} Requires an API token. Set up minimal privileges: pveum role add IncusOSDeployer --privs "VM.Allocate VM.Config.Disk \\ VM.Config.CPU VM.Config.Memory VM.Config.Network VM.Config.CDROM \\ VM.Config.Options VM.Config.HWType VM.PowerMgmt VM.Audit \\ Datastore.AllocateSpace Datastore.AllocateTemplate \\ Datastore.Audit SDN.Use Sys.Audit" pveum user add automation@pve pveum aclmod / -user automation@pve -role IncusOSDeployer pveum user token add automation@pve deploy --privsep 0 For resource pool isolation (recommended): pveum pool add IncusLab --comment "IncusOS Lab VMs" pveum pool modify IncusLab --storage local-lvm,local pveum aclmod /pool/IncusLab -user automation@pve -role IncusOSDeployer Set the token secret: export PROXMOX_TOKEN_SECRET="" And in your config file: proxmox: method: api api_token_id: automation@pve!deploy ${BOLD}PREREQUISITES${RESET} - python3 with PyYAML (python3-yaml / py3-yaml) - jq (for JSON field access; falls back to python3) - ssh + scp (for SSH method) - curl (for API method and CDN downloads) - incusos-seed sibling script (auto-detected) - Bash 4+ (associative arrays) EOF exit 0 } # --------------------------------------------------------------------------- # Argument parsing # --------------------------------------------------------------------------- # Defaults PHASE="all" METHOD="" HOST_OVERRIDE="" LOCAL_ISO="" CERT_FILE="" NO_CERT=false VM_FILTER="" YES=false DOCTOR=false CLEANUP=false CLEANUP_ALL=false DEEP_CLEANUP=false FORCE_CLEANUP=false LAB_UP=false LAB_DOWN=false RESOURCES=false LIST_LABS=false DRY_RUN=false QUIET=false VERBOSE=false INSTALL_RETRIES=3 PROXMOX_CONFIG="" CONFIG_FILE="" parse_args() { while [[ $# -gt 0 ]]; do case "$1" in -p|--phase) PHASE="${2:?'--phase requires a value'}" shift 2 ;; -m|--method) METHOD="${2:?'--method requires a value'}" shift 2 ;; -H|--host) HOST_OVERRIDE="${2:?'--host requires a value'}" shift 2 ;; -i|--iso) LOCAL_ISO="${2:?'--iso requires a file path'}" shift 2 ;; -c|--cert) CERT_FILE="${2:?'--cert requires a file path'}" shift 2 ;; --no-cert) NO_CERT=true shift ;; --vm) VM_FILTER="${2:?'--vm requires a name'}" shift 2 ;; -y|--yes) YES=true shift ;; --retries) INSTALL_RETRIES="${2:?'--retries requires a number'}" if ! [[ "$INSTALL_RETRIES" =~ ^[0-9]+$ ]]; then error "--retries must be a non-negative integer" exit 1 fi shift 2 ;; --status) PHASE="status" shift ;; --doctor) DOCTOR=true shift ;; --cleanup) CLEANUP=true shift ;; --deep) DEEP_CLEANUP=true shift ;; --cleanup-all) CLEANUP_ALL=true shift ;; --force-cleanup) CLEANUP=true FORCE_CLEANUP=true shift ;; --lab-up) LAB_UP=true shift ;; --lab-down) LAB_DOWN=true shift ;; --resources) RESOURCES=true shift ;; --labs) LIST_LABS=true shift ;; --proxmox) PROXMOX_CONFIG="${2:?'--proxmox requires a file path'}" shift 2 ;; --dry-run) DRY_RUN=true shift ;; -v|--verbose) VERBOSE=true shift ;; -q|--quiet) QUIET=true shift ;; -h|--help) usage ;; -V|--version) echo "${SCRIPT_NAME} v${VERSION}" exit 0 ;; -*) error "Unknown option: $1" echo "Run '${SCRIPT_NAME} --help' for usage information." >&2 exit 1 ;; *) if [[ -z "$CONFIG_FILE" ]]; then CONFIG_FILE="$1" else error "Unexpected argument: $1 (config file already set to ${CONFIG_FILE})" echo "Run '${SCRIPT_NAME} --help' for usage information." >&2 exit 1 fi shift ;; esac done } # --------------------------------------------------------------------------- # Validation # --------------------------------------------------------------------------- validate_args() { # Universal checks (apply regardless of mode) if [[ "$VERBOSE" == true ]] && [[ "$QUIET" == true ]]; then error "Cannot use both --verbose and --quiet" exit 1 fi # --doctor can run without a config file if [[ "$DOCTOR" == true ]] && [[ -z "$CONFIG_FILE" ]]; then return 0 fi # --cleanup-all, --labs, --resources can run without a config file (uses proxmox.yaml) if [[ "$CLEANUP_ALL" == true ]] && [[ -z "$CONFIG_FILE" ]]; then return 0 fi if [[ "$LIST_LABS" == true ]] && [[ -z "$CONFIG_FILE" ]]; then return 0 fi if [[ "$RESOURCES" == true ]] && [[ -z "$CONFIG_FILE" ]]; then return 0 fi if [[ -z "$CONFIG_FILE" ]]; then error "No configuration file specified" echo "Usage: ${SCRIPT_NAME} [OPTIONS] CONFIG_FILE" >&2 echo "Run '${SCRIPT_NAME} --help' for usage information." >&2 exit 1 fi if [[ ! -f "$CONFIG_FILE" ]]; then error "Configuration file not found: ${CONFIG_FILE}" exit 1 fi # --deep only makes sense with --cleanup or --cleanup-all if [[ "$DEEP_CLEANUP" == true ]] && [[ "$CLEANUP" != true ]] && [[ "$CLEANUP_ALL" != true ]]; then error "--deep can only be used with --cleanup or --cleanup-all" exit 1 fi case "$PHASE" in validate|prepare|upload|create|install|status|all) ;; *) error "Invalid phase: ${PHASE}" error "Supported: validate, prepare, upload, create, install, status, all" exit 1 ;; esac if [[ -n "$METHOD" ]]; then case "$METHOD" in ssh|api) ;; *) error "Invalid method: ${METHOD}" error "Supported: ssh, api" exit 1 ;; esac fi if [[ -n "$CERT_FILE" ]] && [[ "$NO_CERT" == true ]]; then error "Cannot use both --cert and --no-cert" exit 1 fi if [[ -n "$CERT_FILE" ]] && [[ ! -f "$CERT_FILE" ]]; then error "Certificate file not found: ${CERT_FILE}" exit 1 fi if [[ -n "$LOCAL_ISO" ]] && [[ ! -f "$LOCAL_ISO" ]]; then error "ISO file not found: ${LOCAL_ISO}" exit 1 fi } # --------------------------------------------------------------------------- # Dependency checks # --------------------------------------------------------------------------- SEED_TOOL="" check_dependencies() { step "Checking dependencies" local missing=false # python3 if ! command -v python3 &>/dev/null; then error "python3 is required but not found" error "Install Python 3 for your platform (https://www.python.org/)" missing=true else detail "python3: $(command -v python3)" fi # python3-yaml (optional -- basic YAML fallback built-in) if command -v python3 &>/dev/null; then if python3 -c "import yaml" 2>/dev/null; then detail "PyYAML: available" else detail "PyYAML: not found (using built-in basic parser)" fi fi # jq (optional, python3 fallback) if command -v jq &>/dev/null; then detail "jq: $(command -v jq)" else detail "jq: not found (using python3 fallback)" fi # curl if ! command -v curl &>/dev/null; then error "curl is required but not found" missing=true else detail "curl: $(command -v curl)" fi # ssh/scp (for SSH method) if [[ "${PVE_METHOD:-ssh}" == "ssh" ]]; then if ! command -v ssh &>/dev/null; then error "ssh is required for SSH method but not found" missing=true fi if ! command -v scp &>/dev/null; then error "scp is required for SSH method but not found" missing=true fi fi # incusos-seed sibling script local script_dir script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SEED_TOOL="${script_dir}/incusos-seed" if [[ ! -x "$SEED_TOOL" ]]; then error "incusos-seed not found at: ${SEED_TOOL}" error "The incusos-seed script must be in the same directory as ${SCRIPT_NAME}" missing=true else detail "incusos-seed: ${SEED_TOOL}" fi # bash version if [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then error "Bash 4+ is required (found ${BASH_VERSION})" missing=true else detail "bash: ${BASH_VERSION}" fi if [[ "$missing" == true ]]; then error "Missing dependencies -- cannot continue" exit 1 fi success "All dependencies satisfied" } # --------------------------------------------------------------------------- # Doctor mode # --------------------------------------------------------------------------- run_doctor() { echo -e "${BOLD}${SCRIPT_NAME} doctor${RESET}" echo "" local script_dir script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # -- Platform section -- echo -e " ${BOLD}Platform:${RESET}" echo -e " OS: $(uname -s) $(uname -m)" echo -e " Bash: ${BASH_VERSION}" echo "" # -- Tools section -- echo -e " ${BOLD}Tools:${RESET}" # incus client if command -v incus &>/dev/null; then local incus_ver incus_ver=$(incus version 2>/dev/null | sed -n 's/.*Client version: \([0-9.]*\).*/\1/p') [[ -z "$incus_ver" ]] && incus_ver="unknown" echo -e " incus client: ${GREEN}${incus_ver} ✓${RESET}" else echo -e " incus client: ${YELLOW}not installed${RESET}" fi # flasher-tool if command -v flasher-tool &>/dev/null; then local ft_ver ft_ver=$(flasher-tool --version 2>/dev/null | head -1 | sed 's/^flasher-tool version //' || echo "installed") if [[ -z "$ft_ver" ]] || [[ "$ft_ver" == *"unknown"* ]]; then ft_ver="dev (installed)" fi echo -e " flasher-tool: ${GREEN}${ft_ver} ✓${RESET}" else echo -e " flasher-tool: ${YELLOW}not installed${RESET}" fi # operations-center CLI if command -v operations-center &>/dev/null; then local oc_ver oc_ver=$(operations-center --version 2>/dev/null | head -1 || echo "installed") echo -e " operations-center CLI: ${GREEN}${oc_ver} ✓${RESET}" else echo -e " operations-center CLI: ${DIM}not installed (optional)${RESET}" fi # python3 if command -v python3 &>/dev/null; then local py_ver py_ver=$(python3 --version 2>/dev/null | awk '{print $2}') echo -e " python3: ${GREEN}${py_ver} ✓${RESET}" else echo -e " python3: ${RED}not installed (required)${RESET}" fi # curl if command -v curl &>/dev/null; then local curl_ver curl_ver=$(curl --version 2>/dev/null | head -1 | awk '{print $2}') echo -e " curl: ${GREEN}${curl_ver} ✓${RESET}" else echo -e " curl: ${RED}not installed (required)${RESET}" fi # ISO creation tool (genisoimage on Linux, mkisofs on macOS) if command -v genisoimage &>/dev/null; then echo -e " genisoimage: ${GREEN}installed ✓${RESET}" elif command -v mkisofs &>/dev/null; then echo -e " mkisofs: ${GREEN}installed ✓${RESET} (cdrtools)" else case "$(uname -s)" in Darwin) echo -e " mkisofs: ${RED}not installed (brew install cdrtools)${RESET}" ;; *) echo -e " genisoimage: ${RED}not installed (required for seed ISOs)${RESET}" ;; esac fi # jq if command -v jq &>/dev/null; then local jq_ver jq_ver=$(jq --version 2>/dev/null | sed 's/^jq-//') echo -e " jq: ${GREEN}${jq_ver} ✓${RESET}" else echo -e " jq: ${YELLOW}not installed (python3 fallback used)${RESET}" fi # incusos-seed sibling if [[ -x "${script_dir}/incusos-seed" ]]; then echo -e " incusos-seed: ${GREEN}found ✓${RESET}" else echo -e " incusos-seed: ${RED}not found at ${script_dir}/incusos-seed${RESET}" fi # -- IncusOS section -- echo "" echo -e " ${BOLD}IncusOS:${RESET}" local cdn_latest="" local cdn_index cdn_index=$(mktemp) if curl -fsSL --connect-timeout 5 "$CDN_INDEX" -o "$cdn_index" 2>/dev/null; then if command -v jq &>/dev/null; then cdn_latest=$(jq -r '[.updates[] | select(.channels[] == "stable")] | sort_by(.version) | last | .version // empty' "$cdn_index" 2>/dev/null) else cdn_latest=$(python3 -c " import json data = json.load(open('${cdn_index}')) versions = [u['version'] for u in data.get('updates', []) if 'stable' in u.get('channels', [])] if versions: print(sorted(versions)[-1]) " 2>/dev/null) fi fi rm -f "$cdn_index" if [[ -n "$cdn_latest" ]]; then echo -e " Latest stable: ${cdn_latest}" else echo -e " Latest stable: ${YELLOW}could not reach CDN${RESET}" fi # Check local cache local cache_base="${XDG_CACHE_HOME:-${HOME}/.cache}/${SCRIPT_NAME}/iso-cache" if [[ -d "$cache_base" ]]; then local cached_iso cached_iso=$(ls -1 "${cache_base}"/IncusOS_*.iso 2>/dev/null | sort | tail -1 || true) if [[ -n "$cached_iso" ]]; then local cached_name cached_name=$(basename "$cached_iso" .iso | sed 's/^IncusOS_//') if [[ -n "$cdn_latest" ]] && [[ "$cached_name" == "$cdn_latest" ]]; then echo -e " Cached ISO: ${GREEN}${cached_name} (up to date)${RESET}" else echo -e " Cached ISO: ${YELLOW}${cached_name}${RESET}" fi else echo -e " Cached ISO: ${DIM}none${RESET}" fi else echo -e " Cached ISO: ${DIM}none${RESET}" fi # -- Config section -- echo "" echo -e " ${BOLD}Config:${RESET}" if find_proxmox_config; then echo -e " proxmox.yaml: ${GREEN}${PROXMOX_CONFIG_RESOLVED} ✓${RESET}" else echo -e " proxmox.yaml: ${DIM}not found (optional)${RESET}" fi if [[ -n "$CONFIG_FILE" ]] && [[ -f "$CONFIG_FILE" ]]; then echo -e " Lab config: ${CONFIG_FILE}" fi # -- Proxmox section (if config file or proxmox.yaml provides connection info) -- echo "" if [[ -n "$CONFIG_FILE" ]] && [[ -f "$CONFIG_FILE" ]]; then echo -e " ${BOLD}Proxmox:${RESET}" # Minimal config parsing for doctor mode local doctor_workdir doctor_workdir=$(mktemp -d "${TMPDIR:-/tmp}/${SCRIPT_NAME}-doctor-XXXXXX") WORKDIR="$doctor_workdir" CONFIG_JSON="${doctor_workdir}/config.json" if python3 -c " import json, sys try: import yaml with open(sys.argv[1]) as f: data = yaml.safe_load(f) except ImportError: # Minimal inline parse -- just need proxmox section data = {} with open(sys.argv[1]) as f: import re lines = f.readlines() processed = [] for raw in lines: s = raw.rstrip() if not s or s.lstrip().startswith('#'): continue indent = len(s) - len(s.lstrip()) content = s.lstrip() processed.append((indent, content)) result = {} section = None for indent, content in processed: m = re.match(r'^([^:]+?):\s*(.*?)$', content) if m: key = m.group(1).strip() val = m.group(2).strip() if not val: section = key result[section] = {} elif section: if val.startswith('\"') or val.startswith(\"'\"): val = val[1:-1] result[section][key] = val data = result json.dump(data, sys.stdout, indent=2) " "$CONFIG_FILE" > "$CONFIG_JSON" 2>/dev/null; then local doc_host doc_method doc_token_id doc_pool doc_host=$(json_get "$CONFIG_JSON" ".proxmox.host" "") doc_method=$(json_get "$CONFIG_JSON" ".proxmox.method" "ssh") doc_token_id=$(json_get "$CONFIG_JSON" ".proxmox.api_token_id" "") doc_pool=$(json_get "$CONFIG_JSON" ".proxmox.pool" "") if [[ -z "$doc_host" ]]; then echo -e " Host: ${RED}not set in config${RESET}" else echo -e " Host: ${doc_host} (${doc_method})" # Connectivity check local pve_ok=false if [[ "$doc_method" == "api" ]]; then if [[ -n "$doc_token_id" ]] && [[ -n "${PROXMOX_TOKEN_SECRET:-}" ]]; then local auth="Authorization: PVEAPIToken=${doc_token_id}=${PROXMOX_TOKEN_SECRET}" local pve_ver pve_ver=$(curl -fsSk --connect-timeout 5 -H "$auth" \ "https://${doc_host}:8006/api2/json/version" 2>/dev/null | \ python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('version',''))" 2>/dev/null) || pve_ver="" if [[ -n "$pve_ver" ]]; then echo -e " Connectivity: ${GREEN}connected (PVE ${pve_ver}) ✓${RESET}" pve_ok=true else echo -e " Connectivity: ${RED}API connection failed${RESET}" fi else echo -e " Connectivity: ${YELLOW}API token not configured or PROXMOX_TOKEN_SECRET not set${RESET}" fi else local ssh_user ssh_user=$(json_get "$CONFIG_JSON" ".proxmox.ssh_user" "root") if ssh -o BatchMode=yes -o ConnectTimeout=5 "${ssh_user}@${doc_host}" "true" &>/dev/null; then echo -e " Connectivity: ${GREEN}SSH connected ✓${RESET}" pve_ok=true else echo -e " Connectivity: ${RED}SSH connection failed${RESET}" fi fi # Pool check (API only, if connected) if [[ -n "$doc_pool" ]]; then if [[ "$pve_ok" == true ]] && [[ "$doc_method" == "api" ]]; then local auth="Authorization: PVEAPIToken=${doc_token_id}=${PROXMOX_TOKEN_SECRET}" if curl -fsSk --connect-timeout 5 -H "$auth" \ "https://${doc_host}:8006/api2/json/pools/${doc_pool}" &>/dev/null; then echo -e " Pool: ${GREEN}${doc_pool} exists ✓${RESET}" else echo -e " Pool: ${RED}${doc_pool} not found${RESET}" fi else echo -e " Pool: ${doc_pool} (not verified)" fi fi fi else echo -e " ${RED}Failed to parse config file${RESET}" fi rm -rf "$doctor_workdir" elif [[ -n "$PROXMOX_CONFIG_RESOLVED" ]]; then # Use proxmox.yaml for connectivity check (no lab config provided) echo -e " ${BOLD}Proxmox:${RESET} ${DIM}(from proxmox.yaml)${RESET}" local doctor_workdir doctor_workdir=$(mktemp -d "${TMPDIR:-/tmp}/${SCRIPT_NAME}-doctor-XXXXXX") WORKDIR="$doctor_workdir" load_proxmox_config "$PROXMOX_CONFIG_RESOLVED" [[ -z "$PVE_NODE" ]] && PVE_NODE="pve" [[ -z "$PVE_METHOD" ]] && PVE_METHOD="ssh" [[ -z "$PVE_SSH_USER" ]] && PVE_SSH_USER="root" [[ -n "$METHOD" ]] && PVE_METHOD="$METHOD" [[ -n "$HOST_OVERRIDE" ]] && PVE_HOST="$HOST_OVERRIDE" if [[ -z "$PVE_HOST" ]]; then echo -e " Host: ${RED}not set in proxmox.yaml${RESET}" else echo -e " Host: ${PVE_HOST} (${PVE_METHOD})" local pve_ok=false if [[ "$PVE_METHOD" == "api" ]]; then if [[ -n "$PVE_API_TOKEN_ID" ]] && [[ -n "${PROXMOX_TOKEN_SECRET:-}" ]]; then local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}" local pve_ver pve_ver=$(curl -fsSk --connect-timeout 5 -H "$auth" \ "https://${PVE_HOST}:8006/api2/json/version" 2>/dev/null | \ python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('version',''))" 2>/dev/null) || pve_ver="" if [[ -n "$pve_ver" ]]; then echo -e " Connectivity: ${GREEN}connected (PVE ${pve_ver}) ✓${RESET}" pve_ok=true else echo -e " Connectivity: ${RED}API connection failed${RESET}" fi else echo -e " Connectivity: ${YELLOW}API token not configured or PROXMOX_TOKEN_SECRET not set${RESET}" fi else if ssh -o BatchMode=yes -o ConnectTimeout=5 "${PVE_SSH_USER}@${PVE_HOST}" "true" &>/dev/null; then echo -e " Connectivity: ${GREEN}SSH connected ✓${RESET}" pve_ok=true else echo -e " Connectivity: ${RED}SSH connection failed${RESET}" fi fi if [[ -n "$PVE_POOL" ]]; then if [[ "$pve_ok" == true ]] && [[ "$PVE_METHOD" == "api" ]]; then local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}" if curl -fsSk --connect-timeout 5 -H "$auth" \ "https://${PVE_HOST}:8006/api2/json/pools/${PVE_POOL}" &>/dev/null; then echo -e " Pool: ${GREEN}${PVE_POOL} exists ✓${RESET}" else echo -e " Pool: ${RED}${PVE_POOL} not found${RESET}" fi else echo -e " Pool: ${PVE_POOL} (not verified)" fi fi fi rm -rf "$doctor_workdir" else echo -e " ${BOLD}Proxmox:${RESET} ${DIM}(no config file or proxmox.yaml found)${RESET}" fi echo "" } # --------------------------------------------------------------------------- # Temporary directory management # --------------------------------------------------------------------------- WORKDIR="" CACHEDIR="" setup_workdir() { # Use persistent storage for the work directory -- the ISO download + # decompression can exceed 4 GiB, which overflows tmpfs on most systems. # Honour TMPDIR if explicitly set; otherwise use ~/.cache on real disk. local base="${TMPDIR:-${XDG_CACHE_HOME:-${HOME}/.cache}/${SCRIPT_NAME}}" mkdir -p "$base" # Persistent cache for ISO downloads (survives across runs) CACHEDIR="${base}/iso-cache" mkdir -p "$CACHEDIR" WORKDIR=$(mktemp -d "${base}/${SCRIPT_NAME}-XXXXXX") trap 'rm -rf "$WORKDIR"' EXIT } # --------------------------------------------------------------------------- # YAML parsing via python3 # --------------------------------------------------------------------------- # Convert YAML config to JSON, stored in a file for repeated access CONFIG_JSON="" parse_config() { step "Parsing configuration: ${CONFIG_FILE}" CONFIG_JSON="${WORKDIR}/config.json" # Try PyYAML first, fall back to a basic pure-Python parser if ! python3 -c " import json, sys, re def _parse_value(v): v = v.strip() # Strip inline comments (but not inside quotes) if not v.startswith(('\"', \"'\")): m = re.match(r'^(.*?)\s+#.*$', v) if m: v = m.group(1).rstrip() if not v: return None if (v.startswith('\"') and v.endswith('\"')) or (v.startswith(\"'\") and v.endswith(\"'\")): return v[1:-1] if v == 'true': return True if v == 'false': return False if v == 'null' or v == '~': return None try: return int(v) except ValueError: pass try: return float(v) except ValueError: pass return v def parse_yaml_basic(path): \"\"\"Minimal YAML parser for simple config files. Handles: mappings, lists of mappings, lists of scalars, inline comments. Does NOT handle: anchors, multi-line strings, flow style, merge keys.\"\"\" with open(path) as f: lines = f.readlines() # Pre-process: strip comments and blank lines, track indent + content processed = [] for raw in lines: s = raw.rstrip() if not s or s.lstrip().startswith('#'): continue indent = len(s) - len(s.lstrip()) content = s.lstrip() processed.append((indent, content)) def parse_block(idx, base_indent): \"\"\"Parse a block at the given indentation level, return (result, next_idx).\"\"\" if idx >= len(processed): return None, idx indent, content = processed[idx] # Detect if this block is a list or mapping if content.startswith('- '): return parse_list(idx, indent) else: return parse_mapping(idx, indent) def parse_mapping(idx, base_indent): result = {} while idx < len(processed): indent, content = processed[idx] if indent < base_indent: break if indent > base_indent: break if content.startswith('- '): break # Must be key: value m = re.match(r'^([^:]+?):\s*(.*?)$', content) if not m: idx += 1 continue key = m.group(1).strip() val_str = m.group(2).strip() # Strip inline comment from value if val_str and not val_str.startswith(('\"', \"'\")): cm = re.match(r'^(.*?)\s+#.*$', val_str) if cm: val_str = cm.group(1).rstrip() if val_str and not val_str.startswith('#'): result[key] = _parse_value(val_str) idx += 1 else: # Value is a nested block idx += 1 if idx < len(processed) and processed[idx][0] > base_indent: child_indent = processed[idx][0] child, idx = parse_block(idx, child_indent) result[key] = child if child is not None else {} else: result[key] = {} return result, idx def parse_list(idx, base_indent): result = [] while idx < len(processed): indent, content = processed[idx] if indent < base_indent: break if indent > base_indent: break if not content.startswith('- '): break item_str = content[2:].strip() # Check if it's a mapping item: '- key: value' m = re.match(r'^([^:]+?):\s*(.*?)$', item_str) if m: # It's a mapping -- parse this item and continuation lines obj = {} key = m.group(1).strip() val_str = m.group(2).strip() if val_str and not val_str.startswith('#'): if not val_str.startswith(('\"', \"'\")): cm = re.match(r'^(.*?)\s+#.*$', val_str) if cm: val_str = cm.group(1).rstrip() obj[key] = _parse_value(val_str) else: idx += 1 if idx < len(processed) and processed[idx][0] > indent: child_indent = processed[idx][0] child, idx = parse_block(idx, child_indent) obj[key] = child if child is not None else {} else: obj[key] = {} # Parse remaining keys at item indent + 2 item_indent = indent + 2 while idx < len(processed) and processed[idx][0] == item_indent: ic = processed[idx][1] if ic.startswith('- '): break im = re.match(r'^([^:]+?):\s*(.*?)$', ic) if im: ik = im.group(1).strip() iv = im.group(2).strip() if iv and not iv.startswith('#'): if not iv.startswith(('\"', \"'\")): icm = re.match(r'^(.*?)\s+#.*$', iv) if icm: iv = icm.group(1).rstrip() obj[ik] = _parse_value(iv) idx += 1 else: idx += 1 if idx < len(processed) and processed[idx][0] > item_indent: child_indent = processed[idx][0] child, idx = parse_block(idx, child_indent) obj[ik] = child if child is not None else {} else: obj[ik] = {} else: idx += 1 result.append(obj) continue idx += 1 # Parse remaining keys of this list item item_indent = indent + 2 while idx < len(processed) and processed[idx][0] >= item_indent: if processed[idx][0] == item_indent: ic = processed[idx][1] if ic.startswith('- '): break im = re.match(r'^([^:]+?):\s*(.*?)$', ic) if im: ik = im.group(1).strip() iv = im.group(2).strip() if iv and not iv.startswith('#'): if not iv.startswith(('\"', \"'\")): icm = re.match(r'^(.*?)\s+#.*$', iv) if icm: iv = icm.group(1).rstrip() obj[ik] = _parse_value(iv) idx += 1 else: idx += 1 if idx < len(processed) and processed[idx][0] > item_indent: child_indent = processed[idx][0] child, idx = parse_block(idx, child_indent) obj[ik] = child if child is not None else {} else: obj[ik] = {} else: idx += 1 else: # Deeper nesting handled by recursive calls above break result.append(obj) else: # Scalar list item result.append(_parse_value(item_str)) idx += 1 return result, idx result, _ = parse_block(0, 0) return result if result is not None else {} try: import yaml with open(sys.argv[1]) as f: data = yaml.safe_load(f) except ImportError: data = parse_yaml_basic(sys.argv[1]) json.dump(data, sys.stdout, indent=2) " "$CONFIG_FILE" > "$CONFIG_JSON" 2>/dev/null; then error "Failed to parse YAML configuration: ${CONFIG_FILE}" error "Check that the file is valid YAML" error "For complex YAML features, install PyYAML: pip3 install pyyaml" exit 1 fi success "Configuration parsed" } # JSON field accessor -- uses jq if available, falls back to python3 json_get() { local file="$1" local query="$2" local default="${3:-}" local result if command -v jq &>/dev/null; then result=$(jq -r "$query // empty" "$file" 2>/dev/null) || result="" else result=$(python3 -c " import json, sys data = json.load(open(sys.argv[1])) path = sys.argv[2] # Simple jq-like path traversal for common patterns if path.startswith('.'): path = path[1:] parts = path.replace('[]','').split('.') obj = data for p in parts: if p == '': continue if isinstance(obj, dict): obj = obj.get(p) elif isinstance(obj, list) and p.isdigit(): obj = obj[int(p)] else: obj = None if obj is None: break if obj is None: print('') elif isinstance(obj, bool): print('true' if obj else 'false') elif isinstance(obj, (dict, list)): json.dump(obj, sys.stdout) else: print(obj) " "$file" "$query" 2>/dev/null) || result="" fi if [[ -z "$result" ]] && [[ -n "$default" ]]; then echo "$default" else echo "$result" fi } # Get the number of VMs in config vm_count() { if command -v jq &>/dev/null; then jq -r '.vms | length' "$CONFIG_JSON" 2>/dev/null || echo "0" else python3 -c " import json data = json.load(open('$CONFIG_JSON')) print(len(data.get('vms', []))) " 2>/dev/null || echo "0" fi } # Get a field from a specific VM by index vm_get() { local index="$1" local field="$2" local default="${3:-}" local result if command -v jq &>/dev/null; then result=$(jq -r ".vms[${index}].${field} // empty" "$CONFIG_JSON" 2>/dev/null) || result="" else result=$(python3 -c " import json data = json.load(open('$CONFIG_JSON')) vm = data.get('vms', [])[${index}] val = vm.get('${field}') if val is None: print('') elif isinstance(val, bool): print('true' if val else 'false') else: print(val) " 2>/dev/null) || result="" fi if [[ -z "$result" ]] && [[ -n "$default" ]]; then echo "$default" else echo "$result" fi } # --------------------------------------------------------------------------- # Configuration loading # --------------------------------------------------------------------------- # Proxmox connection settings PVE_HOST="" PVE_NODE="" PVE_STORAGE="" PVE_ISO_STORAGE="" PVE_BRIDGE="" PVE_VLAN="" PVE_GATEWAY="" PVE_DNS="" PVE_METHOD="" PVE_SSH_USER="" PVE_API_TOKEN_ID="" PVE_POOL="" # Default VM settings DEF_CORES="" DEF_MEMORY="" DEF_DISK="" DEF_CHANNEL="" DEF_START_VMID="" # VM arrays (indexed by position) declare -a VM_NAMES=() declare -a VM_APPS=() declare -a VM_APPLY_DEFAULTS=() declare -a VM_CORES=() declare -a VM_MEMORY=() declare -a VM_DISK=() declare -a VM_VMIDS=() declare -a VM_IPS=() declare -a VM_CLUSTERS=() declare -a VM_CLUSTER_ROLES=() # Path to the resolved proxmox connection config file (set by find_proxmox_config) PROXMOX_CONFIG_RESOLVED="" # Auto-discover proxmox.yaml: --proxmox flag > script directory > cwd find_proxmox_config() { if [[ -n "$PROXMOX_CONFIG" ]]; then if [[ ! -f "$PROXMOX_CONFIG" ]]; then error "Proxmox config file not found: ${PROXMOX_CONFIG}" exit 1 fi PROXMOX_CONFIG_RESOLVED="$PROXMOX_CONFIG" detail "Using proxmox config: ${PROXMOX_CONFIG_RESOLVED} (--proxmox)" return 0 fi local script_dir script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" if [[ -f "${script_dir}/proxmox.yaml" ]]; then PROXMOX_CONFIG_RESOLVED="${script_dir}/proxmox.yaml" detail "Using proxmox config: ${PROXMOX_CONFIG_RESOLVED} (auto-discovered)" return 0 fi if [[ -f "proxmox.yaml" ]]; then PROXMOX_CONFIG_RESOLVED="$(pwd)/proxmox.yaml" detail "Using proxmox config: ${PROXMOX_CONFIG_RESOLVED} (auto-discovered)" return 0 fi # No proxmox.yaml found -- caller decides if this is an error return 1 } # Parse proxmox.yaml (flat key: value structure) and set PVE_* globals as base values load_proxmox_config() { local pxconfig="$1" local px_json px_json=$(mktemp "${WORKDIR}/proxmox-config-XXXXXX.json") if ! python3 -c " import json, sys, re result = {} with open(sys.argv[1]) as f: for line in f: s = line.strip() if not s or s.startswith('#'): continue m = re.match(r'^([^:]+?):\s*(.*?)$', s) if m: key = m.group(1).strip() val = m.group(2).strip() # Strip inline comments if val and not val.startswith(('\"', \"'\")): cm = re.match(r'^(.*?)\s+#.*$', val) if cm: val = cm.group(1).rstrip() if (val.startswith('\"') and val.endswith('\"')) or (val.startswith(\"'\") and val.endswith(\"'\")): val = val[1:-1] result[key] = val json.dump(result, sys.stdout, indent=2) " "$pxconfig" > "$px_json" 2>/dev/null; then error "Failed to parse proxmox config: ${pxconfig}" exit 1 fi # Set base values from proxmox.yaml (lab config overlay and CLI flags applied later) local val val=$(json_get "$px_json" ".host" ""); [[ -n "$val" ]] && PVE_HOST="$val" val=$(json_get "$px_json" ".node" ""); [[ -n "$val" ]] && PVE_NODE="$val" val=$(json_get "$px_json" ".storage" ""); [[ -n "$val" ]] && PVE_STORAGE="$val" val=$(json_get "$px_json" ".iso_storage" ""); [[ -n "$val" ]] && PVE_ISO_STORAGE="$val" val=$(json_get "$px_json" ".bridge" ""); [[ -n "$val" ]] && PVE_BRIDGE="$val" val=$(json_get "$px_json" ".vlan" ""); [[ -n "$val" ]] && PVE_VLAN="$val" val=$(json_get "$px_json" ".gateway" ""); [[ -n "$val" ]] && PVE_GATEWAY="$val" val=$(json_get "$px_json" ".dns" ""); [[ -n "$val" ]] && PVE_DNS="$val" val=$(json_get "$px_json" ".ssh_user" ""); [[ -n "$val" ]] && PVE_SSH_USER="$val" val=$(json_get "$px_json" ".api_token_id" ""); [[ -n "$val" ]] && PVE_API_TOKEN_ID="$val" val=$(json_get "$px_json" ".pool" ""); [[ -n "$val" ]] && PVE_POOL="$val" val=$(json_get "$px_json" ".method" ""); [[ -n "$val" ]] && PVE_METHOD="$val" rm -f "$px_json" } load_config() { step "Loading configuration values" # Merge priority: proxmox.yaml (base) → lab config proxmox: (overlay) → CLI flags # Step 1: proxmox.yaml base values were already loaded by the caller via # load_proxmox_config(). PVE_* globals may already be set. # Step 2: overlay from lab config proxmox: section (if present) local val val=$(json_get "$CONFIG_JSON" ".proxmox.host" ""); [[ -n "$val" ]] && PVE_HOST="$val" val=$(json_get "$CONFIG_JSON" ".proxmox.node" ""); [[ -n "$val" ]] && PVE_NODE="$val" val=$(json_get "$CONFIG_JSON" ".proxmox.storage" ""); [[ -n "$val" ]] && PVE_STORAGE="$val" val=$(json_get "$CONFIG_JSON" ".proxmox.iso_storage" ""); [[ -n "$val" ]] && PVE_ISO_STORAGE="$val" val=$(json_get "$CONFIG_JSON" ".proxmox.bridge" ""); [[ -n "$val" ]] && PVE_BRIDGE="$val" val=$(json_get "$CONFIG_JSON" ".proxmox.vlan" ""); [[ -n "$val" ]] && PVE_VLAN="$val" val=$(json_get "$CONFIG_JSON" ".proxmox.gateway" ""); [[ -n "$val" ]] && PVE_GATEWAY="$val" val=$(json_get "$CONFIG_JSON" ".proxmox.dns" ""); [[ -n "$val" ]] && PVE_DNS="$val" val=$(json_get "$CONFIG_JSON" ".proxmox.ssh_user" ""); [[ -n "$val" ]] && PVE_SSH_USER="$val" val=$(json_get "$CONFIG_JSON" ".proxmox.api_token_id" ""); [[ -n "$val" ]] && PVE_API_TOKEN_ID="$val" val=$(json_get "$CONFIG_JSON" ".proxmox.pool" ""); [[ -n "$val" ]] && PVE_POOL="$val" val=$(json_get "$CONFIG_JSON" ".proxmox.method" ""); [[ -n "$val" ]] && PVE_METHOD="$val" # Step 3: apply defaults for anything still unset [[ -z "$PVE_NODE" ]] && PVE_NODE="pve" [[ -z "$PVE_STORAGE" ]] && PVE_STORAGE="local-lvm" [[ -z "$PVE_ISO_STORAGE" ]] && PVE_ISO_STORAGE="local" [[ -z "$PVE_BRIDGE" ]] && PVE_BRIDGE="vmbr0" [[ -z "$PVE_SSH_USER" ]] && PVE_SSH_USER="root" [[ -z "$PVE_METHOD" ]] && PVE_METHOD="ssh" # Step 4: CLI flags override everything if [[ -n "$METHOD" ]]; then PVE_METHOD="$METHOD" fi if [[ -n "$HOST_OVERRIDE" ]]; then PVE_HOST="$HOST_OVERRIDE" fi # Default VM settings DEF_CORES=$(json_get "$CONFIG_JSON" ".defaults.cores" "4") DEF_MEMORY=$(json_get "$CONFIG_JSON" ".defaults.memory" "8192") DEF_DISK=$(json_get "$CONFIG_JSON" ".defaults.disk" "64") DEF_CHANNEL=$(json_get "$CONFIG_JSON" ".defaults.channel" "stable") DEF_START_VMID=$(json_get "$CONFIG_JSON" ".defaults.start_vmid" "200") # Load VMs local count count=$(vm_count) local i=0 while [[ $i -lt $count ]]; do VM_NAMES+=("$(vm_get "$i" "name" "")") VM_APPS+=("$(vm_get "$i" "app" "incus")") VM_APPLY_DEFAULTS+=("$(vm_get "$i" "apply_defaults" "false")") VM_CORES+=("$(vm_get "$i" "cores" "$DEF_CORES")") VM_MEMORY+=("$(vm_get "$i" "memory" "$DEF_MEMORY")") VM_DISK+=("$(vm_get "$i" "disk" "$DEF_DISK")") VM_IPS+=("$(vm_get "$i" "ip" "")") VM_VMIDS+=("$(vm_get "$i" "vmid" "")") VM_CLUSTERS+=("$(vm_get "$i" "cluster" "")") VM_CLUSTER_ROLES+=("$(vm_get "$i" "cluster_role" "")") i=$((i + 1)) done success "Loaded ${count} VM definition(s)" } # --------------------------------------------------------------------------- # Configuration validation # --------------------------------------------------------------------------- validate_config() { step "Validating configuration" local errors=0 # Required: Proxmox host if [[ -z "$PVE_HOST" ]]; then error "proxmox.host is required" errors=$((errors + 1)) fi # Method validation case "$PVE_METHOD" in ssh|api) ;; *) error "Invalid proxmox.method: ${PVE_METHOD} (must be ssh or api)" errors=$((errors + 1)) ;; esac # API method requires token if [[ "$PVE_METHOD" == "api" ]]; then if [[ -z "$PVE_API_TOKEN_ID" ]]; then error "proxmox.api_token_id is required for API method" error "Example: automation@pve!deploy" errors=$((errors + 1)) fi if [[ -z "${PROXMOX_TOKEN_SECRET:-}" ]]; then error "PROXMOX_TOKEN_SECRET is not set (no env file found in repo tree)" error "Create an env file or export directly (see --help)" errors=$((errors + 1)) fi fi # Must have at least one VM local count=${#VM_NAMES[@]} if [[ $count -eq 0 ]]; then error "No VMs defined in configuration" errors=$((errors + 1)) fi # Validate each VM local seen_names="" local cluster_inits="" local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" local app="${VM_APPS[$i]}" local memory="${VM_MEMORY[$i]}" local disk="${VM_DISK[$i]}" local cluster="${VM_CLUSTERS[$i]}" local role="${VM_CLUSTER_ROLES[$i]}" # Name required if [[ -z "$name" ]]; then error "VM #$((i + 1)): name is required" errors=$((errors + 1)) i=$((i + 1)) continue fi # Unique names if echo "$seen_names" | grep -qw "$name"; then error "VM '${name}': duplicate VM name" errors=$((errors + 1)) fi seen_names="${seen_names} ${name}" # Valid app case "$app" in incus|operations-center) ;; *) error "VM '${name}': invalid app '${app}' (must be incus or operations-center)" errors=$((errors + 1)) ;; esac # Minimum memory if [[ "$memory" -lt "$MIN_MEMORY_MIB" ]] 2>/dev/null; then error "VM '${name}': memory ${memory} MiB is below minimum ${MIN_MEMORY_MIB} MiB" errors=$((errors + 1)) fi # Minimum disk if [[ "$disk" -lt "$MIN_DISK_GIB" ]] 2>/dev/null; then error "VM '${name}': disk ${disk} GiB is below minimum ${MIN_DISK_GIB} GiB" errors=$((errors + 1)) fi # Cluster role validation if [[ -n "$role" ]] && [[ -z "$cluster" ]]; then error "VM '${name}': cluster_role '${role}' set but no cluster name specified" errors=$((errors + 1)) fi if [[ -n "$role" ]]; then case "$role" in init|join) ;; *) error "VM '${name}': invalid cluster_role '${role}' (must be init or join)" errors=$((errors + 1)) ;; esac fi # Static IP validation local vm_ip="${VM_IPS[$i]}" if [[ -n "$vm_ip" ]]; then if [[ "$vm_ip" != */* ]]; then error "VM '${name}': ip must include prefix (e.g. 192.168.102.100/22), got '${vm_ip}'" errors=$((errors + 1)) fi if [[ -z "$PVE_GATEWAY" ]]; then error "VM '${name}': static ip requires gateway in proxmox.yaml or lab config" errors=$((errors + 1)) fi fi # Track cluster init nodes if [[ -n "$cluster" ]] && [[ "$role" == "init" ]]; then if echo "$cluster_inits" | grep -qw "$cluster"; then error "Cluster '${cluster}': multiple init nodes defined (only one allowed)" errors=$((errors + 1)) fi cluster_inits="${cluster_inits} ${cluster}" fi i=$((i + 1)) done # VM filter validation if [[ -n "$VM_FILTER" ]]; then local found=false for name in "${VM_NAMES[@]}"; do if [[ "$name" == "$VM_FILTER" ]]; then found=true break fi done if [[ "$found" != true ]]; then error "VM '${VM_FILTER}' not found in configuration" error "Available VMs: ${VM_NAMES[*]}" errors=$((errors + 1)) fi fi if [[ $errors -gt 0 ]]; then error "${errors} validation error(s) found" exit 1 fi success "Configuration valid" } # --------------------------------------------------------------------------- # Proxmox transport layer # --------------------------------------------------------------------------- # Execute a command on the Proxmox host pve_run() { local cmd="$1" if [[ "$DRY_RUN" == true ]]; then info "[dry-run] ${PVE_METHOD}: ${cmd}" return 0 fi case "$PVE_METHOD" in ssh) ssh -o BatchMode=yes -o ConnectTimeout=10 \ "${PVE_SSH_USER}@${PVE_HOST}" "$cmd" ;; api) error "API execution of arbitrary commands is not supported" error "Use specific API functions instead" return 1 ;; esac } # Validate API auth prerequisites and connectivity. # Call this before any code path that uses pve_api() outside of phase_validate(). # Exits with error if credentials are missing or API is unreachable. require_api_auth() { if [[ "$PVE_METHOD" != "api" ]]; then return 0 fi if [[ -z "$PVE_API_TOKEN_ID" ]]; then error "API token ID is required for API method" error "Set proxmox.api_token_id in proxmox.yaml" exit 1 fi if [[ -z "${PROXMOX_TOKEN_SECRET:-}" ]]; then error "PROXMOX_TOKEN_SECRET is not set (no env file found in repo tree)" error "Create an env file: echo 'export PROXMOX_TOKEN_SECRET=\"\"' > env" error "Or export directly: export PROXMOX_TOKEN_SECRET=\"\"" exit 1 fi # Verify we can actually reach the API with these credentials local url="https://${PVE_HOST}:8006/api2/json/version" local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}" if ! curl -fsSk --connect-timeout 5 -H "$auth" "$url" &>/dev/null; then error "Cannot authenticate to Proxmox API at https://${PVE_HOST}:8006" error "Check that the host is reachable, token ID is correct, and secret is valid" exit 1 fi } # Execute a Proxmox API call (API method) # Usage: pve_api METHOD ENDPOINT [DATA] [json] # Pass "json" as 4th arg to send data as application/json instead of form-encoded. pve_api() { local method="$1" local endpoint="$2" local data="${3:-}" local content_type="${4:-form}" local url="https://${PVE_HOST}:8006/api2/json${endpoint}" local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET:-}" if [[ "$DRY_RUN" == true ]]; then info "[dry-run] API ${method} ${endpoint}" if [[ -n "$data" ]]; then detail "[dry-run] data: ${data}" fi return 0 fi local curl_args=(-fsSk -H "$auth") if [[ "$content_type" == "json" ]]; then curl_args+=(-H "Content-Type: application/json") fi case "$method" in GET) curl "${curl_args[@]}" "$url" ;; POST) if [[ -n "$data" ]]; then curl "${curl_args[@]}" -X POST -d "$data" "$url" else curl "${curl_args[@]}" -X POST "$url" fi ;; PUT) if [[ -n "$data" ]]; then curl "${curl_args[@]}" -X PUT -d "$data" "$url" else curl "${curl_args[@]}" -X PUT "$url" fi ;; DELETE) curl "${curl_args[@]}" -X DELETE "$url" ;; esac } # Upload a file to Proxmox ISO storage pve_upload() { local local_path="$1" local remote_filename="$2" if [[ "$DRY_RUN" == true ]]; then info "[dry-run] upload: ${local_path} -> ${PVE_ISO_STORAGE}:iso/${remote_filename}" return 0 fi case "$PVE_METHOD" in ssh) local iso_dir # Query the actual path of the ISO storage iso_dir=$(pve_run "pvesm path ${PVE_ISO_STORAGE}:iso/test 2>/dev/null" | sed 's|/test$||') || true if [[ -z "$iso_dir" ]]; then # Common default paths iso_dir="/var/lib/vz/template/iso" fi scp -o BatchMode=yes -o ConnectTimeout=10 \ "$local_path" "${PVE_SSH_USER}@${PVE_HOST}:${iso_dir}/${remote_filename}" ;; api) local upload_response upload_rc upload_response=$(curl -sSk \ -H "Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET:-}" \ -X POST \ -F "content=iso" \ -F "filename=@${local_path};filename=${remote_filename}" \ "https://${PVE_HOST}:8006/api2/json/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/upload" \ -w "\n%{http_code}" 2>&1) || true local http_code http_code=$(echo "$upload_response" | tail -1) local body body=$(echo "$upload_response" | sed '$d') if [[ "$http_code" -ge 200 ]] && [[ "$http_code" -lt 300 ]] 2>/dev/null; then return 0 else error "Upload failed (HTTP ${http_code})" # Extract error message from JSON response local api_error api_error=$(echo "$body" | python3 -c " import json, sys try: data = json.load(sys.stdin) errors = data.get('errors', {}) if errors: for k, v in errors.items(): print(f' {k}: {v}') elif data.get('message'): print(f' {data[\"message\"]}') else: print(json.dumps(data, indent=2)) except: print(sys.stdin.read() if hasattr(sys.stdin, 'read') else '') " 2>/dev/null) || api_error="$body" if [[ -n "$api_error" ]]; then error "Proxmox API response:" echo "$api_error" | while IFS= read -r line; do error " ${line}" done fi return 1 fi ;; esac } # Check if an ISO exists on Proxmox storage pve_iso_exists() { local filename="$1" if [[ "$DRY_RUN" == true ]]; then return 1 fi case "$PVE_METHOD" in ssh) pve_run "pvesm list ${PVE_ISO_STORAGE} --content iso 2>/dev/null" | grep -q "$filename" ;; api) pve_api GET "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content?content=iso" | \ grep -q "$filename" ;; esac } # Get list of existing VMIDs pve_list_vmids() { if [[ "$DRY_RUN" == true ]]; then echo "" return 0 fi case "$PVE_METHOD" in ssh) pve_run "qm list 2>/dev/null" | awk 'NR>1 {print $1}' || true ;; api) pve_api GET "/nodes/${PVE_NODE}/qemu" 2>/dev/null | \ python3 -c " import json, sys data = json.load(sys.stdin) for vm in data.get('data', []): print(vm.get('vmid', '')) " 2>/dev/null || true ;; esac } # Create a VM via qm create pve_create_vm() { local vmid="$1" local name="$2" local cores="$3" local memory="$4" local disk_gib="$5" local iso_filename="$6" local seed_filename="$7" local description description=$(build_vm_description) local cmd="qm create ${vmid}" cmd+=" --name ${name}" cmd+=" --description '${description}'" cmd+=" --ostype l26" cmd+=" --bios ovmf" cmd+=" --machine q35" cmd+=" --efidisk0 ${PVE_STORAGE}:0,efitype=4m,pre-enrolled-keys=0" cmd+=" --tpmstate0 ${PVE_STORAGE}:1,version=v2.0" cmd+=" --cpu host" cmd+=" --cores ${cores}" cmd+=" --memory ${memory}" cmd+=" --balloon 0" cmd+=" --scsihw virtio-scsi-pci" cmd+=" --scsi0 ${PVE_STORAGE}:${disk_gib},iothread=1" cmd+=" --ide2 ${PVE_ISO_STORAGE}:iso/${iso_filename},media=cdrom" cmd+=" --ide3 ${PVE_ISO_STORAGE}:iso/${seed_filename},media=cdrom" local net0_spec="virtio,bridge=${PVE_BRIDGE}" [[ -n "$PVE_VLAN" ]] && net0_spec+=",tag=${PVE_VLAN}" cmd+=" --net0 ${net0_spec}" cmd+=" --boot order=ide2\\;scsi0" cmd+=" --agent 1" if [[ -n "$PVE_POOL" ]]; then cmd+=" --pool ${PVE_POOL}" fi if [[ "$DRY_RUN" == true ]]; then info "[dry-run] ${cmd}" return 0 fi case "$PVE_METHOD" in ssh) pve_run "$cmd" ;; api) # Build JSON payload -- the Proxmox API requires JSON for VM # creation because comma-separated disk/net specs are misinterpreted # when sent as form-encoded data. local json_payload json_payload=$(python3 -c " import json payload = { 'vmid': ${vmid}, 'name': '${name}', 'description': '''${description}''', 'ostype': 'l26', 'bios': 'ovmf', 'machine': 'q35', 'efidisk0': '${PVE_STORAGE}:0,efitype=4m,pre-enrolled-keys=0', 'tpmstate0': '${PVE_STORAGE}:1,version=v2.0', 'cpu': 'host', 'cores': ${cores}, 'memory': ${memory}, 'balloon': 0, 'scsihw': 'virtio-scsi-pci', 'scsi0': '${PVE_STORAGE}:${disk_gib},iothread=1', 'ide2': '${PVE_ISO_STORAGE}:iso/${iso_filename},media=cdrom', 'ide3': '${PVE_ISO_STORAGE}:iso/${seed_filename},media=cdrom', 'boot': 'order=ide2;scsi0', 'agent': 1, } net0 = 'virtio,bridge=${PVE_BRIDGE}' vlan = '${PVE_VLAN}' if vlan: net0 += ',tag=' + vlan payload['net0'] = net0 pool = '${PVE_POOL}' if pool: payload['pool'] = pool print(json.dumps(payload)) ") local create_response create_response=$(pve_api POST "/nodes/${PVE_NODE}/qemu" "$json_payload" json) # VM creation is async -- the API returns a UPID. Wait for the # task to finish before returning, otherwise the caller may try # to start or configure the VM while it's still being created. local upid upid=$(echo "$create_response" | python3 -c "import json,sys; print(json.load(sys.stdin).get('data',''))" 2>/dev/null) || upid="" if [[ -n "$upid" ]]; then local encoded_upid encoded_upid=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${upid}', safe=''))" 2>/dev/null) local wait_elapsed=0 while [[ $wait_elapsed -lt 60 ]]; do sleep 2 wait_elapsed=$((wait_elapsed + 2)) local task_status task_status=$(pve_api GET "/nodes/${PVE_NODE}/tasks/${encoded_upid}/status" 2>/dev/null | \ python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status',''))" 2>/dev/null) || task_status="" if [[ "$task_status" == "stopped" ]]; then break fi done fi ;; esac } # Start a VM pve_start_vm() { local vmid="$1" case "$PVE_METHOD" in ssh) pve_run "qm start ${vmid}" ;; api) pve_api POST "/nodes/${PVE_NODE}/qemu/${vmid}/status/start" >/dev/null ;; esac } # Stop a VM pve_stop_vm() { local vmid="$1" case "$PVE_METHOD" in ssh) pve_run "qm stop ${vmid}" ;; api) pve_api POST "/nodes/${PVE_NODE}/qemu/${vmid}/status/stop" >/dev/null ;; esac } # Destroy a VM (waits for async task to complete) pve_destroy_vm() { local vmid="$1" case "$PVE_METHOD" in ssh) pve_run "qm destroy ${vmid} --purge" ;; api) local destroy_response destroy_response=$(pve_api DELETE "/nodes/${PVE_NODE}/qemu/${vmid}?purge=1&destroy-unreferenced-disks=1" 2>/dev/null) || return 1 # VM destruction is async -- wait for the UPID task to finish # to avoid VMID/disk reuse issues in rapid destroy+create cycles. local upid upid=$(echo "$destroy_response" | python3 -c "import json,sys; print(json.load(sys.stdin).get('data',''))" 2>/dev/null) || upid="" if [[ -n "$upid" ]]; then local encoded_upid encoded_upid=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${upid}', safe=''))" 2>/dev/null) local wait_elapsed=0 while [[ $wait_elapsed -lt 30 ]]; do sleep 2 wait_elapsed=$((wait_elapsed + 2)) local task_status task_status=$(pve_api GET "/nodes/${PVE_NODE}/tasks/${encoded_upid}/status" 2>/dev/null | \ python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status',''))" 2>/dev/null) || task_status="" if [[ "$task_status" == "stopped" ]]; then break fi done fi ;; esac } # Set VM configuration pve_set_vm() { local vmid="$1" shift local opts="$*" case "$PVE_METHOD" in ssh) pve_run "qm set ${vmid} ${opts}" ;; api) # Convert CLI-style opts (--key value) to JSON. # JSON avoids form-encoding ambiguity with values containing = and ; # Duplicate keys (e.g. --delete ide2 --delete ide3) are merged with commas. # Shell escapes (e.g. \;) are stripped since the API doesn't need them. local json_payload json_payload=$(echo "$opts" | python3 -c " import json, sys tokens = sys.stdin.read().split() obj = {} i = 0 while i < len(tokens): t = tokens[i] if t.startswith('--'): key = t.lstrip('-') if i + 1 < len(tokens) and not tokens[i+1].startswith('--'): val = tokens[i+1].replace(chr(92), '') i += 2 else: val = '' i += 1 if key in obj and obj[key]: obj[key] = obj[key] + ',' + val else: obj[key] = val else: i += 1 print(json.dumps(obj)) " 2>/dev/null) pve_api PUT "/nodes/${PVE_NODE}/qemu/${vmid}/config" "$json_payload" json >/dev/null ;; esac } # Get VM status pve_vm_status() { local vmid="$1" case "$PVE_METHOD" in ssh) pve_run "qm status ${vmid}" | awk '{print $2}' ;; api) pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" | \ python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status','unknown'))" 2>/dev/null || echo "unknown" ;; esac } # Get VM uptime (seconds); returns 0 if stopped or unknown pve_vm_uptime() { local vmid="$1" case "$PVE_METHOD" in ssh) pve_run "qm status ${vmid} --verbose" | awk '/^uptime:/{print $2}' ;; api) pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" | \ python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('uptime',0))" 2>/dev/null || echo "0" ;; esac } # Get cumulative write bytes for scsi0 (from QEMU blockstat). # Returns 0 if unavailable. Only works with api method. pve_vm_disk_writes() { local vmid="$1" case "$PVE_METHOD" in ssh) echo "0" ;; api) pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" | \ python3 -c " import json,sys d = json.load(sys.stdin).get('data',{}) bs = d.get('blockstat',{}).get('scsi0',{}) print(bs.get('wr_bytes',0)) " 2>/dev/null || echo "0" ;; esac } # Check if a VMID exists pve_vm_exists() { local vmid="$1" if [[ "$DRY_RUN" == true ]]; then return 1 fi case "$PVE_METHOD" in ssh) pve_run "qm status ${vmid}" &>/dev/null ;; api) pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" &>/dev/null ;; esac } # Get VM's IP address (best-effort via QEMU guest agent) pve_vm_ip() { local vmid="$1" local known_ip="${2:-}" # If static IP is configured, return bare IP (strip CIDR prefix) if [[ -n "$known_ip" ]]; then echo "${known_ip%%/*}" return 0 fi # Method 1: Try QEMU guest agent (works if agent is installed) local ip="" case "$PVE_METHOD" in ssh) ip=$(pve_run "qm guest cmd ${vmid} network-get-interfaces 2>/dev/null" | \ python3 -c " import json, sys try: data = json.load(sys.stdin) for iface in data: if iface.get('name') == 'lo': continue for addr in iface.get('ip-addresses', []): if addr.get('ip-address-type') == 'ipv4': print(addr['ip-address']) sys.exit(0) except: pass " 2>/dev/null) || true ;; api) ip=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/agent/network-get-interfaces" 2>/dev/null | \ python3 -c " import json, sys try: data = json.load(sys.stdin).get('data', {}).get('result', []) for iface in data: if iface.get('name') == 'lo': continue for addr in iface.get('ip-addresses', []): if addr.get('ip-address-type') == 'ipv4': print(addr['ip-address']) sys.exit(0) except: pass " 2>/dev/null) || true ;; esac if [[ -n "$ip" ]]; then echo "$ip" return 0 fi # Method 2: ARP-based lookup using VM's MAC address. # Works without guest agent (e.g. IncusOS which is immutable). # Get MAC from VM config, flush stale ARP, ping sweep, look up. local mac="" case "$PVE_METHOD" in ssh) mac=$(pve_run "qm config ${vmid}" 2>/dev/null | \ awk -F'=' '/^net0:/{print $1}' | grep -oiE '[0-9a-f:]{17}') || true ;; api) mac=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null | \ python3 -c " import json, sys, re try: cfg = json.load(sys.stdin).get('data', {}) net0 = cfg.get('net0', '') m = re.search(r'([0-9A-Fa-f:]{17})', net0) if m: print(m.group(1).lower()) except: pass " 2>/dev/null) || true ;; esac if [[ -z "$mac" ]]; then return 1 fi # Populate ARP table to find this MAC's IP address. # Strategy: first check existing ARP, then try broadcast ping, then # fall back to a parallel ping sweep of the local subnet. ip neigh flush nud stale 2>/dev/null || true local bridge_subnet bridge_subnet=$(ip -4 addr show | awk '/inet / && !/127\.0\.0/ {print $2; exit}') # Try broadcast ping first (fast, but many hosts ignore it) if [[ -n "$bridge_subnet" ]]; then local broadcast broadcast=$(python3 -c "import ipaddress; print(ipaddress.ip_interface('${bridge_subnet}').network.broadcast_address)" 2>/dev/null) || true if [[ -n "$broadcast" ]]; then ping -c 2 -W 1 -b "$broadcast" &>/dev/null || true sleep 1 fi fi # Check ARP table ip=$(ip neigh show | awk -v mac="$mac" 'tolower($0) ~ tolower(mac) && (/REACHABLE/ || /DELAY/ || /STALE/) {print $1; exit}') || true if [[ -n "$ip" ]] && ping -c 1 -W 1 "$ip" &>/dev/null; then echo "$ip" return 0 fi # Broadcast didn't work — parallel ping sweep of the subnet if [[ -n "$bridge_subnet" ]]; then local net_prefix net_prefix=$(python3 -c " import ipaddress net = ipaddress.ip_interface('${bridge_subnet}').network for h in net.hosts(): print(h) " 2>/dev/null) if [[ -n "$net_prefix" ]]; then # Ping all hosts in parallel (background, 1s timeout each) while IFS= read -r host_ip; do ping -c 1 -W 1 "$host_ip" &>/dev/null & done <<< "$net_prefix" wait sleep 1 # Re-check ARP table ip=$(ip neigh show | awk -v mac="$mac" 'tolower($0) ~ tolower(mac) && (/REACHABLE/ || /DELAY/) {print $1; exit}') || true if [[ -n "$ip" ]] && ping -c 1 -W 1 "$ip" &>/dev/null; then echo "$ip" return 0 fi fi fi return 1 } # --------------------------------------------------------------------------- # VM management marker # --------------------------------------------------------------------------- # Build the description string for a managed VM build_vm_description() { local config_basename config_basename=$(basename "$CONFIG_FILE") local timestamp timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ") local marker="${MANAGED_MARKER} config=${config_basename} created=${timestamp} by=${SCRIPT_NAME}/${VERSION}" if [[ -n "$PVE_POOL" ]]; then marker+=" pool=${PVE_POOL}" fi echo "$marker" } # Check if a VM has our management marker in its description pve_vm_is_managed() { local vmid="$1" if [[ "$DRY_RUN" == true ]]; then return 1 fi local config_output="" case "$PVE_METHOD" in ssh) config_output=$(pve_run "qm config ${vmid}" 2>/dev/null) || return 1 ;; api) config_output=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null) || return 1 ;; esac echo "$config_output" | grep -qF "$MANAGED_MARKER" } # Check if a VM still has install media (ide2) attached. # Returns 0 if install media is present, 1 if not (already installed). pve_vm_has_install_media() { local vmid="$1" if [[ "$DRY_RUN" == true ]]; then return 1 fi local config_output="" case "$PVE_METHOD" in ssh) config_output=$(pve_run "qm config ${vmid}" 2>/dev/null) || return 1 ;; api) config_output=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null) || return 1 ;; esac echo "$config_output" | grep -q "ide2" } # List all VMs as "vmid name" pairs (one per line) # Returns non-zero and prints error message on API/SSH failure. pve_list_vms() { if [[ "$DRY_RUN" == true ]]; then echo "" return 0 fi local raw_response # When a resource pool is configured, list only pool members (type=qemu) if [[ -n "$PVE_POOL" ]]; then case "$PVE_METHOD" in ssh) if ! raw_response=$(pve_run "pvesh get /pools/${PVE_POOL} --output-format json 2>/dev/null" 2>/dev/null); then echo "[error] Failed to list VMs via SSH" >&2 return 1 fi echo "$raw_response" | python3 -c " import json, sys data = json.load(sys.stdin) for m in data.get('members', []): if m.get('type') == 'qemu': print('{} {}'.format(m.get('vmid', ''), m.get('name', ''))) " 2>/dev/null || true ;; api) if ! raw_response=$(pve_api GET "/pools/${PVE_POOL}" 2>/dev/null); then echo "[error] Failed to list VMs from pool '${PVE_POOL}' via API (auth failure or pool not found)" >&2 return 1 fi echo "$raw_response" | python3 -c " import json, sys data = json.load(sys.stdin).get('data', {}) for m in data.get('members', []): if m.get('type') == 'qemu': print('{} {}'.format(m.get('vmid', ''), m.get('name', ''))) " 2>/dev/null || true ;; esac return 0 fi case "$PVE_METHOD" in ssh) pve_run "qm list 2>/dev/null" | awk 'NR>1 {print $1, $2}' || true ;; api) if ! raw_response=$(pve_api GET "/nodes/${PVE_NODE}/qemu" 2>/dev/null); then echo "[error] Failed to list VMs via API (auth failure or node not found)" >&2 return 1 fi echo "$raw_response" | python3 -c " import json, sys data = json.load(sys.stdin) for vm in data.get('data', []): print('{} {}'.format(vm.get('vmid', ''), vm.get('name', ''))) " 2>/dev/null || true ;; esac } # --------------------------------------------------------------------------- # Pre-flight checks # --------------------------------------------------------------------------- preflight_checks() { step "Running pre-flight checks" local errors=0 local warnings=0 # 1. Check that storage pools exist on Proxmox if [[ "$DRY_RUN" != true ]]; then case "$PVE_METHOD" in ssh) if ! pve_run "pvesm status 2>/dev/null" | awk 'NR>1 {print \$1}' | grep -qw "$PVE_STORAGE"; then error "Storage pool '${PVE_STORAGE}' not found on Proxmox" error "Available pools: $(pve_run "pvesm status 2>/dev/null" | awk 'NR>1 {print \$1}' | tr '\n' ' ')" error "Fix: update 'proxmox.storage' in your config file" errors=$((errors + 1)) fi if ! pve_run "pvesm status 2>/dev/null" | awk 'NR>1 {print \$1}' | grep -qw "$PVE_ISO_STORAGE"; then error "ISO storage '${PVE_ISO_STORAGE}' not found on Proxmox" error "Available pools: $(pve_run "pvesm status 2>/dev/null" | awk 'NR>1 {print \$1}' | tr '\n' ' ')" error "Fix: update 'proxmox.iso_storage' in your config file" errors=$((errors + 1)) fi ;; api) local storage_json storage_json=$(pve_api GET "/nodes/${PVE_NODE}/storage" 2>/dev/null) || storage_json="" local storage_list storage_list=$(echo "$storage_json" | python3 -c " import json, sys data = json.load(sys.stdin) for s in data.get('data', []): print(s.get('storage', '')) " 2>/dev/null) || storage_list="" if ! echo "$storage_list" | grep -qw "$PVE_STORAGE"; then error "Storage pool '${PVE_STORAGE}' not found on Proxmox" error "Available pools: $(echo "$storage_list" | tr '\n' ' ')" error "Fix: update 'proxmox.storage' in your config file" errors=$((errors + 1)) fi if ! echo "$storage_list" | grep -qw "$PVE_ISO_STORAGE"; then error "ISO storage '${PVE_ISO_STORAGE}' not found on Proxmox" error "Available pools: $(echo "$storage_list" | tr '\n' ' ')" error "Fix: update 'proxmox.iso_storage' in your config file" errors=$((errors + 1)) else # Check that ISO storage accepts 'iso' content type local iso_content iso_content=$(echo "$storage_json" | python3 -c " import json, sys data = json.load(sys.stdin) for s in data.get('data', []): if s.get('storage') == '${PVE_ISO_STORAGE}': print(s.get('content', '')) break " 2>/dev/null) || iso_content="" if [[ -n "$iso_content" ]] && ! echo "$iso_content" | grep -q "iso"; then error "Storage '${PVE_ISO_STORAGE}' does not accept ISO content (content types: ${iso_content})" error "Fix: enable 'iso' content on '${PVE_ISO_STORAGE}' in Proxmox, or use a different iso_storage" errors=$((errors + 1)) fi fi ;; esac # 2. Network bridge check (SSH only, warn-level) if [[ "$PVE_METHOD" == "ssh" ]]; then if ! pve_run "ip link show ${PVE_BRIDGE} 2>/dev/null" &>/dev/null; then warn "Network bridge '${PVE_BRIDGE}' not found on Proxmox" warn "VMs may fail to get network connectivity" warn "Fix: update 'proxmox.bridge' in your config file" warnings=$((warnings + 1)) fi fi # 3. ISO availability: check if already on Proxmox or CDN reachable if [[ -z "$LOCAL_ISO" ]]; then local any_iso_present=false local iso_list="" case "$PVE_METHOD" in ssh) iso_list=$(pve_run "pvesm list ${PVE_ISO_STORAGE} --content iso 2>/dev/null" 2>/dev/null) || iso_list="" ;; api) iso_list=$(pve_api GET "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content?content=iso" 2>/dev/null) || iso_list="" ;; esac if echo "$iso_list" | grep -q "IncusOS_"; then any_iso_present=true fi if [[ "$any_iso_present" != true ]]; then # No ISO on Proxmox -- check if CDN is reachable if ! curl -fsSL --connect-timeout 5 -o /dev/null "$CDN_INDEX" 2>/dev/null; then error "No IncusOS ISO found on Proxmox and CDN is unreachable: ${CDN_INDEX}" error "Fix: provide a local ISO with --iso FILE, or check internet connectivity" errors=$((errors + 1)) fi fi fi fi # 4. Resource pool check (if configured) if [[ -n "$PVE_POOL" ]] && [[ "$DRY_RUN" != true ]]; then local pool_ok=false case "$PVE_METHOD" in ssh) if pve_run "pvesh get /pools/${PVE_POOL} --output-format json 2>/dev/null" &>/dev/null; then pool_ok=true fi ;; api) if pve_api GET "/pools/${PVE_POOL}" &>/dev/null; then pool_ok=true fi ;; esac if [[ "$pool_ok" != true ]]; then error "Resource pool '${PVE_POOL}' not found on Proxmox" error "Create it with these commands on the Proxmox host:" error " pveum pool add ${PVE_POOL} --comment \"IncusOS Lab VMs\"" error " pveum pool modify ${PVE_POOL} --storage ${PVE_STORAGE},${PVE_ISO_STORAGE}" if [[ "$PVE_METHOD" == "api" ]] && [[ -n "$PVE_API_TOKEN_ID" ]]; then local api_user api_user=$(echo "$PVE_API_TOKEN_ID" | cut -d'!' -f1) error " pveum aclmod /pool/${PVE_POOL} -user ${api_user} -role IncusOSDeployer" fi errors=$((errors + 1)) else detail "Resource pool '${PVE_POOL}' exists" fi fi # 5. ISO creation tool for SEED_DATA ISO 9660 generation if ! command -v genisoimage &>/dev/null && ! command -v mkisofs &>/dev/null; then error "An ISO creation tool is required for SEED_DATA generation but not found" case "$(uname -s)" in Darwin) error "Install with: brew install cdrtools" ;; *) error "Install with: sudo apt install genisoimage" ;; esac errors=$((errors + 1)) fi # 6. Host resource feasibility (API method only) if [[ "$PVE_METHOD" == "api" ]] && [[ "$DRY_RUN" != true ]] && [[ ${#VM_MEMORY[@]} -gt 0 ]]; then local total_ram_mib=0 local total_disk_gib=0 local total_cores=0 local ri=0 while [[ $ri -lt ${#VM_MEMORY[@]} ]]; do total_ram_mib=$((total_ram_mib + VM_MEMORY[$ri])) total_disk_gib=$((total_disk_gib + VM_DISK[$ri])) total_cores=$((total_cores + VM_CORES[$ri])) ri=$((ri + 1)) done local res_node_status res_node_status=$(pve_api GET "/nodes/${PVE_NODE}/status" 2>/dev/null) || res_node_status="" if [[ -n "$res_node_status" ]]; then local resource_lines resource_lines=$(python3 -c " import json, sys data = json.loads('''${res_node_status}''').get('data', {}) mem = data.get('memory', {}) mem_total = mem.get('total', 0) / (1024**3) mem_used = mem.get('used', 0) / (1024**3) mem_free = mem_total - mem_used cpu_count = data.get('cpuinfo', {}).get('cpus', 0) req_ram = ${total_ram_mib} / 1024 req_cores = ${total_cores} # RAM check if mem_free > 0: pct = req_ram / mem_free * 100 if req_ram > mem_free: print(f'error|RAM: {req_ram:.1f} GiB requested, {mem_free:.1f} GiB free -- not enough RAM') elif pct > 70: print(f'warn|RAM: {req_ram:.1f} GiB requested, {mem_free:.1f} GiB free ({pct:.0f}% of free) -- tight fit') else: print(f'ok|RAM: {req_ram:.1f} GiB requested, {mem_free:.1f} GiB free ({pct:.0f}% of free)') # CPU (info only) print(f'ok|CPU: {req_cores} cores requested, {cpu_count} available') " 2>/dev/null) || resource_lines="" local line while IFS= read -r line; do [[ -z "$line" ]] && continue local level="${line%%|*}" local msg="${line#*|}" case "$level" in error) error "$msg" errors=$((errors + 1)) ;; warn) warn "$msg" warnings=$((warnings + 1)) ;; ok) detail "$msg" ;; esac done <<< "$resource_lines" fi fi if [[ $errors -gt 0 ]]; then error "${errors} pre-flight check(s) failed" exit 1 fi if [[ $warnings -gt 0 ]]; then success "Pre-flight checks passed with ${warnings} warning(s)" else success "Pre-flight checks passed" fi } # --------------------------------------------------------------------------- # Connectivity check # --------------------------------------------------------------------------- check_connectivity() { step "Checking Proxmox connectivity (${PVE_METHOD})" case "$PVE_METHOD" in ssh) if ! ssh -o BatchMode=yes -o ConnectTimeout=10 \ "${PVE_SSH_USER}@${PVE_HOST}" "pvesh get /version --output-format json" &>/dev/null; then error "Cannot connect to Proxmox host via SSH" error "Ensure key-based SSH auth is set up:" error " ssh-copy-id ${PVE_SSH_USER}@${PVE_HOST}" error "Test with: ssh ${PVE_SSH_USER}@${PVE_HOST} pvesh get /version" exit 1 fi local pve_version pve_version=$(ssh -o BatchMode=yes "${PVE_SSH_USER}@${PVE_HOST}" \ "pvesh get /version --output-format json" 2>/dev/null | \ python3 -c "import json,sys; print(json.load(sys.stdin).get('version','unknown'))" 2>/dev/null) || pve_version="unknown" success "Connected to ${PVE_HOST} (Proxmox VE ${pve_version})" ;; api) local url="https://${PVE_HOST}:8006/api2/json/version" local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET:-}" local response if ! response=$(curl -fsSk -H "$auth" "$url" 2>/dev/null); then error "Cannot connect to Proxmox API at https://${PVE_HOST}:8006" error "Check that:" error " 1. The host is reachable" error " 2. The API token ID is correct: ${PVE_API_TOKEN_ID}" error " 3. PROXMOX_TOKEN_SECRET is set" exit 1 fi local pve_version pve_version=$(echo "$response" | \ python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('version','unknown'))" 2>/dev/null) || pve_version="unknown" success "Connected to ${PVE_HOST} (Proxmox VE ${pve_version})" ;; esac } # --------------------------------------------------------------------------- # VMID allocation # --------------------------------------------------------------------------- allocate_vmids() { step "Allocating VMIDs" # Get existing VMs as "vmid name" pairs local vm_list vm_list=$(pve_list_vms) # Build lookup: existing IDs and name-to-VMID map local existing_ids="" declare -A existing_name_to_vmid if [[ -n "$vm_list" ]]; then while IFS=' ' read -r eid ename; do [[ -z "$eid" ]] && continue existing_ids="${existing_ids} ${eid}" if [[ -n "$ename" ]]; then existing_name_to_vmid["$ename"]="$eid" fi done <<< "$vm_list" fi local next_id="$DEF_START_VMID" local count=${#VM_NAMES[@]} local errors=0 local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" # Skip if filtered if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi # Check for name collision with existing Proxmox VMs if [[ -n "${existing_name_to_vmid[$name]+x}" ]]; then local collision_vmid="${existing_name_to_vmid[$name]}" if pve_vm_is_managed "$collision_vmid"; then warn "VM '${name}' already exists at VMID ${collision_vmid} (managed by ${SCRIPT_NAME})" warn "Use --cleanup to redeploy, or remove it manually" # Assign the existing VMID so cleanup/skip works correctly VM_VMIDS[$i]="$collision_vmid" i=$((i + 1)) continue else error "VM name '${name}' already exists at VMID ${collision_vmid} but is NOT managed by ${SCRIPT_NAME}" error "This VM was not created by this tool and will not be modified" error "Fix: rename the VM in your config file, or manually remove VMID ${collision_vmid}:" error " ssh ${PVE_SSH_USER}@${PVE_HOST} qm destroy ${collision_vmid} --purge" errors=$((errors + 1)) i=$((i + 1)) continue fi fi if [[ -n "${VM_VMIDS[$i]}" ]]; then # Explicit VMID detail "VM '${name}': VMID ${VM_VMIDS[$i]} (explicit)" else # Auto-assign, skipping existing while echo "$existing_ids" | grep -qw "$next_id" 2>/dev/null; do next_id=$((next_id + 1)) done VM_VMIDS[$i]="$next_id" detail "VM '${name}': VMID ${next_id} (auto-assigned)" next_id=$((next_id + 1)) fi i=$((i + 1)) done if [[ $errors -gt 0 ]]; then error "${errors} name collision(s) with unmanaged VMs -- cannot continue" exit 1 fi success "VMIDs allocated" } # --------------------------------------------------------------------------- # Certificate detection (reused pattern from sibling scripts) # --------------------------------------------------------------------------- CERT_FILE_RESOLVED="" detect_client_cert() { if [[ "$NO_CERT" == true ]]; then info "Skipping certificate injection (--no-cert)" return fi if [[ -n "$CERT_FILE" ]]; then if [[ ! -f "$CERT_FILE" ]]; then error "Certificate file not found: ${CERT_FILE}" exit 1 fi if ! grep -q "BEGIN CERTIFICATE" "$CERT_FILE"; then error "File does not appear to be a PEM certificate: ${CERT_FILE}" exit 1 fi CERT_FILE_RESOLVED="$CERT_FILE" success "Certificate: ${CERT_FILE_RESOLVED}" return fi step "Auto-detecting client certificate" local cert_paths=( "${HOME}/.config/incus/client.crt" "${HOME}/snap/incus/common/config/client.crt" "${HOME}/.config/lxc/client.crt" ) for path in "${cert_paths[@]}"; do if [[ -f "$path" ]] && grep -q "BEGIN CERTIFICATE" "$path"; then CERT_FILE_RESOLVED="$path" success "Certificate: ${CERT_FILE_RESOLVED}" return fi done # Try CLI fallback if command -v incus &>/dev/null; then local tmpfile="${WORKDIR}/client.crt" if incus remote get-client-certificate > "$tmpfile" 2>/dev/null; then CERT_FILE_RESOLVED="$tmpfile" success "Certificate obtained via Incus CLI" return fi fi # Check if any VM is ops-center (which requires a cert) local needs_cert=false for app in "${VM_APPS[@]}"; do if [[ "$app" == "operations-center" ]]; then needs_cert=true break fi done if [[ "$needs_cert" == true ]]; then error "No client certificate found" error "Operations Center requires at least one trusted certificate for access" error "" error "Options:" error " --cert FILE Provide a certificate file directly" error " Generate one: incus remote list (creates ~/.config/incus/client.crt)" error " --no-cert Skip certificate injection (you will be locked out!)" exit 1 fi warn "No client certificate found -- VMs will require manual trust setup" warn "Use --cert FILE to provide a certificate, or --no-cert to suppress this warning" } # --------------------------------------------------------------------------- # CDN operations (reused from incusos-iso) # --------------------------------------------------------------------------- LATEST_VERSION="" fetch_latest_version() { local channel="${DEF_CHANNEL:-stable}" step "Fetching latest IncusOS version (channel: ${channel})" local index_file="${WORKDIR}/cdn-index.json" if ! curl -fsSL "$CDN_INDEX" -o "$index_file" 2>/dev/null; then error "Failed to fetch CDN index from ${CDN_INDEX}" error "Check your internet connection and try again" exit 1 fi if command -v jq &>/dev/null; then LATEST_VERSION=$(jq -r \ --arg ch "$channel" \ '[.updates[] | select(.channels[] == $ch)] | sort_by(.version) | last | .version // empty' \ "$index_file") else LATEST_VERSION=$(python3 -c " import json data = json.load(open('${index_file}')) versions = [u['version'] for u in data.get('updates', []) if '${channel}' in u.get('channels', [])] if versions: print(sorted(versions)[-1]) " 2>/dev/null) fi if [[ -z "$LATEST_VERSION" ]]; then error "No version found for channel '${channel}'" exit 1 fi success "Latest IncusOS version: ${LATEST_VERSION}" } download_iso() { local version="$1" local output_path="$2" local iso_filename iso_filename=$(basename "$output_path") # Check persistent cache first local cached_iso="${CACHEDIR}/${iso_filename}" if [[ -f "$cached_iso" ]]; then success "ISO found in local cache: ${cached_iso}" cp "$cached_iso" "$output_path" return 0 fi local url="${CDN_BASE}/${version}/x86_64/IncusOS_${version}.iso.gz" local gz_path="${cached_iso}.gz" step "Downloading IncusOS ISO" detail "URL: ${url}" detail "Cache directory: ${CACHEDIR}" if ! curl -fL --progress-bar "$url" -o "$gz_path"; then error "Failed to download ISO from ${url}" rm -f "$gz_path" exit 1 fi step "Decompressing ISO" if ! gzip -d "$gz_path"; then local avail avail=$(df -h --output=avail "$(dirname "$gz_path")" 2>/dev/null | tail -1 | tr -d ' ') || avail="unknown" error "Failed to decompress ISO (${avail} available on disk)" error "The decompressed ISO requires several GiB of disk space" error "Free up space, or set TMPDIR to a larger filesystem:" error " TMPDIR=/path/with/space ${SCRIPT_NAME} ${CONFIG_FILE}" rm -f "$gz_path" exit 1 fi # Copy from cache to workdir cp "$cached_iso" "$output_path" success "ISO ready: ${output_path} (cached for future runs)" } # --------------------------------------------------------------------------- # Deployment plan summary # --------------------------------------------------------------------------- print_plan() { local count=${#VM_NAMES[@]} echo "" echo -e "${BOLD}Deployment Plan${RESET}" echo -e " Proxmox host: ${CYAN}${PVE_HOST}${RESET} (${PVE_METHOD})" echo -e " Node: ${PVE_NODE}" echo -e " Storage: ${PVE_STORAGE} (disk), ${PVE_ISO_STORAGE} (iso)" if [[ -n "$PVE_VLAN" ]]; then echo -e " Bridge: ${PVE_BRIDGE} (VLAN ${PVE_VLAN})" else echo -e " Bridge: ${PVE_BRIDGE}" fi if [[ -n "$PVE_GATEWAY" ]]; then echo -e " Gateway: ${PVE_GATEWAY}" fi if [[ -n "$PVE_POOL" ]]; then echo -e " Pool: ${CYAN}${PVE_POOL}${RESET}" fi if [[ -n "$CERT_FILE_RESOLVED" ]]; then echo -e " Certificate: ${GREEN}${CERT_FILE_RESOLVED}${RESET}" elif [[ "$NO_CERT" == true ]]; then echo -e " Certificate: ${DIM}none (--no-cert)${RESET}" else echo -e " Certificate: ${YELLOW}not found${RESET}" fi echo "" echo -e " ${BOLD}VMs (${count}):${RESET}" local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" local skip="" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then skip=" ${DIM}(skipped)${RESET}" fi echo -e " ${CYAN}${name}${RESET}${skip}" echo -e " VMID: ${VM_VMIDS[$i]} App: ${VM_APPS[$i]} Defaults: ${VM_APPLY_DEFAULTS[$i]}" echo -e " CPU: ${VM_CORES[$i]} cores RAM: ${VM_MEMORY[$i]} MiB Disk: ${VM_DISK[$i]} GiB" if [[ -n "${VM_IPS[$i]}" ]]; then echo -e " IP: ${GREEN}${VM_IPS[$i]}${RESET} (static)" fi if [[ -n "${VM_CLUSTERS[$i]}" ]]; then echo -e " Cluster: ${VM_CLUSTERS[$i]} (${VM_CLUSTER_ROLES[$i]})" fi i=$((i + 1)) done echo "" } # --------------------------------------------------------------------------- # Confirmation prompt # --------------------------------------------------------------------------- confirm() { local message="$1" if [[ "$YES" == true ]] || [[ "$DRY_RUN" == true ]]; then return 0 fi echo -n -e "${BOLD}${message} [y/N] ${RESET}" local reply read -r reply if [[ ! "$reply" =~ ^[Yy]$ ]]; then info "Aborted" exit 0 fi } # Like confirm(), but returns 0/1 instead of exiting on "no" confirm_yn() { local message="$1" if [[ "$YES" == true ]]; then return 0 fi echo -n -e "${BOLD}${message} [y/N] ${RESET}" local reply read -r reply [[ "$reply" =~ ^[Yy]$ ]] } # --------------------------------------------------------------------------- # Phase 1: Validate # --------------------------------------------------------------------------- phase_validate() { step "Phase 1: Validate" echo "" # Load proxmox.yaml base values (before lab config overlay) if find_proxmox_config; then load_proxmox_config "$PROXMOX_CONFIG_RESOLVED" fi parse_config load_config validate_config check_connectivity allocate_vmids preflight_checks detect_client_cert print_plan } # --------------------------------------------------------------------------- # Phase 2: Prepare (download ISO + generate seeds) # --------------------------------------------------------------------------- ISO_FILENAME="" phase_prepare() { step "Phase 2: Prepare" echo "" # Determine ISO if [[ -n "$LOCAL_ISO" ]]; then ISO_FILENAME=$(basename "$LOCAL_ISO") info "Using local ISO: ${LOCAL_ISO}" if [[ "$DRY_RUN" != true ]]; then cp "$LOCAL_ISO" "${WORKDIR}/${ISO_FILENAME}" fi elif [[ "$DRY_RUN" == true ]]; then ISO_FILENAME="IncusOS_latest.iso" info "[dry-run] Would fetch latest version from CDN and download ISO" else # Check if ISO already exists on Proxmox fetch_latest_version ISO_FILENAME="IncusOS_${LATEST_VERSION}.iso" if pve_iso_exists "$ISO_FILENAME"; then success "ISO already on Proxmox: ${ISO_FILENAME} (reusing from previous deployment)" else download_iso "$LATEST_VERSION" "${WORKDIR}/${ISO_FILENAME}" fi fi # Generate per-VM seed images step "Generating seed images" local count=${#VM_NAMES[@]} local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" # Skip if filtered if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local app="${VM_APPS[$i]}" local apply_defaults="${VM_APPLY_DEFAULTS[$i]}" local seed_output="${WORKDIR}/seed-${name}.iso" detail "Generating seed for '${name}' (${app}, defaults=${apply_defaults})" # Build incusos-seed command local seed_cmd=("$SEED_TOOL") seed_cmd+=(--format iso) seed_cmd+=(--app "$app") seed_cmd+=(--output "$seed_output") seed_cmd+=(--force-install) # NOTE: --force-reboot intentionally omitted. On Proxmox, we control # the VM externally (stop/start). force_reboot triggers SysRq-B which # causes an "install media detected" intermediate boot — this triggers # the IncusOS state.txt race condition (issue #843) ~50% of the time. # Without force_reboot, the installer sits at "please remove media", # blockstat detects idle, and we cleanly stop/remove media/start from disk. seed_cmd+=(--hostname "$name") seed_cmd+=(--quiet) if [[ "$apply_defaults" == "true" ]]; then seed_cmd+=(--defaults) fi if [[ -n "$CERT_FILE_RESOLVED" ]]; then seed_cmd+=(--cert "$CERT_FILE_RESOLVED") elif [[ "$NO_CERT" == true ]]; then seed_cmd+=(--no-cert) fi # Static IP: pass --ip, --gateway, --dns to seed generator local vm_ip="${VM_IPS[$i]}" if [[ -n "$vm_ip" ]]; then seed_cmd+=(--ip "$vm_ip") seed_cmd+=(--gateway "$PVE_GATEWAY") if [[ -n "$PVE_DNS" ]]; then seed_cmd+=(--dns "$PVE_DNS") fi fi if [[ "$DRY_RUN" == true ]]; then info "[dry-run] ${seed_cmd[*]}" else if ! "${seed_cmd[@]}"; then error "Failed to generate seed for VM '${name}'" exit 1 fi fi i=$((i + 1)) done success "Seed images generated" } # --------------------------------------------------------------------------- # Phase 3: Upload # --------------------------------------------------------------------------- phase_upload() { step "Phase 3: Upload" echo "" # Upload ISO (if we downloaded it) local iso_path="${WORKDIR}/${ISO_FILENAME}" if [[ -f "$iso_path" ]]; then if pve_iso_exists "$ISO_FILENAME"; then if [[ -n "$LOCAL_ISO" ]]; then # --iso was provided: always replace the ISO on Proxmox. # A stale ISO with the same filename (e.g., IncusOS-oc.iso from # a previous OC deployment) would silently use the wrong version. warn "ISO '${ISO_FILENAME}' already exists on Proxmox -- replacing with provided ISO" local encoded_volid encoded_volid=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${PVE_ISO_STORAGE}:iso/${ISO_FILENAME}', safe=''))") pve_api DELETE "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content/${encoded_volid}" &>/dev/null || true step "Uploading ISO: ${ISO_FILENAME}" if ! pve_upload "$iso_path" "$ISO_FILENAME"; then error "Failed to upload ISO to Proxmox" exit 1 fi success "ISO uploaded: ${ISO_FILENAME} (replaced)" else success "ISO already on Proxmox: ${ISO_FILENAME}" fi else step "Uploading ISO: ${ISO_FILENAME}" if ! pve_upload "$iso_path" "$ISO_FILENAME"; then error "Failed to upload ISO to Proxmox" exit 1 fi success "ISO uploaded: ${ISO_FILENAME}" fi else detail "ISO already on Proxmox, skipping upload" fi # Upload seed images local count=${#VM_NAMES[@]} local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local seed_file="seed-${name}.iso" local seed_path="${WORKDIR}/${seed_file}" if [[ -f "$seed_path" ]] || [[ "$DRY_RUN" == true ]]; then step "Uploading seed: ${seed_file}" if ! pve_upload "$seed_path" "$seed_file"; then error "Failed to upload seed for VM '${name}'" exit 1 fi success "Seed uploaded: ${seed_file}" fi i=$((i + 1)) done success "All files uploaded" } # --------------------------------------------------------------------------- # Phase 4: Create VMs # --------------------------------------------------------------------------- phase_create() { step "Phase 4: Create VMs" echo "" local count=${#VM_NAMES[@]} local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local vmid="${VM_VMIDS[$i]}" local cores="${VM_CORES[$i]}" local memory="${VM_MEMORY[$i]}" local disk="${VM_DISK[$i]}" local seed_file="seed-${name}.iso" step "Creating VM '${name}' (VMID ${vmid})" # Check if VM already exists if pve_vm_exists "$vmid"; then if pve_vm_is_managed "$vmid"; then warn "VM ${vmid} ('${name}') already exists (managed) -- skipping" warn "Use --cleanup to redeploy" else warn "VM ${vmid} ('${name}') already exists (NOT managed by ${SCRIPT_NAME}) -- skipping" warn "Choose a different VMID or remove it manually:" warn " ssh ${PVE_SSH_USER}@${PVE_HOST} qm destroy ${vmid} --purge" fi i=$((i + 1)) continue fi detail "cores=${cores} memory=${memory}MiB disk=${disk}GiB" detail "bios=ovmf machine=q35 cpu=host scsihw=virtio-scsi-pci" detail "efidisk: pre-enrolled-keys=0, tpm: v2.0, balloon: 0" detail "iso=${ISO_FILENAME} seed=${seed_file}" if ! pve_create_vm "$vmid" "$name" "$cores" "$memory" "$disk" \ "$ISO_FILENAME" "$seed_file"; then error "Failed to create VM '${name}' (VMID ${vmid})" exit 1 fi success "VM '${name}' created (VMID ${vmid})" i=$((i + 1)) done success "All VMs created" } # --------------------------------------------------------------------------- # Remote registration # --------------------------------------------------------------------------- # Register an incus remote for a deployed VM. # Handles stale remotes: if a remote exists but points to a different IP, # it is removed and re-added with the correct IP (DHCP may reassign addresses # between deploys). # Usage: register_remote register_remote() { local name="$1" local ip="$2" # Only register if incus client is available and a cert was injected if ! command -v incus &>/dev/null; then return fi if [[ -z "$CERT_FILE_RESOLVED" ]]; then return fi # Check if remote already exists local existing_url="" existing_url=$(incus remote list --format csv 2>/dev/null | \ awk -F',' -v name="$name" '$1 == name {print $2; exit}') || true if [[ -n "$existing_url" ]]; then # Remote exists -- check if the IP matches if echo "$existing_url" | grep -q "${ip}"; then detail "Remote '${name}' already configured at ${ip}" return fi # Stale remote -- IP changed (DHCP reassignment between deploys) if [[ "$DRY_RUN" == true ]]; then info "[dry-run] Would update stale remote '${name}': ${existing_url} -> ${ip}" return fi warn "Remote '${name}' points to ${existing_url} but VM is at ${ip} -- updating" # Switch away first -- can't remove the current/default remote local current_remote="" current_remote=$(incus remote get-default 2>/dev/null) || true if [[ "$current_remote" == "$name" ]]; then incus remote switch local 2>/dev/null || true fi incus remote remove "$name" 2>/dev/null || true fi if [[ "$DRY_RUN" == true ]]; then info "[dry-run] Would add incus remote '${name}' at ${ip}" return fi # Wait for port 8443 to become reachable (up to 30s) local attempt=0 while [[ $attempt -lt 6 ]]; do if curl -sk --connect-timeout 3 "https://${ip}:8443" &>/dev/null; then break fi sleep 5 attempt=$((attempt + 1)) done if incus remote add "$name" "$ip" --accept-certificate --quiet 2>/dev/null; then success "Remote '${name}' added (${ip})" else warn "Could not add remote '${name}' at ${ip} -- add manually:" warn " incus remote add ${name} ${ip} --accept-certificate" fi } # --------------------------------------------------------------------------- # fix_scrub_schedule -- proactively fix empty scrub_schedule via IncusOS API # --------------------------------------------------------------------------- # IncusOS has an intermittent bug where state.txt is written with an empty # scrub_schedule field (the encoder skips zero values). When registerJobs() # runs, gocron.IsValid("") fails and the daemon crashes in a loop. The REST # API is briefly reachable during each cycle (Incus starts before the # scheduler), so we can fix it via PUT /os/1.0/system/storage. # # This function is safe to call on every node -- if scrub_schedule is already # set correctly, the GET check returns early without making any changes. # # Usage: fix_scrub_schedule # Returns: 0 on success or not needed, 1 on failure fix_scrub_schedule() { local ip="$1" local name="$2" # Need client cert for the IncusOS API if [[ -z "$CERT_FILE_RESOLVED" ]]; then detail "No client certificate available — skipping scrub_schedule check" return 0 fi local cert="$CERT_FILE_RESOLVED" local key="${CERT_FILE_RESOLVED%.crt}.key" if [[ ! -f "$key" ]]; then detail "Client key not found at ${key} — skipping scrub_schedule check" return 0 fi # GET current storage config local response response=$(curl -sk --connect-timeout 5 --max-time 10 \ --cert "$cert" --key "$key" \ "https://${ip}:8443/os/1.0/system/storage" 2>/dev/null) || { detail "Could not query IncusOS storage API on '${name}' — skipping scrub_schedule check" return 0 } # Check if scrub_schedule is empty or missing local current_schedule current_schedule=$(printf '%s' "$response" | \ python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('metadata',{}).get('config',{}).get('scrub_schedule',''))" 2>/dev/null) || current_schedule="" if [[ -n "$current_schedule" ]]; then detail "scrub_schedule on '${name}' is already set: ${current_schedule}" return 0 fi # scrub_schedule is empty — fix it local put_response put_response=$(curl -sk --connect-timeout 5 --max-time 10 \ --cert "$cert" --key "$key" \ -X PUT -H "Content-Type: application/json" \ -d '{"config":{"scrub_schedule":"0 4 * * 0"}}' \ "https://${ip}:8443/os/1.0/system/storage" 2>/dev/null) || { warn "Failed to fix scrub_schedule on '${name}' via IncusOS API" return 1 } info "Fixed empty scrub_schedule on '${name}' via IncusOS API" return 0 } # --------------------------------------------------------------------------- # Phase 5: Install # --------------------------------------------------------------------------- phase_install() { step "Phase 5: Install" echo "" local count=${#VM_NAMES[@]} local install_failures=0 local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local vmid="${VM_VMIDS[$i]}" if [[ "$DRY_RUN" == true ]]; then step "Starting VM '${name}' (VMID ${vmid})" info "[dry-run] Would start VM, swap boot order, wait for install, clean up" i=$((i + 1)) continue fi # Idempotency: check current VM state before acting if pve_vm_exists "$vmid"; then local cur_status cur_status=$(pve_vm_status "$vmid") || cur_status="unknown" local has_media=true pve_vm_has_install_media "$vmid" || has_media=false if [[ "$cur_status" == "running" ]] && [[ "$has_media" == false ]]; then # Already installed and running — detect IP, register remote, skip success "VM '${name}' (VMID ${vmid}) is already installed and running" local ip ip=$(pve_vm_ip "$vmid" "${VM_IPS[$i]:-}") || ip="" if [[ -n "$ip" ]]; then detail "IP: ${ip}" register_remote "$name" "$ip" fi i=$((i + 1)) continue fi if [[ "$cur_status" == "stopped" ]] && [[ "$has_media" == false ]]; then # Installed but stopped — start from disk step "VM '${name}' is installed but stopped — starting from disk" if pve_start_vm "$vmid"; then sleep 30 local ip ip=$(pve_vm_ip "$vmid" "${VM_IPS[$i]:-}") || ip="" if [[ -n "$ip" ]]; then success "VM '${name}' started at ${ip}" register_remote "$name" "$ip" else success "VM '${name}' started from disk" fi else error "Failed to start VM '${name}' from disk" fi i=$((i + 1)) continue fi if [[ "$cur_status" == "running" ]] && [[ "$has_media" == true ]]; then # Mid-install or stuck — fall through to blockstat monitoring info "VM '${name}' is running with install media — monitoring installation" fi # stopped + has_media → needs install, fall through fi step "Starting VM '${name}' (VMID ${vmid})" # 1. Start VM (boots from ISO, reads SEED_DATA, installs) if ! pve_start_vm "$vmid"; then error "Failed to start VM '${name}' (VMID ${vmid})" i=$((i + 1)) continue fi success "VM '${name}' started" # 2. Wait for installation to complete by monitoring disk writes # # Install flow: boot ISO -> read SEED_DATA -> install to disk -> idle # (no force_reboot: VM sits at "please remove installation media"). # We monitor scsi0 write bytes via blockstat: # - During install: wr_bytes increases steadily (~876 MiB for image clone) # - After install: wr_bytes stops (VM is idle at "remove media" prompt) # # When wr_bytes has been stable for STABLE_COUNT consecutive polls, # installation is complete. We then stop the VM, remove install media, # and start cleanly from disk. step "Waiting for '${name}' to complete installation" info "Monitoring disk write activity (polling every 5s, timeout: 10 minutes)" local timeout=600 local interval=5 local elapsed=0 local install_done=false local prev_wr_bytes=0 local writes_started=false local stable_count=0 local STABLE_THRESHOLD=3 # 3 consecutive polls with no change = 15s idle while [[ $elapsed -lt $timeout ]]; do sleep "$interval" elapsed=$((elapsed + interval)) local status status=$(pve_vm_status "$vmid") || status="unknown" case "$status" in stopped) detail "VM stopped [${elapsed}s]" install_done=true break ;; running) local wr_bytes wr_bytes=$(pve_vm_disk_writes "$vmid") || wr_bytes=0 if [[ $wr_bytes -gt $prev_wr_bytes ]]; then # Disk writes are happening -- install in progress writes_started=true stable_count=0 local wr_mb=$(( wr_bytes / 1048576 )) # Show progress every 15s if [[ $((elapsed % 15)) -eq 0 ]]; then info "Installing... (${wr_mb} MiB written) [${elapsed}s]" fi elif [[ "$writes_started" == true ]]; then # Writes have stopped after being active stable_count=$((stable_count + 1)) if [[ $stable_count -ge $STABLE_THRESHOLD ]]; then local wr_mb=$(( wr_bytes / 1048576 )) info "Disk activity stopped (${wr_mb} MiB written total) [${elapsed}s]" install_done=true break fi fi prev_wr_bytes=$wr_bytes ;; *) detail "VM status: ${status} [${elapsed}s]" ;; esac done if [[ "$install_done" != true ]]; then warn "VM '${name}' did not complete installation within ${timeout}s" warn "Check the VM console on Proxmox for details" i=$((i + 1)) continue fi # 3. Stop the VM (it's either at "remove install media" prompt or # post-reboot "install media detected" error -- either way idle). # Poll until confirmed stopped — do not proceed on assumption. local cur_status cur_status=$(pve_vm_status "$vmid") || cur_status="unknown" if [[ "$cur_status" == "running" ]]; then info "Stopping VM to remove install media..." pve_stop_vm "$vmid" || true local stop_wait=0 while [[ $stop_wait -lt 30 ]]; do cur_status=$(pve_vm_status "$vmid") || cur_status="unknown" [[ "$cur_status" == "stopped" ]] && break sleep 2 stop_wait=$((stop_wait + 2)) done if [[ "$cur_status" != "stopped" ]]; then warn "VM '${name}' did not stop within 30s (status: ${cur_status})" install_failures=$((install_failures + 1)) i=$((i + 1)) continue fi detail "VM stopped after ${stop_wait}s" fi # 4. Remove install media and set clean boot order. # Verify removal succeeded by re-reading VM config. step "Removing install media from '${name}'" pve_set_vm "$vmid" "--delete ide2 --delete ide3 --boot order=scsi0" || true sleep 2 # Verify ide2 and ide3 are gone if pve_vm_has_install_media "$vmid"; then warn "Install media still attached after removal — retrying" pve_set_vm "$vmid" "--delete ide2" || true pve_set_vm "$vmid" "--delete ide3" || true sleep 2 if pve_vm_has_install_media "$vmid"; then error "Failed to remove install media from '${name}' after retry" install_failures=$((install_failures + 1)) i=$((i + 1)) continue fi fi success "Install media detached, boot order set to disk-only" # 5. Start VM from disk step "Starting '${name}' from disk" if ! pve_start_vm "$vmid"; then error "Failed to start VM '${name}' from disk" i=$((i + 1)) continue fi # 6. Wait for services and verify health (port 8443 + scrub_schedule) # # The health check has two levels: # 1. Port 8443 reachable (Incus API is up) # 2. scrub_schedule is set (IncusOS daemon fully initialized) # # Port 8443 reachable but scrub_schedule empty = crontab bug hit. # The IncusOS REST API starts BEFORE registerJobs(), so port 8443 # can respond even when the daemon is about to crash. We use # scrub_schedule as the definitive health indicator and trigger # retries (not just fixes) when it's empty. info "Waiting for services to start..." # First boot downloads SecureBoot update + application sysext (30-120s). # Total first-boot time is typically 60-150s. Wait 60s before polling. sleep 60 local ip ip=$(pve_vm_ip "$vmid" "${VM_IPS[$i]:-}") || ip="" local healthy=false if [[ -n "$ip" ]]; then # Poll port 8443 for up to 120s (the 60s sleep above + 120s = 180s total) local poll=0 while [[ $poll -lt 24 ]]; do if curl -sk --connect-timeout 3 "https://${ip}:8443" &>/dev/null; then # Port is up — check and fix scrub_schedule. # fix_scrub_schedule returns 1 ONLY when it detects empty # scrub_schedule AND the PUT to fix it fails. In that case, # the crontab bug hit and we should retry (not declare success). # Returns 0 for: no cert (can't check), already set, or fixed. local fix_result=0 fix_scrub_schedule "$ip" "$name" || fix_result=$? if [[ $fix_result -eq 0 ]]; then healthy=true else warn "VM '${name}': crontab bug detected (scrub_schedule empty, fix failed)" fi break fi sleep 5 poll=$((poll + 1)) done fi echo "" if [[ "$healthy" == true ]]; then success "VM '${name}' installed and running at ${ip}" register_remote "$name" "$ip" else # Retry loop: if port 8443 didn't come up within 180s, something # went wrong during first boot. A stop+start cycle can help with # transient failures. CAUTION: hard-stopping a VM during first boot # can corrupt the TPM encryption key — only retry if genuinely stuck. local retry=0 local retry_ok=false while [[ $retry -lt $INSTALL_RETRIES ]]; do retry=$((retry + 1)) if [[ -n "$ip" ]]; then warn "VM '${name}': port 8443 not reachable at ${ip} — retry ${retry}/${INSTALL_RETRIES} (stop+start)" else warn "VM '${name}': could not detect IP — retry ${retry}/${INSTALL_RETRIES} (stop+start)" fi info "Stopping VM for retry..." pve_stop_vm "$vmid" || true # Poll until confirmed stopped (not just sleep) local retry_stop_wait=0 while [[ $retry_stop_wait -lt 30 ]]; do local retry_status retry_status=$(pve_vm_status "$vmid") || retry_status="unknown" [[ "$retry_status" == "stopped" ]] && break sleep 2 retry_stop_wait=$((retry_stop_wait + 2)) done info "Starting VM (retry ${retry}/${INSTALL_RETRIES})..." pve_start_vm "$vmid" || true sleep 60 # Re-detect IP (may have changed after restart) ip=$(pve_vm_ip "$vmid" "${VM_IPS[$i]:-}") || ip="" if [[ -n "$ip" ]]; then local retry_poll=0 while [[ $retry_poll -lt 24 ]]; do if curl -sk --connect-timeout 3 "https://${ip}:8443" &>/dev/null; then # Port is up — check and fix scrub_schedule local retry_fix=0 fix_scrub_schedule "$ip" "$name" || retry_fix=$? if [[ $retry_fix -eq 0 ]]; then retry_ok=true fi break fi sleep 5 retry_poll=$((retry_poll + 1)) done fi if [[ "$retry_ok" == true ]]; then success "VM '${name}' recovered after retry ${retry}/${INSTALL_RETRIES} — running at ${ip}" register_remote "$name" "$ip" break fi done if [[ "$retry_ok" != true ]]; then if [[ -n "$ip" ]]; then warn "VM '${name}' installed but port 8443 not reachable at ${ip} (after ${INSTALL_RETRIES} retries)" warn "Likely IncusOS boot failure (check Proxmox console for errors)" warn "Redeploy this VM with: ${SCRIPT_NAME} --cleanup --yes ${CONFIG_FILE} && ${SCRIPT_NAME} --yes ${CONFIG_FILE}" else warn "VM '${name}' installed but could not detect IP (after ${INSTALL_RETRIES} retries)" warn "Check Proxmox console for status" fi install_failures=$((install_failures + 1)) fi fi echo "" i=$((i + 1)) done if [[ $install_failures -gt 0 ]]; then warn "Installation phase complete: ${install_failures} VM(s) failed health check" warn "Failed VMs are likely stuck in an IncusOS boot loop" warn "Re-run deployment to redeploy, or use --cleanup to start fresh" else success "Installation phase complete" fi } # --------------------------------------------------------------------------- # Lab lifecycle (--lab-up, --lab-down) # --------------------------------------------------------------------------- phase_lab_down() { step "Stopping lab VMs" echo "" local count=${#VM_NAMES[@]} local stopped=0 local skipped=0 local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" local vmid="${VM_VMIDS[$i]}" if [[ "$DRY_RUN" == true ]]; then info "Would stop ${name} (VMID ${vmid})" i=$((i + 1)) continue fi if ! pve_vm_exists "$vmid"; then warn "${name} (VMID ${vmid}): not found -- skipping" skipped=$((skipped + 1)) i=$((i + 1)) continue fi local status status=$(pve_vm_status "$vmid") || status="unknown" if [[ "$status" == "running" ]]; then info "Stopping ${name} (VMID ${vmid})..." if pve_stop_vm "$vmid"; then success "${name} stopped" stopped=$((stopped + 1)) else error "Failed to stop ${name}" fi else detail "${name}: already ${status}" skipped=$((skipped + 1)) fi i=$((i + 1)) done echo "" if [[ "$DRY_RUN" == true ]]; then info "Dry run -- no VMs were stopped" else success "Lab down: ${stopped} stopped, ${skipped} skipped" fi } phase_lab_up() { step "Starting lab VMs" echo "" local count=${#VM_NAMES[@]} local started=0 local skipped=0 local missing=0 # First pass: check how many VMs exist if [[ "$DRY_RUN" != true ]]; then local i=0 while [[ $i -lt $count ]]; do if ! pve_vm_exists "${VM_VMIDS[$i]}"; then missing=$((missing + 1)) fi i=$((i + 1)) done # If ALL VMs are missing, offer to deploy if [[ $missing -eq $count ]]; then echo "" warn "No VMs found -- lab has not been deployed yet" if confirm_yn "Deploy lab now?"; then echo "" phase_prepare echo "" phase_upload echo "" phase_create echo "" phase_install echo "" phase_post_deploy return else info "Aborted" return fi fi # If SOME VMs are missing, warn but don't offer auto-deploy if [[ $missing -gt 0 ]]; then warn "${missing} of ${count} VMs not found -- partial state detected" warn "Use --cleanup then redeploy, or deploy missing VMs manually" fi fi # Second pass: start existing VMs local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" local vmid="${VM_VMIDS[$i]}" if [[ "$DRY_RUN" == true ]]; then info "Would start ${name} (VMID ${vmid})" i=$((i + 1)) continue fi if ! pve_vm_exists "$vmid"; then warn "${name} (VMID ${vmid}): not found -- skipping" skipped=$((skipped + 1)) i=$((i + 1)) continue fi local status status=$(pve_vm_status "$vmid") || status="unknown" if [[ "$status" == "stopped" ]]; then # Safety check: refuse to start VMs with install media still attached if pve_vm_has_install_media "$vmid"; then warn "${name}: install media still attached -- skipping (run full deploy instead)" skipped=$((skipped + 1)) i=$((i + 1)) continue fi info "Starting ${name} (VMID ${vmid})..." if pve_start_vm "$vmid"; then success "${name} started" started=$((started + 1)) else error "Failed to start ${name}" fi else detail "${name}: already ${status}" skipped=$((skipped + 1)) fi i=$((i + 1)) done if [[ "$DRY_RUN" == true ]]; then echo "" info "Dry run -- no VMs were started" return fi echo "" success "Lab up: ${started} started, ${skipped} skipped" if [[ $started -gt 0 ]]; then info "Waiting 30s for services to start..." sleep 30 echo "" phase_status fi } # --------------------------------------------------------------------------- # Resource and lab inventory (--resources, --labs) # --------------------------------------------------------------------------- do_resources() { echo -e "${BOLD}${SCRIPT_NAME} resources${RESET}" echo "" echo -e " ${BOLD}Proxmox Host:${RESET} ${PVE_HOST} (node: ${PVE_NODE})" echo "" # Host resources (API only -- SSH would need parsing pvesh output) if [[ "$PVE_METHOD" != "api" ]]; then warn "Resource details require API method (current: ${PVE_METHOD})" warn "Use --method api or set method: api in proxmox.yaml" return fi # Host memory and CPU local node_status node_status=$(pve_api GET "/nodes/${PVE_NODE}/status" 2>/dev/null) || node_status="" if [[ -n "$node_status" ]]; then python3 -c " import json, sys data = json.loads('''${node_status}''').get('data', {}) mem = data.get('memory', {}) cpu_count = data.get('cpuinfo', {}).get('cpus', '?') cpu_model = data.get('cpuinfo', {}).get('model', 'unknown') mem_used = mem.get('used', 0) / (1024**3) mem_total = mem.get('total', 0) / (1024**3) mem_free = mem_total - mem_used mem_pct = (mem_used / mem_total * 100) if mem_total > 0 else 0 print(f' RAM: {mem_used:.1f} GiB / {mem_total:.1f} GiB used ({mem_pct:.0f}%), {mem_free:.1f} GiB free') print(f' CPUs: {cpu_count} cores ({cpu_model})') uptime_s = data.get('uptime', 0) days = uptime_s // 86400 hours = (uptime_s % 86400) // 3600 print(f' Uptime: {days}d {hours}h') " 2>/dev/null || warn "Failed to parse node status" fi # Storage echo "" echo -e " ${BOLD}Storage:${RESET}" local storage_list storage_list=$(pve_api GET "/nodes/${PVE_NODE}/storage" 2>/dev/null) || storage_list="" if [[ -n "$storage_list" ]]; then python3 -c " import json, sys data = json.loads('''${storage_list}''').get('data', []) thin_types = {'zfspool', 'lvmthin'} for s in sorted(data, key=lambda x: x.get('storage', '')): name = s.get('storage', '?') total = s.get('total', 0) / (1024**3) used = s.get('used', 0) / (1024**3) avail = s.get('avail', 0) / (1024**3) pct = (used / total * 100) if total > 0 else 0 stype = s.get('type', '?') content = s.get('content', '?') thin = ' [thin]' if stype in thin_types else '' if total > 0: print(f' {name:<16} {stype:<12} {used:.1f}G / {total:.1f}G ({pct:.0f}%) -- {content}{thin}') else: print(f' {name:<16} {stype:<12} (no capacity info) -- {content}{thin}') " 2>/dev/null || warn "Failed to parse storage list" fi # Per-pool VM summary (if pool is configured) if [[ -n "$PVE_POOL" ]]; then echo "" echo -e " ${BOLD}Pool: ${PVE_POOL}${RESET}" local pool_data pool_data=$(pve_api GET "/pools/${PVE_POOL}" 2>/dev/null) || pool_data="" if [[ -n "$pool_data" ]]; then python3 -c " import json, sys data = json.loads('''${pool_data}''').get('data', {}) members = data.get('members', []) vms = [m for m in members if m.get('type') == 'qemu'] running = sum(1 for v in vms if v.get('status') == 'running') stopped = sum(1 for v in vms if v.get('status') == 'stopped') total_mem = sum(v.get('maxmem', 0) for v in vms) / (1024**3) total_disk = sum(v.get('maxdisk', 0) for v in vms) / (1024**3) print(f' VMs: {len(vms)} total ({running} running, {stopped} stopped)') print(f' Allocated: {total_mem:.1f} GiB RAM, {total_disk:.0f} GiB disk') " 2>/dev/null || warn "Failed to parse pool data" fi # Actual disk usage via storage content API if [[ -n "$pool_data" ]]; then local storage_content storage_content=$(pve_api GET "/nodes/${PVE_NODE}/storage/${PVE_STORAGE}/content" 2>/dev/null) || storage_content="" local storage_status_json storage_status_json=$(echo "$storage_list" | python3 -c " import json, sys data = json.load(sys.stdin).get('data', []) for s in data: if s.get('storage') == '${PVE_STORAGE}': print(json.dumps(s)) break " 2>/dev/null) || storage_status_json="" if [[ -n "$storage_content" ]] && [[ -n "$pool_data" ]]; then python3 -c " import json, sys content = json.loads('''${storage_content}''').get('data', []) pool = json.loads('''${pool_data}''').get('data', {}) vm_vmids = {str(m.get('vmid', '')) for m in pool.get('members', []) if m.get('type') == 'qemu'} # Sum actual (used) bytes for volumes belonging to pool VMs actual_bytes = 0 allocated_bytes = 0 for vol in content: vmid = str(vol.get('vmid', '')) if vmid in vm_vmids: actual_bytes += vol.get('used', 0) allocated_bytes += vol.get('size', 0) if allocated_bytes > 0: actual_gib = actual_bytes / (1024**3) allocated_gib = allocated_bytes / (1024**3) ratio = (actual_bytes / allocated_bytes * 100) if allocated_bytes > 0 else 0 print(f' Actual: {actual_gib:.1f} GiB disk used ({ratio:.0f}% of allocated, thin provisioned)') " 2>/dev/null || true fi # Storage type note if [[ -n "$storage_status_json" ]]; then python3 -c " import json s = json.loads('''${storage_status_json}''') stype = s.get('type', '') notes = {'zfspool': 'ZFS (thin provisioned, lz4 compression)', 'lvmthin': 'LVM-thin (thin provisioned)'} if stype in notes: print(f' Storage: {notes[stype]}') " 2>/dev/null || true fi fi fi echo "" } do_list_labs() { echo -e "${BOLD}${SCRIPT_NAME} labs${RESET}" echo "" if [[ "$PVE_METHOD" != "api" ]]; then warn "Lab listing requires API method (current: ${PVE_METHOD})" warn "Use --method api or set method: api in proxmox.yaml" return fi # Get all managed VMs and group by config file local all_vms if [[ -n "$PVE_POOL" ]]; then all_vms=$(pve_api GET "/pools/${PVE_POOL}" 2>/dev/null) || all_vms="" else all_vms=$(pve_api GET "/nodes/${PVE_NODE}/qemu" 2>/dev/null) || all_vms="" fi if [[ -z "$all_vms" ]]; then info "No VMs found" return fi # Collect managed VMs and group by config= tag in description local vmids=() if [[ -n "$PVE_POOL" ]]; then # Pool response has members[] with type=qemu mapfile -t vmids < <(python3 -c " import json data = json.loads('''${all_vms}''').get('data', {}) for m in data.get('members', []): if m.get('type') == 'qemu': print(m.get('vmid', '')) " 2>/dev/null) else mapfile -t vmids < <(python3 -c " import json data = json.loads('''${all_vms}''').get('data', []) for v in data: print(v.get('vmid', '')) " 2>/dev/null) fi # For each VM, get config and check for managed marker declare -A lab_configs=() # config_name -> "vmid1:name1:status1 vmid2:name2:status2 ..." declare -A lab_ram=() # config_name -> total RAM in MiB declare -A lab_disk=() # config_name -> total disk in GiB local unmanaged=0 for vmid in "${vmids[@]}"; do [[ -z "$vmid" ]] && continue local config_output config_output=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null) || continue local name="" mem_mib="" disk_gib="" desc="" status { read -r name read -r mem_mib read -r disk_gib read -r desc } < <(python3 -c " import json data = json.loads('''${config_output}''').get('data', {}) desc = data.get('description', '') name = data.get('name', 'unknown') mem = int(data.get('memory', 0)) # Rough disk from scsi0 size scsi0 = data.get('scsi0', '') disk_g = 0 for part in scsi0.split(','): if part.strip().startswith('size='): sz = part.strip().split('=')[1] if sz.endswith('G'): disk_g = int(sz[:-1]) print(name) print(mem) print(disk_g) print(desc) " 2>/dev/null || true) # Check for managed marker if [[ "$desc" != *"$MANAGED_MARKER"* ]]; then unmanaged=$((unmanaged + 1)) continue fi # Extract config= tag local config_tag="unknown" if [[ "$desc" =~ config=([^\ ]+) ]]; then config_tag="${BASH_REMATCH[1]}" fi # Get VM status status=$(pve_vm_status "$vmid") || status="unknown" # Accumulate lab_configs["$config_tag"]="${lab_configs[$config_tag]:-} ${vmid}:${name}:${status}" lab_ram["$config_tag"]=$(( ${lab_ram[$config_tag]:-0} + ${mem_mib:-0} )) lab_disk["$config_tag"]=$(( ${lab_disk[$config_tag]:-0} + ${disk_gib:-0} )) done if [[ ${#lab_configs[@]} -eq 0 ]]; then info "No managed labs found" [[ $unmanaged -gt 0 ]] && detail "${unmanaged} unmanaged VMs in pool" return 0 fi local sorted_configs sorted_configs=$(printf '%s\n' "${!lab_configs[@]}" | sort) for config in $sorted_configs; do local vms="${lab_configs[$config]}" local ram_mib="${lab_ram[$config]}" local disk_gib="${lab_disk[$config]}" local vm_count=0 local running=0 local stopped=0 echo -e " ${BOLD}${config}${RESET}" for entry in $vms; do [[ -z "$entry" ]] && continue local v_vmid v_name v_status IFS=':' read -r v_vmid v_name v_status <<< "$entry" vm_count=$((vm_count + 1)) [[ "$v_status" == "running" ]] && running=$((running + 1)) [[ "$v_status" == "stopped" ]] && stopped=$((stopped + 1)) local status_color="${DIM}" [[ "$v_status" == "running" ]] && status_color="${GREEN}" [[ "$v_status" == "stopped" ]] && status_color="${YELLOW}" echo -e " ${v_name} (${v_vmid}): ${status_color}${v_status}${RESET}" done local lab_status="${GREEN}all running${RESET}" if [[ $running -eq 0 ]] && [[ $stopped -gt 0 ]]; then lab_status="${YELLOW}all stopped${RESET}" elif [[ $running -gt 0 ]] && [[ $stopped -gt 0 ]]; then lab_status="${YELLOW}mixed${RESET}" fi echo -e " Status: ${lab_status} | RAM: $((ram_mib / 1024))G | Disk: ${disk_gib}G" echo "" done [[ $unmanaged -gt 0 ]] && detail "${unmanaged} unmanaged VMs in pool" return 0 } # --------------------------------------------------------------------------- # Status check # --------------------------------------------------------------------------- phase_status() { step "Deployment Status" echo "" local count=${#VM_NAMES[@]} local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local vmid="${VM_VMIDS[$i]}" local app="${VM_APPS[$i]}" echo -e " ${BOLD}${name}${RESET} (VMID ${vmid}, app: ${app})" # Proxmox state if [[ "$DRY_RUN" == true ]]; then echo -e " Proxmox: ${DIM}[dry-run]${RESET}" i=$((i + 1)) echo "" continue fi if ! pve_vm_exists "$vmid"; then echo -e " Proxmox: ${RED}not found${RESET}" i=$((i + 1)) echo "" continue fi local managed=false pve_vm_is_managed "$vmid" && managed=true local status status=$(pve_vm_status "$vmid") || status="unknown" local has_media=false pve_vm_has_install_media "$vmid" && has_media=true local install_state="installed" if [[ "$has_media" == true ]]; then install_state="not installed (media attached)" fi local status_color="$GREEN" if [[ "$status" != "running" ]]; then status_color="$YELLOW" fi echo -e " Proxmox: exists managed: ${managed} status: ${status_color}${status}${RESET} ${install_state}" # Network and service checks (only for running, installed VMs) if [[ "$status" == "running" ]] && [[ "$has_media" == false ]]; then local ip ip=$(pve_vm_ip "$vmid" "${VM_IPS[$i]:-}") || ip="" if [[ -n "$ip" ]]; then local port_status="${RED}closed${RESET}" if curl -sk --connect-timeout 5 "https://${ip}:8443" &>/dev/null; then port_status="${GREEN}open${RESET}" fi echo -e " Network: ${ip} port 8443: ${port_status}" # Incus remote check + auto-register if command -v incus &>/dev/null; then local remote_name="" remote_name=$(incus remote list --format csv 2>/dev/null | \ awk -F',' -v addr="$ip" '$2 ~ addr {print $1; exit}') || true if [[ -n "$remote_name" ]]; then echo -e " Remote: ${GREEN}incus remote '${remote_name}' configured${RESET}" else register_remote "$name" "$ip" # Re-check after registration attempt remote_name=$(incus remote list --format csv 2>/dev/null | \ awk -F',' -v addr="$ip" '$2 ~ addr {print $1; exit}') || true if [[ -n "$remote_name" ]]; then echo -e " Remote: ${GREEN}incus remote '${remote_name}' configured${RESET}" else echo -e " Remote: ${YELLOW}no incus remote for ${ip}${RESET}" fi fi fi else echo -e " Network: ${YELLOW}IP not detected${RESET}" fi fi i=$((i + 1)) echo "" done } # --------------------------------------------------------------------------- # Post-deployment checks # --------------------------------------------------------------------------- phase_post_deploy() { step "Post-Deployment Checks" echo "" local count=${#VM_NAMES[@]} local all_ok=true local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local vmid="${VM_VMIDS[$i]}" local app="${VM_APPS[$i]}" if [[ "$DRY_RUN" == true ]]; then info "[dry-run] Would check '${name}' (VMID ${vmid})" i=$((i + 1)) continue fi # Only check running, installed VMs if ! pve_vm_exists "$vmid"; then i=$((i + 1)) continue fi local status status=$(pve_vm_status "$vmid") || status="unknown" if [[ "$status" != "running" ]]; then i=$((i + 1)) continue fi local has_media=false pve_vm_has_install_media "$vmid" && has_media=true if [[ "$has_media" == true ]]; then i=$((i + 1)) continue fi local ip ip=$(pve_vm_ip "$vmid" "${VM_IPS[$i]:-}") || ip="" if [[ -z "$ip" ]]; then warn "VM '${name}': could not detect IP address" all_ok=false i=$((i + 1)) continue fi # Port 8443 check if curl -sk --connect-timeout 5 "https://${ip}:8443" &>/dev/null; then success "VM '${name}' (${ip}): port 8443 reachable" else warn "VM '${name}' (${ip}): port 8443 not reachable (may still be starting)" all_ok=false i=$((i + 1)) continue fi # App-specific checks if [[ "$app" == "incus" ]] && command -v incus &>/dev/null; then local remote_name="" remote_name=$(incus remote list --format csv 2>/dev/null | \ awk -F',' -v addr="$ip" '$2 ~ addr {print $1; exit}') || true if [[ -n "$remote_name" ]]; then if incus info "${remote_name}:" &>/dev/null; then success "VM '${name}': incus remote '${remote_name}' connected" else warn "VM '${name}': incus remote '${remote_name}' exists but connection failed" all_ok=false fi else # Auto-register the remote register_remote "$name" "$ip" # Verify remote_name=$(incus remote list --format csv 2>/dev/null | \ awk -F',' -v addr="$ip" '$2 ~ addr {print $1; exit}') || true if [[ -n "$remote_name" ]]; then success "VM '${name}': incus remote '${remote_name}' connected" else info "VM '${name}': no incus remote configured" info " Add with: incus remote add ${name} ${ip} --accept-certificate" fi fi elif [[ "$app" == "operations-center" ]]; then success "VM '${name}': Operations Center at https://${ip}:8443" info " Import PKCS#12 cert in browser for web UI access:" info " openssl pkcs12 -export -inkey ~/.config/incus/client.key -in ~/.config/incus/client.crt -out client.pfx" fi i=$((i + 1)) done if [[ "$all_ok" == true ]] && [[ "$DRY_RUN" != true ]]; then echo "" success "All post-deployment checks passed" fi } # --------------------------------------------------------------------------- # Cleanup mode # --------------------------------------------------------------------------- do_cleanup() { step "Cleanup: destroying VMs defined in config" if [[ "$FORCE_CLEANUP" == true ]]; then warn "Force cleanup enabled -- will destroy VMs even without management marker" fi echo "" local count=${#VM_NAMES[@]} local destroyed=0 local not_found=0 local skipped_unmanaged=0 local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local vmid="${VM_VMIDS[$i]}" # 1. Check if VM exists if ! pve_vm_exists "$vmid"; then detail "VM '${name}' (VMID ${vmid}): does not exist" not_found=$((not_found + 1)) i=$((i + 1)) continue fi # 2. Check if VM is managed by us if [[ "$DRY_RUN" != true ]]; then if ! pve_vm_is_managed "$vmid"; then if [[ "$FORCE_CLEANUP" == true ]]; then warn "VM '${name}' (VMID ${vmid}): NOT managed by ${SCRIPT_NAME} -- destroying anyway (--force-cleanup)" else warn "VM '${name}' (VMID ${vmid}): NOT managed by ${SCRIPT_NAME} -- refusing to destroy" warn " This VM was not created by this tool. To destroy it manually:" warn " ssh ${PVE_SSH_USER}@${PVE_HOST} qm stop ${vmid} && ssh ${PVE_SSH_USER}@${PVE_HOST} qm destroy ${vmid} --purge" warn " Or use --force-cleanup to override this safety check" skipped_unmanaged=$((skipped_unmanaged + 1)) i=$((i + 1)) continue fi fi fi # 3. Destroy the VM step "Destroying VM '${name}' (VMID ${vmid})" if [[ "$DRY_RUN" == true ]]; then info "[dry-run] Would stop and destroy VM ${vmid}" destroyed=$((destroyed + 1)) i=$((i + 1)) continue fi # Stop if running local status status=$(pve_vm_status "$vmid") || status="unknown" if [[ "$status" == "running" ]]; then info "Stopping VM..." pve_stop_vm "$vmid" || true sleep 5 fi # Destroy if pve_destroy_vm "$vmid"; then success "VM '${name}' (VMID ${vmid}) destroyed" destroyed=$((destroyed + 1)) else error "Failed to destroy VM '${name}' (VMID ${vmid})" fi i=$((i + 1)) done # Summary echo "" local summary="Cleanup complete: ${destroyed} destroyed" if [[ $not_found -gt 0 ]]; then summary+=", ${not_found} not found" fi if [[ $skipped_unmanaged -gt 0 ]]; then summary+=", ${skipped_unmanaged} skipped (unmanaged)" fi success "$summary" } # Detect which IncusOS ISO was used by checking ide2 config on existing VMs. # Sets DEPLOYMENT_ISO to the filename (e.g. "IncusOS_25.03.iso") or "" if unknown. DEPLOYMENT_ISO="" detect_deployment_iso() { DEPLOYMENT_ISO="" if [[ "$DRY_RUN" == true ]]; then return fi local count=${#VM_NAMES[@]} local i=0 while [[ $i -lt $count ]]; do local vmid="${VM_VMIDS[$i]}" if pve_vm_exists "$vmid"; then local config="" case "$PVE_METHOD" in ssh) config=$(pve_run "qm config ${vmid} 2>/dev/null" 2>/dev/null) || config="" ;; api) config=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null) || config="" ;; esac if [[ -n "$config" ]]; then local iso_name # Extract IncusOS ISO name from ide2 line or JSON iso_name=$(echo "$config" | python3 -c " import sys, json, re text = sys.stdin.read() # Try JSON (API response) try: data = json.loads(text) ide2 = data.get('data', data).get('ide2', '') except (json.JSONDecodeError, AttributeError): ide2 = '' for line in text.splitlines(): if line.startswith('ide2:'): ide2 = line.split(':', 1)[1].strip() break m = re.search(r'(IncusOS_[^,]+\.iso)', ide2) if m: print(m.group(1)) " 2>/dev/null) || iso_name="" if [[ -n "$iso_name" ]]; then DEPLOYMENT_ISO="$iso_name" detail "Detected deployment ISO: ${DEPLOYMENT_ISO} (from VM ${vmid})" return fi fi fi i=$((i + 1)) done } # Deep cleanup: remove seeds and the specific IncusOS ISO used by this deployment cleanup_storage() { step "Deep cleanup: removing seeds and deployment ISO from Proxmox storage" local count=${#VM_NAMES[@]} local i=0 # Remove per-VM seed ISOs while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local seed_file="seed-${name}.iso" if [[ "$DRY_RUN" == true ]]; then info "[dry-run] Would delete ${PVE_ISO_STORAGE}:iso/${seed_file}" else case "$PVE_METHOD" in ssh) pve_run "pvesm free ${PVE_ISO_STORAGE}:iso/${seed_file} 2>/dev/null" &>/dev/null || true ;; api) pve_api DELETE "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content/${PVE_ISO_STORAGE}:iso/${seed_file}" &>/dev/null || true ;; esac info "Deleted seed: ${seed_file}" fi i=$((i + 1)) done # Remove only the specific IncusOS ISO used by this deployment if [[ -n "$DEPLOYMENT_ISO" ]]; then local volid="${PVE_ISO_STORAGE}:iso/${DEPLOYMENT_ISO}" if [[ "$DRY_RUN" == true ]]; then info "[dry-run] Would delete ${volid}" else case "$PVE_METHOD" in ssh) pve_run "pvesm free ${volid} 2>/dev/null" &>/dev/null || true ;; api) local encoded_volid encoded_volid=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${volid}', safe=''))" 2>/dev/null) pve_api DELETE "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content/${encoded_volid}" &>/dev/null || true ;; esac info "Deleted ISO: ${DEPLOYMENT_ISO}" fi else detail "No deployment ISO detected (VMs may already be destroyed)" fi success "Storage cleanup complete" } # Aggressive cleanup: remove ALL IncusOS ISOs and seed ISOs from storage cleanup_storage_all() { step "Deep cleanup: removing ALL IncusOS ISOs and seeds from Proxmox storage" local iso_list="" if [[ "$DRY_RUN" != true ]]; then case "$PVE_METHOD" in ssh) iso_list=$(pve_run "pvesm list ${PVE_ISO_STORAGE} --content iso 2>/dev/null" 2>/dev/null | grep -E "(IncusOS_|seed-)" | awk '{print $1}') || iso_list="" ;; api) iso_list=$(pve_api GET "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content?content=iso" 2>/dev/null | \ python3 -c " import json, sys data = json.load(sys.stdin) for item in data.get('data', []): volid = item.get('volid', '') if 'IncusOS_' in volid or 'seed-' in volid: print(volid) " 2>/dev/null) || iso_list="" ;; esac fi if [[ -n "$iso_list" ]]; then while IFS= read -r volid; do [[ -z "$volid" ]] && continue if [[ "$DRY_RUN" == true ]]; then info "[dry-run] Would delete ${volid}" else case "$PVE_METHOD" in ssh) pve_run "pvesm free ${volid} 2>/dev/null" &>/dev/null || true ;; api) local encoded_volid encoded_volid=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${volid}', safe=''))" 2>/dev/null) pve_api DELETE "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content/${encoded_volid}" &>/dev/null || true ;; esac info "Deleted: ${volid}" fi done <<< "$iso_list" else info "No IncusOS ISOs or seeds found in storage" fi success "Storage cleanup complete" } # Remove incus remotes. Uses VM_NAMES (config-based cleanup) or # DESTROYED_VM_NAMES (pool-wide cleanup) depending on what's populated. cleanup_remotes() { step "Deep cleanup: removing incus remotes" if ! command -v incus &>/dev/null; then detail "incus client not installed, skipping remote cleanup" return fi # Determine which name list to use local names_to_check=() if [[ ${#DESTROYED_VM_NAMES[@]} -gt 0 ]]; then # Pool-wide cleanup: use names of VMs actually destroyed names_to_check=("${DESTROYED_VM_NAMES[@]}") elif [[ ${#VM_NAMES[@]} -gt 0 ]]; then # Config-based cleanup local i=0 while [[ $i -lt ${#VM_NAMES[@]} ]]; do if [[ -n "$VM_FILTER" ]] && [[ "${VM_NAMES[$i]}" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi names_to_check+=("${VM_NAMES[$i]}") i=$((i + 1)) done fi if [[ ${#names_to_check[@]} -eq 0 ]]; then detail "No VM names to check for remote cleanup" success "Remote cleanup complete" return fi for name in "${names_to_check[@]}"; do # Check if a remote with this name exists if incus remote list --format csv 2>/dev/null | grep -q "^${name},"; then if [[ "$DRY_RUN" == true ]]; then info "[dry-run] Would remove incus remote '${name}'" else incus remote remove "$name" 2>/dev/null || true info "Removed incus remote: ${name}" fi fi done success "Remote cleanup complete" } cleanup_local_cache() { step "Deep cleanup: clearing local ISO cache" local cache_base="${XDG_CACHE_HOME:-${HOME}/.cache}/${SCRIPT_NAME}/iso-cache" if [[ -d "$cache_base" ]]; then if [[ "$DRY_RUN" == true ]]; then info "[dry-run] Would clear ${cache_base}" else rm -rf "$cache_base" info "Cleared cache: ${cache_base}" fi else detail "No local cache to clear" fi success "Local cache cleanup complete" } # Names of VMs destroyed during cleanup-all (used by cleanup_remotes) declare -a DESTROYED_VM_NAMES=() # Pool-wide cleanup: destroy ALL managed VMs in the resource pool do_cleanup_all() { step "Pool-wide cleanup: destroying all managed VMs" echo "" if [[ -z "$PVE_POOL" ]]; then error "Pool-wide cleanup requires a resource pool (proxmox.pool)" error "Set pool in proxmox.yaml or pass a config with proxmox.pool" error "Without a pool, there is no safe way to scope which VMs to destroy" exit 1 fi # Get all VMs in the pool local vm_list if ! vm_list=$(pve_list_vms); then error "Failed to list VMs in pool '${PVE_POOL}'" error "Check Proxmox connectivity and API credentials" exit 1 fi if [[ -z "$vm_list" ]]; then info "No VMs found in pool '${PVE_POOL}'" return fi # Filter for managed VMs local managed_vms=() local managed_names=() while IFS=' ' read -r eid ename; do [[ -z "$eid" ]] && continue if pve_vm_is_managed "$eid"; then managed_vms+=("$eid") managed_names+=("${ename:-unknown}") fi done <<< "$vm_list" if [[ ${#managed_vms[@]} -eq 0 ]]; then info "No managed VMs found in pool '${PVE_POOL}'" return fi # Show what will be destroyed echo -e " ${BOLD}Managed VMs in pool '${PVE_POOL}':${RESET}" local idx=0 while [[ $idx -lt ${#managed_vms[@]} ]]; do echo -e " VMID ${managed_vms[$idx]}: ${managed_names[$idx]}" idx=$((idx + 1)) done echo "" # Confirm confirm "Destroy all ${#managed_vms[@]} managed VM(s) in pool '${PVE_POOL}'?" # Destroy each local destroyed=0 idx=0 while [[ $idx -lt ${#managed_vms[@]} ]]; do local vmid="${managed_vms[$idx]}" local name="${managed_names[$idx]}" step "Destroying VM '${name}' (VMID ${vmid})" if [[ "$DRY_RUN" == true ]]; then info "[dry-run] Would stop and destroy VM ${vmid}" destroyed=$((destroyed + 1)) DESTROYED_VM_NAMES+=("$name") idx=$((idx + 1)) continue fi local status status=$(pve_vm_status "$vmid") || status="unknown" if [[ "$status" == "running" ]]; then info "Stopping VM..." pve_stop_vm "$vmid" || true sleep 5 fi if pve_destroy_vm "$vmid"; then success "VM '${name}' (VMID ${vmid}) destroyed" destroyed=$((destroyed + 1)) DESTROYED_VM_NAMES+=("$name") else error "Failed to destroy VM '${name}' (VMID ${vmid})" fi idx=$((idx + 1)) done echo "" success "Pool cleanup complete: ${destroyed} VM(s) destroyed" } # --------------------------------------------------------------------------- # Reconcile existing deployments # --------------------------------------------------------------------------- # Check if VMs from config already exist and offer interactive options. # Only called for PHASE=all, non-cleanup, non-dry-run runs. # Returns 0 to continue with fresh deploy, or exits. reconcile_existing() { local count=${#VM_NAMES[@]} local existing=0 local running=0 local installed=0 local i=0 while [[ $i -lt $count ]]; do local name="${VM_NAMES[$i]}" if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then i=$((i + 1)) continue fi local vmid="${VM_VMIDS[$i]}" if pve_vm_exists "$vmid"; then existing=$((existing + 1)) local status status=$(pve_vm_status "$vmid") || status="unknown" if [[ "$status" == "running" ]]; then running=$((running + 1)) fi local has_media=false pve_vm_has_install_media "$vmid" && has_media=true if [[ "$has_media" == false ]]; then installed=$((installed + 1)) fi fi i=$((i + 1)) done # No existing VMs — fresh deploy, proceed normally if [[ $existing -eq 0 ]]; then return 0 fi # Count of VMs we're deploying (respecting filter) local target=0 for name in "${VM_NAMES[@]}"; do if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then continue fi target=$((target + 1)) done echo "" echo -e "${BOLD}Existing deployment detected${RESET}" echo -e " ${existing}/${target} VMs exist (${running} running, ${installed} installed)" echo "" if [[ "$existing" -lt "$target" ]]; then local incomplete=$((target - existing)) echo -e " ${YELLOW}Partial deployment:${RESET} ${incomplete} VM(s) still need to be created/installed" fi # With --yes, default to safe option (post-deploy checks) if [[ "$YES" == true ]]; then info "Existing VMs detected with --yes; running post-deployment checks" echo "" phase_status phase_post_deploy exit 0 fi echo -e " Options:" echo -e " ${BOLD}1)${RESET} Run status checks and post-deployment verification" echo -e " ${BOLD}2)${RESET} Continue deployment (skip existing, install incomplete VMs)" echo -e " ${BOLD}3)${RESET} Destroy and redeploy cleanly" echo -e " ${BOLD}4)${RESET} Abort" echo "" echo -n -e " ${BOLD}Choose [1/2/3/4]: ${RESET}" local choice read -r choice case "$choice" in 1) phase_status phase_post_deploy exit 0 ;; 2) info "Continuing deployment (existing VMs will be skipped)" return 0 ;; 3) warn "Destroying existing VMs before redeployment" do_cleanup echo "" return 0 ;; 4|*) info "Aborted" exit 0 ;; esac } # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- main() { setup_colors parse_args "$@" validate_args # Handle doctor mode early (before config parsing) if [[ "$DOCTOR" == true ]]; then run_doctor return fi if [[ "$QUIET" != true ]]; then echo -e "${BOLD}${SCRIPT_NAME}${RESET} v${VERSION}" echo "" fi setup_workdir # Handle --labs and --resources without a config file (only needs proxmox.yaml) if { [[ "$LIST_LABS" == true ]] || [[ "$RESOURCES" == true ]]; } && [[ -z "$CONFIG_FILE" ]]; then if ! find_proxmox_config; then error "No proxmox.yaml found and no config file specified" error "Create proxmox.yaml or use --proxmox FILE" exit 1 fi load_proxmox_config "$PROXMOX_CONFIG_RESOLVED" [[ -z "$PVE_NODE" ]] && PVE_NODE="pve" [[ -z "$PVE_STORAGE" ]] && PVE_STORAGE="local-lvm" [[ -z "$PVE_ISO_STORAGE" ]] && PVE_ISO_STORAGE="local" [[ -z "$PVE_BRIDGE" ]] && PVE_BRIDGE="vmbr0" [[ -z "$PVE_SSH_USER" ]] && PVE_SSH_USER="root" [[ -z "$PVE_METHOD" ]] && PVE_METHOD="ssh" [[ -n "$METHOD" ]] && PVE_METHOD="$METHOD" [[ -n "$HOST_OVERRIDE" ]] && PVE_HOST="$HOST_OVERRIDE" if [[ -z "$PVE_HOST" ]]; then error "proxmox.yaml must contain 'host:'" exit 1 fi require_api_auth [[ "$RESOURCES" == true ]] && do_resources [[ "$LIST_LABS" == true ]] && do_list_labs return 0 fi # Handle cleanup-all without a config file (only needs proxmox.yaml) if [[ "$CLEANUP_ALL" == true ]] && [[ -z "$CONFIG_FILE" ]]; then if ! find_proxmox_config; then error "No proxmox.yaml found and no config file specified" error "Create proxmox.yaml or use --proxmox FILE" exit 1 fi load_proxmox_config "$PROXMOX_CONFIG_RESOLVED" # Apply defaults for anything still unset [[ -z "$PVE_NODE" ]] && PVE_NODE="pve" [[ -z "$PVE_STORAGE" ]] && PVE_STORAGE="local-lvm" [[ -z "$PVE_ISO_STORAGE" ]] && PVE_ISO_STORAGE="local" [[ -z "$PVE_BRIDGE" ]] && PVE_BRIDGE="vmbr0" [[ -z "$PVE_SSH_USER" ]] && PVE_SSH_USER="root" [[ -z "$PVE_METHOD" ]] && PVE_METHOD="ssh" [[ -n "$METHOD" ]] && PVE_METHOD="$METHOD" [[ -n "$HOST_OVERRIDE" ]] && PVE_HOST="$HOST_OVERRIDE" if [[ -z "$PVE_HOST" ]]; then error "proxmox.yaml must contain 'host:'" exit 1 fi require_api_auth do_cleanup_all if [[ "$DEEP_CLEANUP" == true ]]; then echo "" cleanup_storage_all cleanup_remotes cleanup_local_cache fi return fi if [[ "$DRY_RUN" != true ]]; then check_dependencies fi echo "" # Always run validate first phase_validate # Handle cleanup-all mode (with config file for additional context) if [[ "$CLEANUP_ALL" == true ]]; then do_cleanup_all if [[ "$DEEP_CLEANUP" == true ]]; then echo "" cleanup_storage_all cleanup_remotes cleanup_local_cache fi return fi # Handle cleanup mode if [[ "$CLEANUP" == true ]]; then if [[ "$DEEP_CLEANUP" == true ]]; then # Detect which ISO was used before destroying VMs detect_deployment_iso fi confirm "Destroy VMs listed above?" do_cleanup if [[ "$DEEP_CLEANUP" == true ]]; then echo "" cleanup_storage cleanup_remotes cleanup_local_cache fi return fi # Handle lab lifecycle commands if [[ "$LAB_DOWN" == true ]]; then phase_lab_down return fi if [[ "$LAB_UP" == true ]]; then phase_lab_up return fi # Handle --resources and --labs with config file if [[ "$RESOURCES" == true ]] || [[ "$LIST_LABS" == true ]]; then [[ "$RESOURCES" == true ]] && do_resources [[ "$LIST_LABS" == true ]] && do_list_labs return 0 fi # Reconcile existing deployments (interactive menu on re-runs) if [[ "$PHASE" == "all" ]] && [[ "$DRY_RUN" != true ]]; then reconcile_existing fi # Confirm before proceeding if [[ "$PHASE" == "all" ]]; then confirm "Proceed with deployment?" fi # Run requested phases case "$PHASE" in validate) # Already done above ;; prepare) phase_prepare ;; upload) phase_prepare phase_upload ;; create) phase_prepare phase_upload phase_create ;; install) phase_prepare phase_upload phase_create phase_install ;; status) phase_status phase_post_deploy ;; all) phase_prepare echo "" phase_upload echo "" phase_create echo "" phase_install echo "" phase_post_deploy ;; esac echo "" if [[ "$DRY_RUN" == true ]]; then info "Dry run complete -- no changes were made" else success "Done" fi } main "$@"