4084 lines
136 KiB
Bash
Executable File
4084 lines
136 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# incusos-proxmox - Declarative IncusOS deployment on Proxmox VE
|
|
#
|
|
# Reads a YAML configuration file defining VMs and deploys them on a Proxmox
|
|
# host with IncusOS-correct settings (UEFI, TPM 2.0, VirtIO-SCSI, etc.).
|
|
# Generates per-VM seed images, uploads ISOs, creates VMs, and boots them
|
|
# through automated installation.
|
|
#
|
|
# Usage: incusos-proxmox [OPTIONS] CONFIG_FILE
|
|
# Run 'incusos-proxmox --help' for full usage information.
|
|
|
|
set -euo pipefail
|
|
|
|
readonly VERSION="0.1.0"
|
|
readonly SCRIPT_NAME="incusos-proxmox"
|
|
readonly CDN_INDEX="https://images.linuxcontainers.org/os/index.json"
|
|
readonly CDN_BASE="https://images.linuxcontainers.org/os"
|
|
readonly MIN_DISK_GIB=50
|
|
readonly MIN_MEMORY_MIB=4096
|
|
readonly MANAGED_MARKER="[incusos-lab:managed]"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Color and formatting
|
|
# ---------------------------------------------------------------------------
|
|
|
|
setup_colors() {
|
|
if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[0;33m'
|
|
BLUE='\033[0;34m'
|
|
CYAN='\033[0;36m'
|
|
BOLD='\033[1m'
|
|
DIM='\033[2m'
|
|
RESET='\033[0m'
|
|
else
|
|
RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' RESET=''
|
|
fi
|
|
}
|
|
|
|
info() { [[ "$QUIET" == true ]] && return; echo -e "${BLUE}[info]${RESET} $*"; }
|
|
success() { [[ "$QUIET" == true ]] && return; echo -e "${GREEN}[ok]${RESET} $*"; }
|
|
warn() { echo -e "${YELLOW}[warn]${RESET} $*" >&2; }
|
|
error() { echo -e "${RED}[error]${RESET} $*" >&2; }
|
|
step() { [[ "$QUIET" == true ]] && return; echo -e "${CYAN}[step]${RESET} ${BOLD}$*${RESET}"; }
|
|
detail() { [[ "$VERBOSE" != true ]] && return; echo -e "${DIM} $*${RESET}"; }
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Help
|
|
# ---------------------------------------------------------------------------
|
|
|
|
usage() {
|
|
cat <<EOF
|
|
${BOLD}${SCRIPT_NAME}${RESET} v${VERSION} - Declarative IncusOS deployment on Proxmox VE
|
|
|
|
${BOLD}USAGE${RESET}
|
|
${SCRIPT_NAME} [OPTIONS] CONFIG_FILE
|
|
|
|
${BOLD}DESCRIPTION${RESET}
|
|
Reads a YAML configuration file defining VMs and deploys them on a Proxmox
|
|
host with IncusOS-correct settings. Handles seed generation, ISO upload, VM
|
|
creation, and automated installation.
|
|
|
|
IncusOS requires specific Proxmox VM settings:
|
|
- UEFI firmware without pre-enrolled Secure Boot keys
|
|
- TPM 2.0 virtual device
|
|
- VirtIO-SCSI controller (not VirtIO-blk)
|
|
- CPU type 'host' (x86_64_v3 instruction set requirement)
|
|
- Minimum 50 GiB disk, minimum 4 GiB RAM
|
|
|
|
This script ensures all settings are correct, preventing install failures.
|
|
|
|
${BOLD}OPTIONS${RESET}
|
|
${BOLD}-p, --phase${RESET} PHASE Run a specific phase:
|
|
${CYAN}validate${RESET} -- parse config, check connectivity
|
|
${CYAN}prepare${RESET} -- download ISO, generate seeds
|
|
${CYAN}upload${RESET} -- upload ISO and seeds to Proxmox
|
|
${CYAN}create${RESET} -- create VMs with correct settings
|
|
${CYAN}install${RESET} -- start VMs and monitor installation
|
|
${CYAN}status${RESET} -- check deployment status
|
|
${CYAN}all${RESET} -- run all phases (default)
|
|
${BOLD}--status${RESET} Shortcut for --phase status
|
|
${BOLD}-m, --method${RESET} METHOD Connection method: ${CYAN}ssh${RESET} (default), ${CYAN}api${RESET}
|
|
${BOLD}-H, --host${RESET} HOST Override Proxmox host from config
|
|
${BOLD}-i, --iso${RESET} FILE Use local ISO file (skip CDN download)
|
|
${BOLD}-c, --cert${RESET} FILE Client certificate PEM file
|
|
${BOLD}--no-cert${RESET} Skip certificate injection
|
|
${BOLD}--vm${RESET} NAME Operate on a single VM only
|
|
${BOLD}-y, --yes${RESET} Skip confirmation prompts
|
|
${BOLD}--doctor${RESET} Check tool versions and environment health
|
|
${BOLD}--cleanup${RESET} Destroy VMs defined in config (managed only)
|
|
${BOLD}--cleanup --deep${RESET} Also remove ISOs, seeds, remotes, and local cache
|
|
${BOLD}--cleanup-all${RESET} Destroy ALL managed VMs in the pool
|
|
${BOLD}--proxmox${RESET} FILE Proxmox connection config (default: auto-discover)
|
|
${BOLD}--force-cleanup${RESET} Destroy VMs even without management marker
|
|
${BOLD}--dry-run${RESET} Preview actions without executing
|
|
${BOLD}-v, --verbose${RESET} Show detailed output (tool paths, API calls)
|
|
${BOLD}-q, --quiet${RESET} Suppress informational output
|
|
${BOLD}-h, --help${RESET} Show this help message
|
|
${BOLD}-V, --version${RESET} Show version
|
|
|
|
${BOLD}PROXMOX CONNECTION CONFIG${RESET}
|
|
Proxmox connection settings can be stored in a separate proxmox.yaml file
|
|
to avoid duplicating credentials across lab configs. Auto-discovered in:
|
|
script directory, then current directory. Override with --proxmox FILE.
|
|
|
|
${BOLD}proxmox.yaml${RESET} (flat structure, no wrapping key):
|
|
host: 192.168.1.10 # Proxmox host (required)
|
|
method: api # ssh or api (default: ssh)
|
|
api_token_id: user!token # API token (for api method)
|
|
node: pve # Node name (default: pve)
|
|
storage: local-lvm # VM disk storage (default: local-lvm)
|
|
iso_storage: local # ISO storage (default: local)
|
|
bridge: vmbr0 # Network bridge (default: vmbr0)
|
|
pool: IncusLab # Resource pool (optional)
|
|
ssh_user: root # SSH user (default: root)
|
|
|
|
Merge priority: proxmox.yaml → lab config proxmox: section → CLI flags
|
|
|
|
${BOLD}CONFIG FILE${RESET}
|
|
YAML file defining VM specifications (and optional Proxmox overrides).
|
|
See examples/proxmox-lab.yaml for a full example.
|
|
|
|
${BOLD}proxmox:${RESET} # Optional: overrides proxmox.yaml
|
|
host: 192.168.1.10 # Proxmox host
|
|
storage: local-zfs # Override storage for this deployment
|
|
|
|
${BOLD}defaults:${RESET}
|
|
cores: 4 # Default CPU cores
|
|
memory: 8192 # Default RAM in MiB (min: 4096)
|
|
disk: 64 # Default disk in GiB (min: 50)
|
|
channel: stable # Update channel
|
|
start_vmid: 200 # Auto-assign VMIDs from here
|
|
|
|
${BOLD}vms:${RESET}
|
|
- name: my-vm # VM name (required)
|
|
app: incus # incus or operations-center (required)
|
|
apply_defaults: true # Apply IncusOS defaults
|
|
cores: 4 # Override default cores
|
|
memory: 8192 # Override default memory
|
|
disk: 64 # Override default disk
|
|
vmid: 200 # Explicit VMID (optional)
|
|
|
|
${BOLD}EXAMPLES${RESET}
|
|
# Preview a full deployment
|
|
${SCRIPT_NAME} --dry-run examples/proxmox-lab.yaml
|
|
|
|
# Validate config and check Proxmox connectivity
|
|
${SCRIPT_NAME} --phase validate examples/proxmox-lab.yaml
|
|
|
|
# Deploy everything (with confirmation prompts)
|
|
${SCRIPT_NAME} examples/proxmox-lab.yaml
|
|
|
|
# Deploy without prompts
|
|
${SCRIPT_NAME} --yes examples/proxmox-lab.yaml
|
|
|
|
# Deploy a single VM from the config
|
|
${SCRIPT_NAME} --vm ops-center examples/proxmox-lab.yaml
|
|
|
|
# Use a local ISO instead of downloading
|
|
${SCRIPT_NAME} --iso IncusOS_20250201.iso examples/proxmox-lab.yaml
|
|
|
|
# Check deployment status
|
|
${SCRIPT_NAME} --status examples/proxmox-lab.yaml
|
|
|
|
# Destroy all VMs defined in config
|
|
${SCRIPT_NAME} --cleanup examples/proxmox-lab.yaml
|
|
|
|
# Deep cleanup: VMs + the IncusOS ISO used + seeds + remotes + cache
|
|
${SCRIPT_NAME} --cleanup --deep examples/proxmox-lab.yaml
|
|
|
|
# Destroy ALL managed VMs in the pool (uses proxmox.yaml, no config needed)
|
|
${SCRIPT_NAME} --cleanup-all
|
|
${SCRIPT_NAME} --cleanup-all --deep # also remove all ISOs + seeds
|
|
|
|
# Check tool versions and environment health
|
|
${SCRIPT_NAME} --doctor
|
|
${SCRIPT_NAME} --doctor examples/proxmox-lab.yaml # also check Proxmox
|
|
|
|
# Verbose output (show dependency paths, API calls, per-file actions)
|
|
${SCRIPT_NAME} --doctor -v
|
|
|
|
# Use API method with token
|
|
export PROXMOX_TOKEN_SECRET="xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
|
|
${SCRIPT_NAME} --method api examples/proxmox-lab.yaml
|
|
|
|
${BOLD}PROXMOX SETUP${RESET}
|
|
${BOLD}SSH method (default):${RESET}
|
|
Requires key-based SSH authentication to the Proxmox host.
|
|
Test with: ssh root@<host> pvesh get /version
|
|
|
|
${BOLD}API method:${RESET}
|
|
Requires an API token. Set up minimal privileges:
|
|
|
|
pveum role add IncusOSDeploy --privs "VM.Allocate VM.Config.Disk \\
|
|
VM.Config.CPU VM.Config.Memory VM.Config.Network VM.Config.CDROM \\
|
|
VM.Config.Options VM.Config.HWType VM.PowerMgmt VM.Audit \\
|
|
Datastore.AllocateSpace Datastore.AllocateTemplate \\
|
|
Datastore.Audit SDN.Use"
|
|
pveum user add automation@pve
|
|
pveum aclmod / -user automation@pve -role IncusOSDeploy
|
|
pveum user token add automation@pve deploy --privsep 0
|
|
|
|
For resource pool isolation (recommended):
|
|
pveum pool add IncusLab --comment "IncusOS Lab VMs"
|
|
pveum pool modify IncusLab --storage local-lvm,local
|
|
pveum aclmod /pool/IncusLab -user automation@pve -role IncusOSDeploy
|
|
|
|
Set the token secret:
|
|
export PROXMOX_TOKEN_SECRET="<secret-from-above>"
|
|
|
|
And in your config file:
|
|
proxmox:
|
|
method: api
|
|
api_token_id: automation@pve!deploy
|
|
|
|
${BOLD}PREREQUISITES${RESET}
|
|
- python3 with PyYAML (python3-yaml / py3-yaml)
|
|
- jq (for JSON field access; falls back to python3)
|
|
- ssh + scp (for SSH method)
|
|
- curl (for API method and CDN downloads)
|
|
- incusos-seed sibling script (auto-detected)
|
|
- Bash 4+ (associative arrays)
|
|
|
|
EOF
|
|
exit 0
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Argument parsing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Defaults
|
|
PHASE="all"
|
|
METHOD=""
|
|
HOST_OVERRIDE=""
|
|
LOCAL_ISO=""
|
|
CERT_FILE=""
|
|
NO_CERT=false
|
|
VM_FILTER=""
|
|
YES=false
|
|
DOCTOR=false
|
|
CLEANUP=false
|
|
CLEANUP_ALL=false
|
|
DEEP_CLEANUP=false
|
|
FORCE_CLEANUP=false
|
|
DRY_RUN=false
|
|
QUIET=false
|
|
VERBOSE=false
|
|
PROXMOX_CONFIG=""
|
|
CONFIG_FILE=""
|
|
|
|
parse_args() {
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
-p|--phase)
|
|
PHASE="${2:?'--phase requires a value'}"
|
|
shift 2
|
|
;;
|
|
-m|--method)
|
|
METHOD="${2:?'--method requires a value'}"
|
|
shift 2
|
|
;;
|
|
-H|--host)
|
|
HOST_OVERRIDE="${2:?'--host requires a value'}"
|
|
shift 2
|
|
;;
|
|
-i|--iso)
|
|
LOCAL_ISO="${2:?'--iso requires a file path'}"
|
|
shift 2
|
|
;;
|
|
-c|--cert)
|
|
CERT_FILE="${2:?'--cert requires a file path'}"
|
|
shift 2
|
|
;;
|
|
--no-cert)
|
|
NO_CERT=true
|
|
shift
|
|
;;
|
|
--vm)
|
|
VM_FILTER="${2:?'--vm requires a name'}"
|
|
shift 2
|
|
;;
|
|
-y|--yes)
|
|
YES=true
|
|
shift
|
|
;;
|
|
--status)
|
|
PHASE="status"
|
|
shift
|
|
;;
|
|
--doctor)
|
|
DOCTOR=true
|
|
shift
|
|
;;
|
|
--cleanup)
|
|
CLEANUP=true
|
|
shift
|
|
;;
|
|
--deep)
|
|
DEEP_CLEANUP=true
|
|
shift
|
|
;;
|
|
--cleanup-all)
|
|
CLEANUP_ALL=true
|
|
shift
|
|
;;
|
|
--force-cleanup)
|
|
CLEANUP=true
|
|
FORCE_CLEANUP=true
|
|
shift
|
|
;;
|
|
--proxmox)
|
|
PROXMOX_CONFIG="${2:?'--proxmox requires a file path'}"
|
|
shift 2
|
|
;;
|
|
--dry-run)
|
|
DRY_RUN=true
|
|
shift
|
|
;;
|
|
-v|--verbose)
|
|
VERBOSE=true
|
|
shift
|
|
;;
|
|
-q|--quiet)
|
|
QUIET=true
|
|
shift
|
|
;;
|
|
-h|--help)
|
|
usage
|
|
;;
|
|
-V|--version)
|
|
echo "${SCRIPT_NAME} v${VERSION}"
|
|
exit 0
|
|
;;
|
|
-*)
|
|
error "Unknown option: $1"
|
|
echo "Run '${SCRIPT_NAME} --help' for usage information." >&2
|
|
exit 1
|
|
;;
|
|
*)
|
|
if [[ -z "$CONFIG_FILE" ]]; then
|
|
CONFIG_FILE="$1"
|
|
else
|
|
error "Unexpected argument: $1 (config file already set to ${CONFIG_FILE})"
|
|
echo "Run '${SCRIPT_NAME} --help' for usage information." >&2
|
|
exit 1
|
|
fi
|
|
shift
|
|
;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Validation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
validate_args() {
|
|
# Universal checks (apply regardless of mode)
|
|
if [[ "$VERBOSE" == true ]] && [[ "$QUIET" == true ]]; then
|
|
error "Cannot use both --verbose and --quiet"
|
|
exit 1
|
|
fi
|
|
|
|
# --doctor can run without a config file
|
|
if [[ "$DOCTOR" == true ]] && [[ -z "$CONFIG_FILE" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
# --cleanup-all can run without a config file (uses proxmox.yaml)
|
|
if [[ "$CLEANUP_ALL" == true ]] && [[ -z "$CONFIG_FILE" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
if [[ -z "$CONFIG_FILE" ]]; then
|
|
error "No configuration file specified"
|
|
echo "Usage: ${SCRIPT_NAME} [OPTIONS] CONFIG_FILE" >&2
|
|
echo "Run '${SCRIPT_NAME} --help' for usage information." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ ! -f "$CONFIG_FILE" ]]; then
|
|
error "Configuration file not found: ${CONFIG_FILE}"
|
|
exit 1
|
|
fi
|
|
|
|
# --deep only makes sense with --cleanup or --cleanup-all
|
|
if [[ "$DEEP_CLEANUP" == true ]] && [[ "$CLEANUP" != true ]] && [[ "$CLEANUP_ALL" != true ]]; then
|
|
error "--deep can only be used with --cleanup or --cleanup-all"
|
|
exit 1
|
|
fi
|
|
|
|
case "$PHASE" in
|
|
validate|prepare|upload|create|install|status|all) ;;
|
|
*)
|
|
error "Invalid phase: ${PHASE}"
|
|
error "Supported: validate, prepare, upload, create, install, status, all"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
if [[ -n "$METHOD" ]]; then
|
|
case "$METHOD" in
|
|
ssh|api) ;;
|
|
*)
|
|
error "Invalid method: ${METHOD}"
|
|
error "Supported: ssh, api"
|
|
exit 1
|
|
;;
|
|
esac
|
|
fi
|
|
|
|
if [[ -n "$CERT_FILE" ]] && [[ "$NO_CERT" == true ]]; then
|
|
error "Cannot use both --cert and --no-cert"
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -n "$CERT_FILE" ]] && [[ ! -f "$CERT_FILE" ]]; then
|
|
error "Certificate file not found: ${CERT_FILE}"
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -n "$LOCAL_ISO" ]] && [[ ! -f "$LOCAL_ISO" ]]; then
|
|
error "ISO file not found: ${LOCAL_ISO}"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Dependency checks
|
|
# ---------------------------------------------------------------------------
|
|
|
|
SEED_TOOL=""
|
|
|
|
check_dependencies() {
|
|
step "Checking dependencies"
|
|
|
|
local missing=false
|
|
|
|
# python3
|
|
if ! command -v python3 &>/dev/null; then
|
|
error "python3 is required but not found"
|
|
error "Install Python 3 for your platform (https://www.python.org/)"
|
|
missing=true
|
|
else
|
|
detail "python3: $(command -v python3)"
|
|
fi
|
|
|
|
# python3-yaml (optional -- basic YAML fallback built-in)
|
|
if command -v python3 &>/dev/null; then
|
|
if python3 -c "import yaml" 2>/dev/null; then
|
|
detail "PyYAML: available"
|
|
else
|
|
detail "PyYAML: not found (using built-in basic parser)"
|
|
fi
|
|
fi
|
|
|
|
# jq (optional, python3 fallback)
|
|
if command -v jq &>/dev/null; then
|
|
detail "jq: $(command -v jq)"
|
|
else
|
|
detail "jq: not found (using python3 fallback)"
|
|
fi
|
|
|
|
# curl
|
|
if ! command -v curl &>/dev/null; then
|
|
error "curl is required but not found"
|
|
missing=true
|
|
else
|
|
detail "curl: $(command -v curl)"
|
|
fi
|
|
|
|
# ssh/scp (for SSH method)
|
|
if [[ "${PVE_METHOD:-ssh}" == "ssh" ]]; then
|
|
if ! command -v ssh &>/dev/null; then
|
|
error "ssh is required for SSH method but not found"
|
|
missing=true
|
|
fi
|
|
if ! command -v scp &>/dev/null; then
|
|
error "scp is required for SSH method but not found"
|
|
missing=true
|
|
fi
|
|
fi
|
|
|
|
# incusos-seed sibling script
|
|
local script_dir
|
|
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
SEED_TOOL="${script_dir}/incusos-seed"
|
|
if [[ ! -x "$SEED_TOOL" ]]; then
|
|
error "incusos-seed not found at: ${SEED_TOOL}"
|
|
error "The incusos-seed script must be in the same directory as ${SCRIPT_NAME}"
|
|
missing=true
|
|
else
|
|
detail "incusos-seed: ${SEED_TOOL}"
|
|
fi
|
|
|
|
# bash version
|
|
if [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then
|
|
error "Bash 4+ is required (found ${BASH_VERSION})"
|
|
missing=true
|
|
else
|
|
detail "bash: ${BASH_VERSION}"
|
|
fi
|
|
|
|
if [[ "$missing" == true ]]; then
|
|
error "Missing dependencies -- cannot continue"
|
|
exit 1
|
|
fi
|
|
|
|
success "All dependencies satisfied"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Doctor mode
|
|
# ---------------------------------------------------------------------------
|
|
|
|
run_doctor() {
|
|
echo -e "${BOLD}${SCRIPT_NAME} doctor${RESET}"
|
|
echo ""
|
|
|
|
local script_dir
|
|
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
# -- Tools section --
|
|
echo -e " ${BOLD}Tools:${RESET}"
|
|
|
|
# incus client
|
|
if command -v incus &>/dev/null; then
|
|
local incus_ver
|
|
incus_ver=$(incus version 2>/dev/null | grep -oP 'Client version: \K[0-9.]+' || echo "unknown")
|
|
echo -e " incus client: ${GREEN}${incus_ver} ✓${RESET}"
|
|
else
|
|
echo -e " incus client: ${YELLOW}not installed${RESET}"
|
|
fi
|
|
|
|
# flasher-tool
|
|
if command -v flasher-tool &>/dev/null; then
|
|
local ft_ver
|
|
ft_ver=$(flasher-tool --version 2>/dev/null | head -1 | sed 's/^flasher-tool version //' || echo "installed")
|
|
if [[ -z "$ft_ver" ]] || [[ "$ft_ver" == *"unknown"* ]]; then
|
|
ft_ver="dev (installed)"
|
|
fi
|
|
echo -e " flasher-tool: ${GREEN}${ft_ver} ✓${RESET}"
|
|
else
|
|
echo -e " flasher-tool: ${YELLOW}not installed${RESET}"
|
|
fi
|
|
|
|
# operations-center CLI
|
|
if command -v operations-center &>/dev/null; then
|
|
local oc_ver
|
|
oc_ver=$(operations-center version 2>/dev/null | head -1 || echo "installed")
|
|
echo -e " operations-center CLI: ${GREEN}${oc_ver} ✓${RESET}"
|
|
else
|
|
echo -e " operations-center CLI: ${DIM}not installed (optional)${RESET}"
|
|
fi
|
|
|
|
# python3
|
|
if command -v python3 &>/dev/null; then
|
|
local py_ver
|
|
py_ver=$(python3 --version 2>/dev/null | awk '{print $2}')
|
|
echo -e " python3: ${GREEN}${py_ver} ✓${RESET}"
|
|
else
|
|
echo -e " python3: ${RED}not installed (required)${RESET}"
|
|
fi
|
|
|
|
# curl
|
|
if command -v curl &>/dev/null; then
|
|
local curl_ver
|
|
curl_ver=$(curl --version 2>/dev/null | head -1 | awk '{print $2}')
|
|
echo -e " curl: ${GREEN}${curl_ver} ✓${RESET}"
|
|
else
|
|
echo -e " curl: ${RED}not installed (required)${RESET}"
|
|
fi
|
|
|
|
# genisoimage
|
|
if command -v genisoimage &>/dev/null; then
|
|
echo -e " genisoimage: ${GREEN}installed ✓${RESET}"
|
|
else
|
|
echo -e " genisoimage: ${RED}not installed (required for seed ISOs)${RESET}"
|
|
fi
|
|
|
|
# jq
|
|
if command -v jq &>/dev/null; then
|
|
local jq_ver
|
|
jq_ver=$(jq --version 2>/dev/null | sed 's/^jq-//')
|
|
echo -e " jq: ${GREEN}${jq_ver} ✓${RESET}"
|
|
else
|
|
echo -e " jq: ${YELLOW}not installed (python3 fallback used)${RESET}"
|
|
fi
|
|
|
|
# incusos-seed sibling
|
|
if [[ -x "${script_dir}/incusos-seed" ]]; then
|
|
echo -e " incusos-seed: ${GREEN}found ✓${RESET}"
|
|
else
|
|
echo -e " incusos-seed: ${RED}not found at ${script_dir}/incusos-seed${RESET}"
|
|
fi
|
|
|
|
# -- IncusOS section --
|
|
echo ""
|
|
echo -e " ${BOLD}IncusOS:${RESET}"
|
|
|
|
local cdn_latest=""
|
|
local cdn_index
|
|
cdn_index=$(mktemp)
|
|
if curl -fsSL --connect-timeout 5 "$CDN_INDEX" -o "$cdn_index" 2>/dev/null; then
|
|
if command -v jq &>/dev/null; then
|
|
cdn_latest=$(jq -r '[.updates[] | select(.channels[] == "stable")] | sort_by(.version) | last | .version // empty' "$cdn_index" 2>/dev/null)
|
|
else
|
|
cdn_latest=$(python3 -c "
|
|
import json
|
|
data = json.load(open('${cdn_index}'))
|
|
versions = [u['version'] for u in data.get('updates', []) if 'stable' in u.get('channels', [])]
|
|
if versions:
|
|
print(sorted(versions)[-1])
|
|
" 2>/dev/null)
|
|
fi
|
|
fi
|
|
rm -f "$cdn_index"
|
|
|
|
if [[ -n "$cdn_latest" ]]; then
|
|
echo -e " Latest stable: ${cdn_latest}"
|
|
else
|
|
echo -e " Latest stable: ${YELLOW}could not reach CDN${RESET}"
|
|
fi
|
|
|
|
# Check local cache
|
|
local cache_base="${XDG_CACHE_HOME:-${HOME}/.cache}/${SCRIPT_NAME}/iso-cache"
|
|
if [[ -d "$cache_base" ]]; then
|
|
local cached_iso
|
|
cached_iso=$(ls -1 "${cache_base}"/IncusOS_*.iso 2>/dev/null | sort | tail -1)
|
|
if [[ -n "$cached_iso" ]]; then
|
|
local cached_name
|
|
cached_name=$(basename "$cached_iso" .iso | sed 's/^IncusOS_//')
|
|
if [[ -n "$cdn_latest" ]] && [[ "$cached_name" == "$cdn_latest" ]]; then
|
|
echo -e " Cached ISO: ${GREEN}${cached_name} (up to date)${RESET}"
|
|
else
|
|
echo -e " Cached ISO: ${YELLOW}${cached_name}${RESET}"
|
|
fi
|
|
else
|
|
echo -e " Cached ISO: ${DIM}none${RESET}"
|
|
fi
|
|
else
|
|
echo -e " Cached ISO: ${DIM}none${RESET}"
|
|
fi
|
|
|
|
# -- Config section --
|
|
echo ""
|
|
echo -e " ${BOLD}Config:${RESET}"
|
|
if find_proxmox_config; then
|
|
echo -e " proxmox.yaml: ${GREEN}${PROXMOX_CONFIG_RESOLVED} ✓${RESET}"
|
|
else
|
|
echo -e " proxmox.yaml: ${DIM}not found (optional)${RESET}"
|
|
fi
|
|
if [[ -n "$CONFIG_FILE" ]] && [[ -f "$CONFIG_FILE" ]]; then
|
|
echo -e " Lab config: ${CONFIG_FILE}"
|
|
fi
|
|
|
|
# -- Proxmox section (if config file or proxmox.yaml provides connection info) --
|
|
echo ""
|
|
if [[ -n "$CONFIG_FILE" ]] && [[ -f "$CONFIG_FILE" ]]; then
|
|
echo -e " ${BOLD}Proxmox:${RESET}"
|
|
|
|
# Minimal config parsing for doctor mode
|
|
local doctor_workdir
|
|
doctor_workdir=$(mktemp -d "${TMPDIR:-/tmp}/${SCRIPT_NAME}-doctor-XXXXXX")
|
|
WORKDIR="$doctor_workdir"
|
|
CONFIG_JSON="${doctor_workdir}/config.json"
|
|
|
|
if python3 -c "
|
|
import json, sys
|
|
try:
|
|
import yaml
|
|
with open(sys.argv[1]) as f:
|
|
data = yaml.safe_load(f)
|
|
except ImportError:
|
|
# Minimal inline parse -- just need proxmox section
|
|
data = {}
|
|
with open(sys.argv[1]) as f:
|
|
import re
|
|
lines = f.readlines()
|
|
processed = []
|
|
for raw in lines:
|
|
s = raw.rstrip()
|
|
if not s or s.lstrip().startswith('#'):
|
|
continue
|
|
indent = len(s) - len(s.lstrip())
|
|
content = s.lstrip()
|
|
processed.append((indent, content))
|
|
result = {}
|
|
section = None
|
|
for indent, content in processed:
|
|
m = re.match(r'^([^:]+?):\s*(.*?)$', content)
|
|
if m:
|
|
key = m.group(1).strip()
|
|
val = m.group(2).strip()
|
|
if not val:
|
|
section = key
|
|
result[section] = {}
|
|
elif section:
|
|
if val.startswith('\"') or val.startswith(\"'\"):
|
|
val = val[1:-1]
|
|
result[section][key] = val
|
|
data = result
|
|
json.dump(data, sys.stdout, indent=2)
|
|
" "$CONFIG_FILE" > "$CONFIG_JSON" 2>/dev/null; then
|
|
local doc_host doc_method doc_token_id doc_pool
|
|
|
|
doc_host=$(json_get "$CONFIG_JSON" ".proxmox.host" "")
|
|
doc_method=$(json_get "$CONFIG_JSON" ".proxmox.method" "ssh")
|
|
doc_token_id=$(json_get "$CONFIG_JSON" ".proxmox.api_token_id" "")
|
|
doc_pool=$(json_get "$CONFIG_JSON" ".proxmox.pool" "")
|
|
|
|
if [[ -z "$doc_host" ]]; then
|
|
echo -e " Host: ${RED}not set in config${RESET}"
|
|
else
|
|
echo -e " Host: ${doc_host} (${doc_method})"
|
|
|
|
# Connectivity check
|
|
local pve_ok=false
|
|
if [[ "$doc_method" == "api" ]]; then
|
|
if [[ -n "$doc_token_id" ]] && [[ -n "${PROXMOX_TOKEN_SECRET:-}" ]]; then
|
|
local auth="Authorization: PVEAPIToken=${doc_token_id}=${PROXMOX_TOKEN_SECRET}"
|
|
local pve_ver
|
|
pve_ver=$(curl -fsSk --connect-timeout 5 -H "$auth" \
|
|
"https://${doc_host}:8006/api2/json/version" 2>/dev/null | \
|
|
python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('version',''))" 2>/dev/null) || pve_ver=""
|
|
if [[ -n "$pve_ver" ]]; then
|
|
echo -e " Connectivity: ${GREEN}connected (PVE ${pve_ver}) ✓${RESET}"
|
|
pve_ok=true
|
|
else
|
|
echo -e " Connectivity: ${RED}API connection failed${RESET}"
|
|
fi
|
|
else
|
|
echo -e " Connectivity: ${YELLOW}API token not configured or PROXMOX_TOKEN_SECRET not set${RESET}"
|
|
fi
|
|
else
|
|
local ssh_user
|
|
ssh_user=$(json_get "$CONFIG_JSON" ".proxmox.ssh_user" "root")
|
|
if ssh -o BatchMode=yes -o ConnectTimeout=5 "${ssh_user}@${doc_host}" "true" &>/dev/null; then
|
|
echo -e " Connectivity: ${GREEN}SSH connected ✓${RESET}"
|
|
pve_ok=true
|
|
else
|
|
echo -e " Connectivity: ${RED}SSH connection failed${RESET}"
|
|
fi
|
|
fi
|
|
|
|
# Pool check (API only, if connected)
|
|
if [[ -n "$doc_pool" ]]; then
|
|
if [[ "$pve_ok" == true ]] && [[ "$doc_method" == "api" ]]; then
|
|
local auth="Authorization: PVEAPIToken=${doc_token_id}=${PROXMOX_TOKEN_SECRET}"
|
|
if curl -fsSk --connect-timeout 5 -H "$auth" \
|
|
"https://${doc_host}:8006/api2/json/pools/${doc_pool}" &>/dev/null; then
|
|
echo -e " Pool: ${GREEN}${doc_pool} exists ✓${RESET}"
|
|
else
|
|
echo -e " Pool: ${RED}${doc_pool} not found${RESET}"
|
|
fi
|
|
else
|
|
echo -e " Pool: ${doc_pool} (not verified)"
|
|
fi
|
|
fi
|
|
fi
|
|
else
|
|
echo -e " ${RED}Failed to parse config file${RESET}"
|
|
fi
|
|
|
|
rm -rf "$doctor_workdir"
|
|
elif [[ -n "$PROXMOX_CONFIG_RESOLVED" ]]; then
|
|
# Use proxmox.yaml for connectivity check (no lab config provided)
|
|
echo -e " ${BOLD}Proxmox:${RESET} ${DIM}(from proxmox.yaml)${RESET}"
|
|
|
|
local doctor_workdir
|
|
doctor_workdir=$(mktemp -d "${TMPDIR:-/tmp}/${SCRIPT_NAME}-doctor-XXXXXX")
|
|
WORKDIR="$doctor_workdir"
|
|
|
|
load_proxmox_config "$PROXMOX_CONFIG_RESOLVED"
|
|
[[ -z "$PVE_NODE" ]] && PVE_NODE="pve"
|
|
[[ -z "$PVE_METHOD" ]] && PVE_METHOD="ssh"
|
|
[[ -z "$PVE_SSH_USER" ]] && PVE_SSH_USER="root"
|
|
[[ -n "$METHOD" ]] && PVE_METHOD="$METHOD"
|
|
[[ -n "$HOST_OVERRIDE" ]] && PVE_HOST="$HOST_OVERRIDE"
|
|
|
|
if [[ -z "$PVE_HOST" ]]; then
|
|
echo -e " Host: ${RED}not set in proxmox.yaml${RESET}"
|
|
else
|
|
echo -e " Host: ${PVE_HOST} (${PVE_METHOD})"
|
|
local pve_ok=false
|
|
if [[ "$PVE_METHOD" == "api" ]]; then
|
|
if [[ -n "$PVE_API_TOKEN_ID" ]] && [[ -n "${PROXMOX_TOKEN_SECRET:-}" ]]; then
|
|
local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}"
|
|
local pve_ver
|
|
pve_ver=$(curl -fsSk --connect-timeout 5 -H "$auth" \
|
|
"https://${PVE_HOST}:8006/api2/json/version" 2>/dev/null | \
|
|
python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('version',''))" 2>/dev/null) || pve_ver=""
|
|
if [[ -n "$pve_ver" ]]; then
|
|
echo -e " Connectivity: ${GREEN}connected (PVE ${pve_ver}) ✓${RESET}"
|
|
pve_ok=true
|
|
else
|
|
echo -e " Connectivity: ${RED}API connection failed${RESET}"
|
|
fi
|
|
else
|
|
echo -e " Connectivity: ${YELLOW}API token not configured or PROXMOX_TOKEN_SECRET not set${RESET}"
|
|
fi
|
|
else
|
|
if ssh -o BatchMode=yes -o ConnectTimeout=5 "${PVE_SSH_USER}@${PVE_HOST}" "true" &>/dev/null; then
|
|
echo -e " Connectivity: ${GREEN}SSH connected ✓${RESET}"
|
|
pve_ok=true
|
|
else
|
|
echo -e " Connectivity: ${RED}SSH connection failed${RESET}"
|
|
fi
|
|
fi
|
|
if [[ -n "$PVE_POOL" ]]; then
|
|
if [[ "$pve_ok" == true ]] && [[ "$PVE_METHOD" == "api" ]]; then
|
|
local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}"
|
|
if curl -fsSk --connect-timeout 5 -H "$auth" \
|
|
"https://${PVE_HOST}:8006/api2/json/pools/${PVE_POOL}" &>/dev/null; then
|
|
echo -e " Pool: ${GREEN}${PVE_POOL} exists ✓${RESET}"
|
|
else
|
|
echo -e " Pool: ${RED}${PVE_POOL} not found${RESET}"
|
|
fi
|
|
else
|
|
echo -e " Pool: ${PVE_POOL} (not verified)"
|
|
fi
|
|
fi
|
|
fi
|
|
rm -rf "$doctor_workdir"
|
|
else
|
|
echo -e " ${BOLD}Proxmox:${RESET} ${DIM}(no config file or proxmox.yaml found)${RESET}"
|
|
fi
|
|
|
|
echo ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Temporary directory management
|
|
# ---------------------------------------------------------------------------
|
|
|
|
WORKDIR=""
|
|
CACHEDIR=""
|
|
|
|
setup_workdir() {
|
|
# Use persistent storage for the work directory -- the ISO download +
|
|
# decompression can exceed 4 GiB, which overflows tmpfs on most systems.
|
|
# Honour TMPDIR if explicitly set; otherwise use ~/.cache on real disk.
|
|
local base="${TMPDIR:-${XDG_CACHE_HOME:-${HOME}/.cache}/${SCRIPT_NAME}}"
|
|
mkdir -p "$base"
|
|
|
|
# Persistent cache for ISO downloads (survives across runs)
|
|
CACHEDIR="${base}/iso-cache"
|
|
mkdir -p "$CACHEDIR"
|
|
|
|
WORKDIR=$(mktemp -d "${base}/${SCRIPT_NAME}-XXXXXX")
|
|
trap 'rm -rf "$WORKDIR"' EXIT
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# YAML parsing via python3
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Convert YAML config to JSON, stored in a file for repeated access
|
|
CONFIG_JSON=""
|
|
|
|
parse_config() {
|
|
step "Parsing configuration: ${CONFIG_FILE}"
|
|
|
|
CONFIG_JSON="${WORKDIR}/config.json"
|
|
|
|
# Try PyYAML first, fall back to a basic pure-Python parser
|
|
if ! python3 -c "
|
|
import json, sys, re
|
|
|
|
def _parse_value(v):
|
|
v = v.strip()
|
|
# Strip inline comments (but not inside quotes)
|
|
if not v.startswith(('\"', \"'\")):
|
|
m = re.match(r'^(.*?)\s+#.*$', v)
|
|
if m:
|
|
v = m.group(1).rstrip()
|
|
if not v:
|
|
return None
|
|
if (v.startswith('\"') and v.endswith('\"')) or (v.startswith(\"'\") and v.endswith(\"'\")):
|
|
return v[1:-1]
|
|
if v == 'true':
|
|
return True
|
|
if v == 'false':
|
|
return False
|
|
if v == 'null' or v == '~':
|
|
return None
|
|
try:
|
|
return int(v)
|
|
except ValueError:
|
|
pass
|
|
try:
|
|
return float(v)
|
|
except ValueError:
|
|
pass
|
|
return v
|
|
|
|
def parse_yaml_basic(path):
|
|
\"\"\"Minimal YAML parser for simple config files.
|
|
Handles: mappings, lists of mappings, lists of scalars, inline comments.
|
|
Does NOT handle: anchors, multi-line strings, flow style, merge keys.\"\"\"
|
|
with open(path) as f:
|
|
lines = f.readlines()
|
|
|
|
# Pre-process: strip comments and blank lines, track indent + content
|
|
processed = []
|
|
for raw in lines:
|
|
s = raw.rstrip()
|
|
if not s or s.lstrip().startswith('#'):
|
|
continue
|
|
indent = len(s) - len(s.lstrip())
|
|
content = s.lstrip()
|
|
processed.append((indent, content))
|
|
|
|
def parse_block(idx, base_indent):
|
|
\"\"\"Parse a block at the given indentation level, return (result, next_idx).\"\"\"
|
|
if idx >= len(processed):
|
|
return None, idx
|
|
|
|
indent, content = processed[idx]
|
|
|
|
# Detect if this block is a list or mapping
|
|
if content.startswith('- '):
|
|
return parse_list(idx, indent)
|
|
else:
|
|
return parse_mapping(idx, indent)
|
|
|
|
def parse_mapping(idx, base_indent):
|
|
result = {}
|
|
while idx < len(processed):
|
|
indent, content = processed[idx]
|
|
if indent < base_indent:
|
|
break
|
|
if indent > base_indent:
|
|
break
|
|
if content.startswith('- '):
|
|
break
|
|
|
|
# Must be key: value
|
|
m = re.match(r'^([^:]+?):\s*(.*?)$', content)
|
|
if not m:
|
|
idx += 1
|
|
continue
|
|
|
|
key = m.group(1).strip()
|
|
val_str = m.group(2).strip()
|
|
|
|
# Strip inline comment from value
|
|
if val_str and not val_str.startswith(('\"', \"'\")):
|
|
cm = re.match(r'^(.*?)\s+#.*$', val_str)
|
|
if cm:
|
|
val_str = cm.group(1).rstrip()
|
|
|
|
if val_str and not val_str.startswith('#'):
|
|
result[key] = _parse_value(val_str)
|
|
idx += 1
|
|
else:
|
|
# Value is a nested block
|
|
idx += 1
|
|
if idx < len(processed) and processed[idx][0] > base_indent:
|
|
child_indent = processed[idx][0]
|
|
child, idx = parse_block(idx, child_indent)
|
|
result[key] = child if child is not None else {}
|
|
else:
|
|
result[key] = {}
|
|
|
|
return result, idx
|
|
|
|
def parse_list(idx, base_indent):
|
|
result = []
|
|
while idx < len(processed):
|
|
indent, content = processed[idx]
|
|
if indent < base_indent:
|
|
break
|
|
if indent > base_indent:
|
|
break
|
|
if not content.startswith('- '):
|
|
break
|
|
|
|
item_str = content[2:].strip()
|
|
# Check if it's a mapping item: '- key: value'
|
|
m = re.match(r'^([^:]+?):\s*(.*?)$', item_str)
|
|
if m:
|
|
# It's a mapping -- parse this item and continuation lines
|
|
obj = {}
|
|
key = m.group(1).strip()
|
|
val_str = m.group(2).strip()
|
|
if val_str and not val_str.startswith('#'):
|
|
if not val_str.startswith(('\"', \"'\")):
|
|
cm = re.match(r'^(.*?)\s+#.*$', val_str)
|
|
if cm:
|
|
val_str = cm.group(1).rstrip()
|
|
obj[key] = _parse_value(val_str)
|
|
else:
|
|
idx += 1
|
|
if idx < len(processed) and processed[idx][0] > indent:
|
|
child_indent = processed[idx][0]
|
|
child, idx = parse_block(idx, child_indent)
|
|
obj[key] = child if child is not None else {}
|
|
else:
|
|
obj[key] = {}
|
|
# Parse remaining keys at item indent + 2
|
|
item_indent = indent + 2
|
|
while idx < len(processed) and processed[idx][0] == item_indent:
|
|
ic = processed[idx][1]
|
|
if ic.startswith('- '):
|
|
break
|
|
im = re.match(r'^([^:]+?):\s*(.*?)$', ic)
|
|
if im:
|
|
ik = im.group(1).strip()
|
|
iv = im.group(2).strip()
|
|
if iv and not iv.startswith('#'):
|
|
if not iv.startswith(('\"', \"'\")):
|
|
icm = re.match(r'^(.*?)\s+#.*$', iv)
|
|
if icm:
|
|
iv = icm.group(1).rstrip()
|
|
obj[ik] = _parse_value(iv)
|
|
idx += 1
|
|
else:
|
|
idx += 1
|
|
if idx < len(processed) and processed[idx][0] > item_indent:
|
|
child_indent = processed[idx][0]
|
|
child, idx = parse_block(idx, child_indent)
|
|
obj[ik] = child if child is not None else {}
|
|
else:
|
|
obj[ik] = {}
|
|
else:
|
|
idx += 1
|
|
result.append(obj)
|
|
continue
|
|
|
|
idx += 1
|
|
# Parse remaining keys of this list item
|
|
item_indent = indent + 2
|
|
while idx < len(processed) and processed[idx][0] >= item_indent:
|
|
if processed[idx][0] == item_indent:
|
|
ic = processed[idx][1]
|
|
if ic.startswith('- '):
|
|
break
|
|
im = re.match(r'^([^:]+?):\s*(.*?)$', ic)
|
|
if im:
|
|
ik = im.group(1).strip()
|
|
iv = im.group(2).strip()
|
|
if iv and not iv.startswith('#'):
|
|
if not iv.startswith(('\"', \"'\")):
|
|
icm = re.match(r'^(.*?)\s+#.*$', iv)
|
|
if icm:
|
|
iv = icm.group(1).rstrip()
|
|
obj[ik] = _parse_value(iv)
|
|
idx += 1
|
|
else:
|
|
idx += 1
|
|
if idx < len(processed) and processed[idx][0] > item_indent:
|
|
child_indent = processed[idx][0]
|
|
child, idx = parse_block(idx, child_indent)
|
|
obj[ik] = child if child is not None else {}
|
|
else:
|
|
obj[ik] = {}
|
|
else:
|
|
idx += 1
|
|
else:
|
|
# Deeper nesting handled by recursive calls above
|
|
break
|
|
result.append(obj)
|
|
else:
|
|
# Scalar list item
|
|
result.append(_parse_value(item_str))
|
|
idx += 1
|
|
|
|
return result, idx
|
|
|
|
result, _ = parse_block(0, 0)
|
|
return result if result is not None else {}
|
|
|
|
try:
|
|
import yaml
|
|
with open(sys.argv[1]) as f:
|
|
data = yaml.safe_load(f)
|
|
except ImportError:
|
|
data = parse_yaml_basic(sys.argv[1])
|
|
|
|
json.dump(data, sys.stdout, indent=2)
|
|
" "$CONFIG_FILE" > "$CONFIG_JSON" 2>/dev/null; then
|
|
error "Failed to parse YAML configuration: ${CONFIG_FILE}"
|
|
error "Check that the file is valid YAML"
|
|
error "For complex YAML features, install PyYAML: pip3 install pyyaml"
|
|
exit 1
|
|
fi
|
|
|
|
success "Configuration parsed"
|
|
}
|
|
|
|
# JSON field accessor -- uses jq if available, falls back to python3
|
|
json_get() {
|
|
local file="$1"
|
|
local query="$2"
|
|
local default="${3:-}"
|
|
|
|
local result
|
|
if command -v jq &>/dev/null; then
|
|
result=$(jq -r "$query // empty" "$file" 2>/dev/null) || result=""
|
|
else
|
|
result=$(python3 -c "
|
|
import json, sys
|
|
data = json.load(open(sys.argv[1]))
|
|
path = sys.argv[2]
|
|
# Simple jq-like path traversal for common patterns
|
|
if path.startswith('.'):
|
|
path = path[1:]
|
|
parts = path.replace('[]','').split('.')
|
|
obj = data
|
|
for p in parts:
|
|
if p == '':
|
|
continue
|
|
if isinstance(obj, dict):
|
|
obj = obj.get(p)
|
|
elif isinstance(obj, list) and p.isdigit():
|
|
obj = obj[int(p)]
|
|
else:
|
|
obj = None
|
|
if obj is None:
|
|
break
|
|
if obj is None:
|
|
print('')
|
|
elif isinstance(obj, bool):
|
|
print('true' if obj else 'false')
|
|
elif isinstance(obj, (dict, list)):
|
|
json.dump(obj, sys.stdout)
|
|
else:
|
|
print(obj)
|
|
" "$file" "$query" 2>/dev/null) || result=""
|
|
fi
|
|
|
|
if [[ -z "$result" ]] && [[ -n "$default" ]]; then
|
|
echo "$default"
|
|
else
|
|
echo "$result"
|
|
fi
|
|
}
|
|
|
|
# Get the number of VMs in config
|
|
vm_count() {
|
|
if command -v jq &>/dev/null; then
|
|
jq -r '.vms | length' "$CONFIG_JSON" 2>/dev/null || echo "0"
|
|
else
|
|
python3 -c "
|
|
import json
|
|
data = json.load(open('$CONFIG_JSON'))
|
|
print(len(data.get('vms', [])))
|
|
" 2>/dev/null || echo "0"
|
|
fi
|
|
}
|
|
|
|
# Get a field from a specific VM by index
|
|
vm_get() {
|
|
local index="$1"
|
|
local field="$2"
|
|
local default="${3:-}"
|
|
|
|
local result
|
|
if command -v jq &>/dev/null; then
|
|
result=$(jq -r ".vms[${index}].${field} // empty" "$CONFIG_JSON" 2>/dev/null) || result=""
|
|
else
|
|
result=$(python3 -c "
|
|
import json
|
|
data = json.load(open('$CONFIG_JSON'))
|
|
vm = data.get('vms', [])[${index}]
|
|
val = vm.get('${field}')
|
|
if val is None:
|
|
print('')
|
|
elif isinstance(val, bool):
|
|
print('true' if val else 'false')
|
|
else:
|
|
print(val)
|
|
" 2>/dev/null) || result=""
|
|
fi
|
|
|
|
if [[ -z "$result" ]] && [[ -n "$default" ]]; then
|
|
echo "$default"
|
|
else
|
|
echo "$result"
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Configuration loading
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Proxmox connection settings
|
|
PVE_HOST=""
|
|
PVE_NODE=""
|
|
PVE_STORAGE=""
|
|
PVE_ISO_STORAGE=""
|
|
PVE_BRIDGE=""
|
|
PVE_METHOD=""
|
|
PVE_SSH_USER=""
|
|
PVE_API_TOKEN_ID=""
|
|
PVE_POOL=""
|
|
|
|
# Default VM settings
|
|
DEF_CORES=""
|
|
DEF_MEMORY=""
|
|
DEF_DISK=""
|
|
DEF_CHANNEL=""
|
|
DEF_START_VMID=""
|
|
|
|
# VM arrays (indexed by position)
|
|
declare -a VM_NAMES=()
|
|
declare -a VM_APPS=()
|
|
declare -a VM_APPLY_DEFAULTS=()
|
|
declare -a VM_CORES=()
|
|
declare -a VM_MEMORY=()
|
|
declare -a VM_DISK=()
|
|
declare -a VM_VMIDS=()
|
|
declare -a VM_CLUSTERS=()
|
|
declare -a VM_CLUSTER_ROLES=()
|
|
|
|
# Path to the resolved proxmox connection config file (set by find_proxmox_config)
|
|
PROXMOX_CONFIG_RESOLVED=""
|
|
|
|
# Auto-discover proxmox.yaml: --proxmox flag > script directory > cwd
|
|
find_proxmox_config() {
|
|
if [[ -n "$PROXMOX_CONFIG" ]]; then
|
|
if [[ ! -f "$PROXMOX_CONFIG" ]]; then
|
|
error "Proxmox config file not found: ${PROXMOX_CONFIG}"
|
|
exit 1
|
|
fi
|
|
PROXMOX_CONFIG_RESOLVED="$PROXMOX_CONFIG"
|
|
detail "Using proxmox config: ${PROXMOX_CONFIG_RESOLVED} (--proxmox)"
|
|
return 0
|
|
fi
|
|
|
|
local script_dir
|
|
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
if [[ -f "${script_dir}/proxmox.yaml" ]]; then
|
|
PROXMOX_CONFIG_RESOLVED="${script_dir}/proxmox.yaml"
|
|
detail "Using proxmox config: ${PROXMOX_CONFIG_RESOLVED} (auto-discovered)"
|
|
return 0
|
|
fi
|
|
|
|
if [[ -f "proxmox.yaml" ]]; then
|
|
PROXMOX_CONFIG_RESOLVED="$(pwd)/proxmox.yaml"
|
|
detail "Using proxmox config: ${PROXMOX_CONFIG_RESOLVED} (auto-discovered)"
|
|
return 0
|
|
fi
|
|
|
|
# No proxmox.yaml found -- caller decides if this is an error
|
|
return 1
|
|
}
|
|
|
|
# Parse proxmox.yaml (flat key: value structure) and set PVE_* globals as base values
|
|
load_proxmox_config() {
|
|
local pxconfig="$1"
|
|
|
|
local px_json
|
|
px_json=$(mktemp "${WORKDIR}/proxmox-config-XXXXXX.json")
|
|
|
|
if ! python3 -c "
|
|
import json, sys, re
|
|
|
|
result = {}
|
|
with open(sys.argv[1]) as f:
|
|
for line in f:
|
|
s = line.strip()
|
|
if not s or s.startswith('#'):
|
|
continue
|
|
m = re.match(r'^([^:]+?):\s*(.*?)$', s)
|
|
if m:
|
|
key = m.group(1).strip()
|
|
val = m.group(2).strip()
|
|
# Strip inline comments
|
|
if val and not val.startswith(('\"', \"'\")):
|
|
cm = re.match(r'^(.*?)\s+#.*$', val)
|
|
if cm:
|
|
val = cm.group(1).rstrip()
|
|
if (val.startswith('\"') and val.endswith('\"')) or (val.startswith(\"'\") and val.endswith(\"'\")):
|
|
val = val[1:-1]
|
|
result[key] = val
|
|
|
|
json.dump(result, sys.stdout, indent=2)
|
|
" "$pxconfig" > "$px_json" 2>/dev/null; then
|
|
error "Failed to parse proxmox config: ${pxconfig}"
|
|
exit 1
|
|
fi
|
|
|
|
# Set base values from proxmox.yaml (lab config overlay and CLI flags applied later)
|
|
local val
|
|
val=$(json_get "$px_json" ".host" ""); [[ -n "$val" ]] && PVE_HOST="$val"
|
|
val=$(json_get "$px_json" ".node" ""); [[ -n "$val" ]] && PVE_NODE="$val"
|
|
val=$(json_get "$px_json" ".storage" ""); [[ -n "$val" ]] && PVE_STORAGE="$val"
|
|
val=$(json_get "$px_json" ".iso_storage" ""); [[ -n "$val" ]] && PVE_ISO_STORAGE="$val"
|
|
val=$(json_get "$px_json" ".bridge" ""); [[ -n "$val" ]] && PVE_BRIDGE="$val"
|
|
val=$(json_get "$px_json" ".ssh_user" ""); [[ -n "$val" ]] && PVE_SSH_USER="$val"
|
|
val=$(json_get "$px_json" ".api_token_id" ""); [[ -n "$val" ]] && PVE_API_TOKEN_ID="$val"
|
|
val=$(json_get "$px_json" ".pool" ""); [[ -n "$val" ]] && PVE_POOL="$val"
|
|
val=$(json_get "$px_json" ".method" ""); [[ -n "$val" ]] && PVE_METHOD="$val"
|
|
|
|
rm -f "$px_json"
|
|
}
|
|
|
|
load_config() {
|
|
step "Loading configuration values"
|
|
|
|
# Merge priority: proxmox.yaml (base) → lab config proxmox: (overlay) → CLI flags
|
|
# Step 1: proxmox.yaml base values were already loaded by the caller via
|
|
# load_proxmox_config(). PVE_* globals may already be set.
|
|
|
|
# Step 2: overlay from lab config proxmox: section (if present)
|
|
local val
|
|
val=$(json_get "$CONFIG_JSON" ".proxmox.host" ""); [[ -n "$val" ]] && PVE_HOST="$val"
|
|
val=$(json_get "$CONFIG_JSON" ".proxmox.node" ""); [[ -n "$val" ]] && PVE_NODE="$val"
|
|
val=$(json_get "$CONFIG_JSON" ".proxmox.storage" ""); [[ -n "$val" ]] && PVE_STORAGE="$val"
|
|
val=$(json_get "$CONFIG_JSON" ".proxmox.iso_storage" ""); [[ -n "$val" ]] && PVE_ISO_STORAGE="$val"
|
|
val=$(json_get "$CONFIG_JSON" ".proxmox.bridge" ""); [[ -n "$val" ]] && PVE_BRIDGE="$val"
|
|
val=$(json_get "$CONFIG_JSON" ".proxmox.ssh_user" ""); [[ -n "$val" ]] && PVE_SSH_USER="$val"
|
|
val=$(json_get "$CONFIG_JSON" ".proxmox.api_token_id" ""); [[ -n "$val" ]] && PVE_API_TOKEN_ID="$val"
|
|
val=$(json_get "$CONFIG_JSON" ".proxmox.pool" ""); [[ -n "$val" ]] && PVE_POOL="$val"
|
|
val=$(json_get "$CONFIG_JSON" ".proxmox.method" ""); [[ -n "$val" ]] && PVE_METHOD="$val"
|
|
|
|
# Step 3: apply defaults for anything still unset
|
|
[[ -z "$PVE_NODE" ]] && PVE_NODE="pve"
|
|
[[ -z "$PVE_STORAGE" ]] && PVE_STORAGE="local-lvm"
|
|
[[ -z "$PVE_ISO_STORAGE" ]] && PVE_ISO_STORAGE="local"
|
|
[[ -z "$PVE_BRIDGE" ]] && PVE_BRIDGE="vmbr0"
|
|
[[ -z "$PVE_SSH_USER" ]] && PVE_SSH_USER="root"
|
|
[[ -z "$PVE_METHOD" ]] && PVE_METHOD="ssh"
|
|
|
|
# Step 4: CLI flags override everything
|
|
if [[ -n "$METHOD" ]]; then
|
|
PVE_METHOD="$METHOD"
|
|
fi
|
|
if [[ -n "$HOST_OVERRIDE" ]]; then
|
|
PVE_HOST="$HOST_OVERRIDE"
|
|
fi
|
|
|
|
# Default VM settings
|
|
DEF_CORES=$(json_get "$CONFIG_JSON" ".defaults.cores" "4")
|
|
DEF_MEMORY=$(json_get "$CONFIG_JSON" ".defaults.memory" "8192")
|
|
DEF_DISK=$(json_get "$CONFIG_JSON" ".defaults.disk" "64")
|
|
DEF_CHANNEL=$(json_get "$CONFIG_JSON" ".defaults.channel" "stable")
|
|
DEF_START_VMID=$(json_get "$CONFIG_JSON" ".defaults.start_vmid" "200")
|
|
|
|
# Load VMs
|
|
local count
|
|
count=$(vm_count)
|
|
|
|
local i=0
|
|
while [[ $i -lt $count ]]; do
|
|
VM_NAMES+=("$(vm_get "$i" "name" "")")
|
|
VM_APPS+=("$(vm_get "$i" "app" "incus")")
|
|
VM_APPLY_DEFAULTS+=("$(vm_get "$i" "apply_defaults" "false")")
|
|
VM_CORES+=("$(vm_get "$i" "cores" "$DEF_CORES")")
|
|
VM_MEMORY+=("$(vm_get "$i" "memory" "$DEF_MEMORY")")
|
|
VM_DISK+=("$(vm_get "$i" "disk" "$DEF_DISK")")
|
|
VM_VMIDS+=("$(vm_get "$i" "vmid" "")")
|
|
VM_CLUSTERS+=("$(vm_get "$i" "cluster" "")")
|
|
VM_CLUSTER_ROLES+=("$(vm_get "$i" "cluster_role" "")")
|
|
|
|
i=$((i + 1))
|
|
done
|
|
|
|
success "Loaded ${count} VM definition(s)"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Configuration validation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
validate_config() {
|
|
step "Validating configuration"
|
|
|
|
local errors=0
|
|
|
|
# Required: Proxmox host
|
|
if [[ -z "$PVE_HOST" ]]; then
|
|
error "proxmox.host is required"
|
|
errors=$((errors + 1))
|
|
fi
|
|
|
|
# Method validation
|
|
case "$PVE_METHOD" in
|
|
ssh|api) ;;
|
|
*)
|
|
error "Invalid proxmox.method: ${PVE_METHOD} (must be ssh or api)"
|
|
errors=$((errors + 1))
|
|
;;
|
|
esac
|
|
|
|
# API method requires token
|
|
if [[ "$PVE_METHOD" == "api" ]]; then
|
|
if [[ -z "$PVE_API_TOKEN_ID" ]]; then
|
|
error "proxmox.api_token_id is required for API method"
|
|
error "Example: automation@pve!deploy"
|
|
errors=$((errors + 1))
|
|
fi
|
|
if [[ -z "${PROXMOX_TOKEN_SECRET:-}" ]]; then
|
|
error "PROXMOX_TOKEN_SECRET environment variable is required for API method"
|
|
error "Set it with: export PROXMOX_TOKEN_SECRET=\"<your-token-secret>\""
|
|
errors=$((errors + 1))
|
|
fi
|
|
fi
|
|
|
|
# Must have at least one VM
|
|
local count=${#VM_NAMES[@]}
|
|
if [[ $count -eq 0 ]]; then
|
|
error "No VMs defined in configuration"
|
|
errors=$((errors + 1))
|
|
fi
|
|
|
|
# Validate each VM
|
|
local seen_names=""
|
|
local cluster_inits=""
|
|
local i=0
|
|
while [[ $i -lt $count ]]; do
|
|
local name="${VM_NAMES[$i]}"
|
|
local app="${VM_APPS[$i]}"
|
|
local memory="${VM_MEMORY[$i]}"
|
|
local disk="${VM_DISK[$i]}"
|
|
local cluster="${VM_CLUSTERS[$i]}"
|
|
local role="${VM_CLUSTER_ROLES[$i]}"
|
|
|
|
# Name required
|
|
if [[ -z "$name" ]]; then
|
|
error "VM #$((i + 1)): name is required"
|
|
errors=$((errors + 1))
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
# Unique names
|
|
if echo "$seen_names" | grep -qw "$name"; then
|
|
error "VM '${name}': duplicate VM name"
|
|
errors=$((errors + 1))
|
|
fi
|
|
seen_names="${seen_names} ${name}"
|
|
|
|
# Valid app
|
|
case "$app" in
|
|
incus|operations-center) ;;
|
|
*)
|
|
error "VM '${name}': invalid app '${app}' (must be incus or operations-center)"
|
|
errors=$((errors + 1))
|
|
;;
|
|
esac
|
|
|
|
# Minimum memory
|
|
if [[ "$memory" -lt "$MIN_MEMORY_MIB" ]] 2>/dev/null; then
|
|
error "VM '${name}': memory ${memory} MiB is below minimum ${MIN_MEMORY_MIB} MiB"
|
|
errors=$((errors + 1))
|
|
fi
|
|
|
|
# Minimum disk
|
|
if [[ "$disk" -lt "$MIN_DISK_GIB" ]] 2>/dev/null; then
|
|
error "VM '${name}': disk ${disk} GiB is below minimum ${MIN_DISK_GIB} GiB"
|
|
errors=$((errors + 1))
|
|
fi
|
|
|
|
# Cluster role validation
|
|
if [[ -n "$role" ]] && [[ -z "$cluster" ]]; then
|
|
error "VM '${name}': cluster_role '${role}' set but no cluster name specified"
|
|
errors=$((errors + 1))
|
|
fi
|
|
if [[ -n "$role" ]]; then
|
|
case "$role" in
|
|
init|join) ;;
|
|
*)
|
|
error "VM '${name}': invalid cluster_role '${role}' (must be init or join)"
|
|
errors=$((errors + 1))
|
|
;;
|
|
esac
|
|
fi
|
|
|
|
# Track cluster init nodes
|
|
if [[ -n "$cluster" ]] && [[ "$role" == "init" ]]; then
|
|
if echo "$cluster_inits" | grep -qw "$cluster"; then
|
|
error "Cluster '${cluster}': multiple init nodes defined (only one allowed)"
|
|
errors=$((errors + 1))
|
|
fi
|
|
cluster_inits="${cluster_inits} ${cluster}"
|
|
fi
|
|
|
|
i=$((i + 1))
|
|
done
|
|
|
|
# VM filter validation
|
|
if [[ -n "$VM_FILTER" ]]; then
|
|
local found=false
|
|
for name in "${VM_NAMES[@]}"; do
|
|
if [[ "$name" == "$VM_FILTER" ]]; then
|
|
found=true
|
|
break
|
|
fi
|
|
done
|
|
if [[ "$found" != true ]]; then
|
|
error "VM '${VM_FILTER}' not found in configuration"
|
|
error "Available VMs: ${VM_NAMES[*]}"
|
|
errors=$((errors + 1))
|
|
fi
|
|
fi
|
|
|
|
if [[ $errors -gt 0 ]]; then
|
|
error "${errors} validation error(s) found"
|
|
exit 1
|
|
fi
|
|
|
|
success "Configuration valid"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Proxmox transport layer
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Execute a command on the Proxmox host
|
|
pve_run() {
|
|
local cmd="$1"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] ${PVE_METHOD}: ${cmd}"
|
|
return 0
|
|
fi
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
ssh -o BatchMode=yes -o ConnectTimeout=10 \
|
|
"${PVE_SSH_USER}@${PVE_HOST}" "$cmd"
|
|
;;
|
|
api)
|
|
error "API execution of arbitrary commands is not supported"
|
|
error "Use specific API functions instead"
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Execute a Proxmox API call (API method)
|
|
# Usage: pve_api METHOD ENDPOINT [DATA] [json]
|
|
# Pass "json" as 4th arg to send data as application/json instead of form-encoded.
|
|
pve_api() {
|
|
local method="$1"
|
|
local endpoint="$2"
|
|
local data="${3:-}"
|
|
local content_type="${4:-form}"
|
|
|
|
local url="https://${PVE_HOST}:8006/api2/json${endpoint}"
|
|
local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET:-}"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] API ${method} ${endpoint}"
|
|
if [[ -n "$data" ]]; then
|
|
detail "[dry-run] data: ${data}"
|
|
fi
|
|
return 0
|
|
fi
|
|
|
|
local curl_args=(-fsSk -H "$auth")
|
|
if [[ "$content_type" == "json" ]]; then
|
|
curl_args+=(-H "Content-Type: application/json")
|
|
fi
|
|
|
|
case "$method" in
|
|
GET)
|
|
curl "${curl_args[@]}" "$url"
|
|
;;
|
|
POST)
|
|
if [[ -n "$data" ]]; then
|
|
curl "${curl_args[@]}" -X POST -d "$data" "$url"
|
|
else
|
|
curl "${curl_args[@]}" -X POST "$url"
|
|
fi
|
|
;;
|
|
PUT)
|
|
if [[ -n "$data" ]]; then
|
|
curl "${curl_args[@]}" -X PUT -d "$data" "$url"
|
|
else
|
|
curl "${curl_args[@]}" -X PUT "$url"
|
|
fi
|
|
;;
|
|
DELETE)
|
|
curl "${curl_args[@]}" -X DELETE "$url"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Upload a file to Proxmox ISO storage
|
|
pve_upload() {
|
|
local local_path="$1"
|
|
local remote_filename="$2"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] upload: ${local_path} -> ${PVE_ISO_STORAGE}:iso/${remote_filename}"
|
|
return 0
|
|
fi
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
local iso_dir
|
|
# Query the actual path of the ISO storage
|
|
iso_dir=$(pve_run "pvesm path ${PVE_ISO_STORAGE}:iso/test 2>/dev/null" | sed 's|/test$||') || true
|
|
if [[ -z "$iso_dir" ]]; then
|
|
# Common default paths
|
|
iso_dir="/var/lib/vz/template/iso"
|
|
fi
|
|
scp -o BatchMode=yes -o ConnectTimeout=10 \
|
|
"$local_path" "${PVE_SSH_USER}@${PVE_HOST}:${iso_dir}/${remote_filename}"
|
|
;;
|
|
api)
|
|
local upload_response upload_rc
|
|
upload_response=$(curl -sSk \
|
|
-H "Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET:-}" \
|
|
-X POST \
|
|
-F "content=iso" \
|
|
-F "filename=@${local_path};filename=${remote_filename}" \
|
|
"https://${PVE_HOST}:8006/api2/json/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/upload" \
|
|
-w "\n%{http_code}" 2>&1) || true
|
|
local http_code
|
|
http_code=$(echo "$upload_response" | tail -1)
|
|
local body
|
|
body=$(echo "$upload_response" | sed '$d')
|
|
|
|
if [[ "$http_code" -ge 200 ]] && [[ "$http_code" -lt 300 ]] 2>/dev/null; then
|
|
return 0
|
|
else
|
|
error "Upload failed (HTTP ${http_code})"
|
|
# Extract error message from JSON response
|
|
local api_error
|
|
api_error=$(echo "$body" | python3 -c "
|
|
import json, sys
|
|
try:
|
|
data = json.load(sys.stdin)
|
|
errors = data.get('errors', {})
|
|
if errors:
|
|
for k, v in errors.items():
|
|
print(f' {k}: {v}')
|
|
elif data.get('message'):
|
|
print(f' {data[\"message\"]}')
|
|
else:
|
|
print(json.dumps(data, indent=2))
|
|
except:
|
|
print(sys.stdin.read() if hasattr(sys.stdin, 'read') else '')
|
|
" 2>/dev/null) || api_error="$body"
|
|
if [[ -n "$api_error" ]]; then
|
|
error "Proxmox API response:"
|
|
echo "$api_error" | while IFS= read -r line; do
|
|
error " ${line}"
|
|
done
|
|
fi
|
|
return 1
|
|
fi
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Check if an ISO exists on Proxmox storage
|
|
pve_iso_exists() {
|
|
local filename="$1"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
return 1
|
|
fi
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
pve_run "pvesm list ${PVE_ISO_STORAGE} --content iso 2>/dev/null" | grep -q "$filename"
|
|
;;
|
|
api)
|
|
pve_api GET "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content?content=iso" | \
|
|
grep -q "$filename"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Get list of existing VMIDs
|
|
pve_list_vmids() {
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
echo ""
|
|
return 0
|
|
fi
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
pve_run "qm list 2>/dev/null" | awk 'NR>1 {print $1}' || true
|
|
;;
|
|
api)
|
|
pve_api GET "/nodes/${PVE_NODE}/qemu" 2>/dev/null | \
|
|
python3 -c "
|
|
import json, sys
|
|
data = json.load(sys.stdin)
|
|
for vm in data.get('data', []):
|
|
print(vm.get('vmid', ''))
|
|
" 2>/dev/null || true
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Create a VM via qm create
|
|
pve_create_vm() {
|
|
local vmid="$1"
|
|
local name="$2"
|
|
local cores="$3"
|
|
local memory="$4"
|
|
local disk_gib="$5"
|
|
local iso_filename="$6"
|
|
local seed_filename="$7"
|
|
|
|
local description
|
|
description=$(build_vm_description)
|
|
|
|
local cmd="qm create ${vmid}"
|
|
cmd+=" --name ${name}"
|
|
cmd+=" --description '${description}'"
|
|
cmd+=" --ostype l26"
|
|
cmd+=" --bios ovmf"
|
|
cmd+=" --machine q35"
|
|
cmd+=" --efidisk0 ${PVE_STORAGE}:0,efitype=4m,pre-enrolled-keys=0"
|
|
cmd+=" --tpmstate0 ${PVE_STORAGE}:1,version=v2.0"
|
|
cmd+=" --cpu host"
|
|
cmd+=" --cores ${cores}"
|
|
cmd+=" --memory ${memory}"
|
|
cmd+=" --balloon 0"
|
|
cmd+=" --scsihw virtio-scsi-pci"
|
|
cmd+=" --scsi0 ${PVE_STORAGE}:${disk_gib},iothread=1"
|
|
cmd+=" --ide2 ${PVE_ISO_STORAGE}:iso/${iso_filename},media=cdrom"
|
|
cmd+=" --ide3 ${PVE_ISO_STORAGE}:iso/${seed_filename},media=cdrom"
|
|
cmd+=" --net0 virtio,bridge=${PVE_BRIDGE}"
|
|
cmd+=" --boot order=ide2\\;scsi0"
|
|
cmd+=" --agent 1"
|
|
if [[ -n "$PVE_POOL" ]]; then
|
|
cmd+=" --pool ${PVE_POOL}"
|
|
fi
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] ${cmd}"
|
|
return 0
|
|
fi
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
pve_run "$cmd"
|
|
;;
|
|
api)
|
|
# Build JSON payload -- the Proxmox API requires JSON for VM
|
|
# creation because comma-separated disk/net specs are misinterpreted
|
|
# when sent as form-encoded data.
|
|
local json_payload
|
|
json_payload=$(python3 -c "
|
|
import json
|
|
payload = {
|
|
'vmid': ${vmid},
|
|
'name': '${name}',
|
|
'description': '''${description}''',
|
|
'ostype': 'l26',
|
|
'bios': 'ovmf',
|
|
'machine': 'q35',
|
|
'efidisk0': '${PVE_STORAGE}:0,efitype=4m,pre-enrolled-keys=0',
|
|
'tpmstate0': '${PVE_STORAGE}:1,version=v2.0',
|
|
'cpu': 'host',
|
|
'cores': ${cores},
|
|
'memory': ${memory},
|
|
'balloon': 0,
|
|
'scsihw': 'virtio-scsi-pci',
|
|
'scsi0': '${PVE_STORAGE}:${disk_gib},iothread=1',
|
|
'ide2': '${PVE_ISO_STORAGE}:iso/${iso_filename},media=cdrom',
|
|
'ide3': '${PVE_ISO_STORAGE}:iso/${seed_filename},media=cdrom',
|
|
'net0': 'virtio,bridge=${PVE_BRIDGE}',
|
|
'boot': 'order=ide2;scsi0',
|
|
'agent': 1,
|
|
}
|
|
pool = '${PVE_POOL}'
|
|
if pool:
|
|
payload['pool'] = pool
|
|
print(json.dumps(payload))
|
|
")
|
|
local create_response
|
|
create_response=$(pve_api POST "/nodes/${PVE_NODE}/qemu" "$json_payload" json)
|
|
# VM creation is async -- the API returns a UPID. Wait for the
|
|
# task to finish before returning, otherwise the caller may try
|
|
# to start or configure the VM while it's still being created.
|
|
local upid
|
|
upid=$(echo "$create_response" | python3 -c "import json,sys; print(json.load(sys.stdin).get('data',''))" 2>/dev/null) || upid=""
|
|
if [[ -n "$upid" ]]; then
|
|
local encoded_upid
|
|
encoded_upid=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${upid}', safe=''))" 2>/dev/null)
|
|
local wait_elapsed=0
|
|
while [[ $wait_elapsed -lt 60 ]]; do
|
|
sleep 2
|
|
wait_elapsed=$((wait_elapsed + 2))
|
|
local task_status
|
|
task_status=$(pve_api GET "/nodes/${PVE_NODE}/tasks/${encoded_upid}/status" 2>/dev/null | \
|
|
python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status',''))" 2>/dev/null) || task_status=""
|
|
if [[ "$task_status" == "stopped" ]]; then
|
|
break
|
|
fi
|
|
done
|
|
fi
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Start a VM
|
|
pve_start_vm() {
|
|
local vmid="$1"
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh) pve_run "qm start ${vmid}" ;;
|
|
api) pve_api POST "/nodes/${PVE_NODE}/qemu/${vmid}/status/start" >/dev/null ;;
|
|
esac
|
|
}
|
|
|
|
# Stop a VM
|
|
pve_stop_vm() {
|
|
local vmid="$1"
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh) pve_run "qm stop ${vmid}" ;;
|
|
api) pve_api POST "/nodes/${PVE_NODE}/qemu/${vmid}/status/stop" >/dev/null ;;
|
|
esac
|
|
}
|
|
|
|
# Destroy a VM
|
|
pve_destroy_vm() {
|
|
local vmid="$1"
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh) pve_run "qm destroy ${vmid} --purge" ;;
|
|
api) pve_api DELETE "/nodes/${PVE_NODE}/qemu/${vmid}?purge=1" >/dev/null ;;
|
|
esac
|
|
}
|
|
|
|
# Set VM configuration
|
|
pve_set_vm() {
|
|
local vmid="$1"
|
|
shift
|
|
local opts="$*"
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh) pve_run "qm set ${vmid} ${opts}" ;;
|
|
api)
|
|
# Convert CLI-style opts (--key value) to JSON.
|
|
# JSON avoids form-encoding ambiguity with values containing = and ;
|
|
# Duplicate keys (e.g. --delete ide2 --delete ide3) are merged with commas.
|
|
# Shell escapes (e.g. \;) are stripped since the API doesn't need them.
|
|
local json_payload
|
|
json_payload=$(echo "$opts" | python3 -c "
|
|
import json, sys
|
|
tokens = sys.stdin.read().split()
|
|
obj = {}
|
|
i = 0
|
|
while i < len(tokens):
|
|
t = tokens[i]
|
|
if t.startswith('--'):
|
|
key = t.lstrip('-')
|
|
if i + 1 < len(tokens) and not tokens[i+1].startswith('--'):
|
|
val = tokens[i+1].replace(chr(92), '')
|
|
i += 2
|
|
else:
|
|
val = ''
|
|
i += 1
|
|
if key in obj and obj[key]:
|
|
obj[key] = obj[key] + ',' + val
|
|
else:
|
|
obj[key] = val
|
|
else:
|
|
i += 1
|
|
print(json.dumps(obj))
|
|
" 2>/dev/null)
|
|
pve_api PUT "/nodes/${PVE_NODE}/qemu/${vmid}/config" "$json_payload" json >/dev/null
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Get VM status
|
|
pve_vm_status() {
|
|
local vmid="$1"
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
pve_run "qm status ${vmid}" | awk '{print $2}'
|
|
;;
|
|
api)
|
|
pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" | \
|
|
python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status','unknown'))" 2>/dev/null || echo "unknown"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Get VM uptime (seconds); returns 0 if stopped or unknown
|
|
pve_vm_uptime() {
|
|
local vmid="$1"
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
pve_run "qm status ${vmid} --verbose" | awk '/^uptime:/{print $2}'
|
|
;;
|
|
api)
|
|
pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" | \
|
|
python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('uptime',0))" 2>/dev/null || echo "0"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Get cumulative write bytes for scsi0 (from QEMU blockstat).
|
|
# Returns 0 if unavailable. Only works with api method.
|
|
pve_vm_disk_writes() {
|
|
local vmid="$1"
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
echo "0"
|
|
;;
|
|
api)
|
|
pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" | \
|
|
python3 -c "
|
|
import json,sys
|
|
d = json.load(sys.stdin).get('data',{})
|
|
bs = d.get('blockstat',{}).get('scsi0',{})
|
|
print(bs.get('wr_bytes',0))
|
|
" 2>/dev/null || echo "0"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Check if a VMID exists
|
|
pve_vm_exists() {
|
|
local vmid="$1"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
return 1
|
|
fi
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
pve_run "qm status ${vmid}" &>/dev/null
|
|
;;
|
|
api)
|
|
pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" &>/dev/null
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Get VM's IP address (best-effort via QEMU guest agent)
|
|
pve_vm_ip() {
|
|
local vmid="$1"
|
|
|
|
# Method 1: Try QEMU guest agent (works if agent is installed)
|
|
local ip=""
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
ip=$(pve_run "qm guest cmd ${vmid} network-get-interfaces 2>/dev/null" | \
|
|
python3 -c "
|
|
import json, sys
|
|
try:
|
|
data = json.load(sys.stdin)
|
|
for iface in data:
|
|
if iface.get('name') == 'lo':
|
|
continue
|
|
for addr in iface.get('ip-addresses', []):
|
|
if addr.get('ip-address-type') == 'ipv4':
|
|
print(addr['ip-address'])
|
|
sys.exit(0)
|
|
except:
|
|
pass
|
|
" 2>/dev/null) || true
|
|
;;
|
|
api)
|
|
ip=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/agent/network-get-interfaces" 2>/dev/null | \
|
|
python3 -c "
|
|
import json, sys
|
|
try:
|
|
data = json.load(sys.stdin).get('data', {}).get('result', [])
|
|
for iface in data:
|
|
if iface.get('name') == 'lo':
|
|
continue
|
|
for addr in iface.get('ip-addresses', []):
|
|
if addr.get('ip-address-type') == 'ipv4':
|
|
print(addr['ip-address'])
|
|
sys.exit(0)
|
|
except:
|
|
pass
|
|
" 2>/dev/null) || true
|
|
;;
|
|
esac
|
|
|
|
if [[ -n "$ip" ]]; then
|
|
echo "$ip"
|
|
return 0
|
|
fi
|
|
|
|
# Method 2: ARP-based lookup using VM's MAC address.
|
|
# Works without guest agent (e.g. IncusOS which is immutable).
|
|
# Get MAC from VM config, flush stale ARP, ping sweep, look up.
|
|
local mac=""
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
mac=$(pve_run "qm config ${vmid}" 2>/dev/null | \
|
|
awk -F'=' '/^net0:/{print $1}' | grep -oiE '[0-9a-f:]{17}') || true
|
|
;;
|
|
api)
|
|
mac=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null | \
|
|
python3 -c "
|
|
import json, sys, re
|
|
try:
|
|
cfg = json.load(sys.stdin).get('data', {})
|
|
net0 = cfg.get('net0', '')
|
|
m = re.search(r'([0-9A-Fa-f:]{17})', net0)
|
|
if m:
|
|
print(m.group(1).lower())
|
|
except:
|
|
pass
|
|
" 2>/dev/null) || true
|
|
;;
|
|
esac
|
|
|
|
if [[ -z "$mac" ]]; then
|
|
return 1
|
|
fi
|
|
|
|
# Populate ARP table to find this MAC's IP address.
|
|
# Strategy: first check existing ARP, then try broadcast ping, then
|
|
# fall back to a parallel ping sweep of the local subnet.
|
|
ip neigh flush nud stale 2>/dev/null || true
|
|
|
|
local bridge_subnet
|
|
bridge_subnet=$(ip -4 addr show | awk '/inet / && !/127\.0\.0/ {print $2; exit}')
|
|
|
|
# Try broadcast ping first (fast, but many hosts ignore it)
|
|
if [[ -n "$bridge_subnet" ]]; then
|
|
local broadcast
|
|
broadcast=$(python3 -c "import ipaddress; print(ipaddress.ip_interface('${bridge_subnet}').network.broadcast_address)" 2>/dev/null) || true
|
|
if [[ -n "$broadcast" ]]; then
|
|
ping -c 2 -W 1 -b "$broadcast" &>/dev/null || true
|
|
sleep 1
|
|
fi
|
|
fi
|
|
|
|
# Check ARP table
|
|
ip=$(ip neigh show | awk -v mac="$mac" 'tolower($0) ~ tolower(mac) && (/REACHABLE/ || /DELAY/ || /STALE/) {print $1; exit}') || true
|
|
|
|
if [[ -n "$ip" ]] && ping -c 1 -W 1 "$ip" &>/dev/null; then
|
|
echo "$ip"
|
|
return 0
|
|
fi
|
|
|
|
# Broadcast didn't work — parallel ping sweep of the subnet
|
|
if [[ -n "$bridge_subnet" ]]; then
|
|
local net_prefix
|
|
net_prefix=$(python3 -c "
|
|
import ipaddress
|
|
net = ipaddress.ip_interface('${bridge_subnet}').network
|
|
for h in net.hosts():
|
|
print(h)
|
|
" 2>/dev/null)
|
|
if [[ -n "$net_prefix" ]]; then
|
|
# Ping all hosts in parallel (background, 1s timeout each)
|
|
while IFS= read -r host_ip; do
|
|
ping -c 1 -W 1 "$host_ip" &>/dev/null &
|
|
done <<< "$net_prefix"
|
|
wait
|
|
sleep 1
|
|
|
|
# Re-check ARP table
|
|
ip=$(ip neigh show | awk -v mac="$mac" 'tolower($0) ~ tolower(mac) && (/REACHABLE/ || /DELAY/) {print $1; exit}') || true
|
|
|
|
if [[ -n "$ip" ]] && ping -c 1 -W 1 "$ip" &>/dev/null; then
|
|
echo "$ip"
|
|
return 0
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
return 1
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# VM management marker
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Build the description string for a managed VM
|
|
build_vm_description() {
|
|
local config_basename
|
|
config_basename=$(basename "$CONFIG_FILE")
|
|
local timestamp
|
|
timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
local marker="${MANAGED_MARKER} config=${config_basename} created=${timestamp} by=${SCRIPT_NAME}/${VERSION}"
|
|
if [[ -n "$PVE_POOL" ]]; then
|
|
marker+=" pool=${PVE_POOL}"
|
|
fi
|
|
echo "$marker"
|
|
}
|
|
|
|
# Check if a VM has our management marker in its description
|
|
pve_vm_is_managed() {
|
|
local vmid="$1"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
return 1
|
|
fi
|
|
|
|
local config_output=""
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
config_output=$(pve_run "qm config ${vmid}" 2>/dev/null) || return 1
|
|
;;
|
|
api)
|
|
config_output=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null) || return 1
|
|
;;
|
|
esac
|
|
|
|
echo "$config_output" | grep -qF "$MANAGED_MARKER"
|
|
}
|
|
|
|
# Check if a VM still has install media (ide2) attached.
|
|
# Returns 0 if install media is present, 1 if not (already installed).
|
|
pve_vm_has_install_media() {
|
|
local vmid="$1"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
return 1
|
|
fi
|
|
|
|
local config_output=""
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
config_output=$(pve_run "qm config ${vmid}" 2>/dev/null) || return 1
|
|
;;
|
|
api)
|
|
config_output=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null) || return 1
|
|
;;
|
|
esac
|
|
|
|
echo "$config_output" | grep -q "ide2"
|
|
}
|
|
|
|
# List all VMs as "vmid name" pairs (one per line)
|
|
pve_list_vms() {
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
echo ""
|
|
return 0
|
|
fi
|
|
|
|
# When a resource pool is configured, list only pool members (type=qemu)
|
|
if [[ -n "$PVE_POOL" ]]; then
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
pve_run "pvesh get /pools/${PVE_POOL} --output-format json 2>/dev/null" | \
|
|
python3 -c "
|
|
import json, sys
|
|
data = json.load(sys.stdin)
|
|
for m in data.get('members', []):
|
|
if m.get('type') == 'qemu':
|
|
print('{} {}'.format(m.get('vmid', ''), m.get('name', '')))
|
|
" 2>/dev/null || true
|
|
;;
|
|
api)
|
|
pve_api GET "/pools/${PVE_POOL}" 2>/dev/null | \
|
|
python3 -c "
|
|
import json, sys
|
|
data = json.load(sys.stdin).get('data', {})
|
|
for m in data.get('members', []):
|
|
if m.get('type') == 'qemu':
|
|
print('{} {}'.format(m.get('vmid', ''), m.get('name', '')))
|
|
" 2>/dev/null || true
|
|
;;
|
|
esac
|
|
return 0
|
|
fi
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
pve_run "qm list 2>/dev/null" | awk 'NR>1 {print $1, $2}' || true
|
|
;;
|
|
api)
|
|
pve_api GET "/nodes/${PVE_NODE}/qemu" 2>/dev/null | \
|
|
python3 -c "
|
|
import json, sys
|
|
data = json.load(sys.stdin)
|
|
for vm in data.get('data', []):
|
|
print('{} {}'.format(vm.get('vmid', ''), vm.get('name', '')))
|
|
" 2>/dev/null || true
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Pre-flight checks
|
|
# ---------------------------------------------------------------------------
|
|
|
|
preflight_checks() {
|
|
step "Running pre-flight checks"
|
|
|
|
local errors=0
|
|
local warnings=0
|
|
|
|
# 1. Check that storage pools exist on Proxmox
|
|
if [[ "$DRY_RUN" != true ]]; then
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
if ! pve_run "pvesm status 2>/dev/null" | awk 'NR>1 {print \$1}' | grep -qw "$PVE_STORAGE"; then
|
|
error "Storage pool '${PVE_STORAGE}' not found on Proxmox"
|
|
error "Available pools: $(pve_run "pvesm status 2>/dev/null" | awk 'NR>1 {print \$1}' | tr '\n' ' ')"
|
|
error "Fix: update 'proxmox.storage' in your config file"
|
|
errors=$((errors + 1))
|
|
fi
|
|
if ! pve_run "pvesm status 2>/dev/null" | awk 'NR>1 {print \$1}' | grep -qw "$PVE_ISO_STORAGE"; then
|
|
error "ISO storage '${PVE_ISO_STORAGE}' not found on Proxmox"
|
|
error "Available pools: $(pve_run "pvesm status 2>/dev/null" | awk 'NR>1 {print \$1}' | tr '\n' ' ')"
|
|
error "Fix: update 'proxmox.iso_storage' in your config file"
|
|
errors=$((errors + 1))
|
|
fi
|
|
;;
|
|
api)
|
|
local storage_json
|
|
storage_json=$(pve_api GET "/nodes/${PVE_NODE}/storage" 2>/dev/null) || storage_json=""
|
|
local storage_list
|
|
storage_list=$(echo "$storage_json" | python3 -c "
|
|
import json, sys
|
|
data = json.load(sys.stdin)
|
|
for s in data.get('data', []):
|
|
print(s.get('storage', ''))
|
|
" 2>/dev/null) || storage_list=""
|
|
if ! echo "$storage_list" | grep -qw "$PVE_STORAGE"; then
|
|
error "Storage pool '${PVE_STORAGE}' not found on Proxmox"
|
|
error "Available pools: $(echo "$storage_list" | tr '\n' ' ')"
|
|
error "Fix: update 'proxmox.storage' in your config file"
|
|
errors=$((errors + 1))
|
|
fi
|
|
if ! echo "$storage_list" | grep -qw "$PVE_ISO_STORAGE"; then
|
|
error "ISO storage '${PVE_ISO_STORAGE}' not found on Proxmox"
|
|
error "Available pools: $(echo "$storage_list" | tr '\n' ' ')"
|
|
error "Fix: update 'proxmox.iso_storage' in your config file"
|
|
errors=$((errors + 1))
|
|
else
|
|
# Check that ISO storage accepts 'iso' content type
|
|
local iso_content
|
|
iso_content=$(echo "$storage_json" | python3 -c "
|
|
import json, sys
|
|
data = json.load(sys.stdin)
|
|
for s in data.get('data', []):
|
|
if s.get('storage') == '${PVE_ISO_STORAGE}':
|
|
print(s.get('content', ''))
|
|
break
|
|
" 2>/dev/null) || iso_content=""
|
|
if [[ -n "$iso_content" ]] && ! echo "$iso_content" | grep -q "iso"; then
|
|
error "Storage '${PVE_ISO_STORAGE}' does not accept ISO content (content types: ${iso_content})"
|
|
error "Fix: enable 'iso' content on '${PVE_ISO_STORAGE}' in Proxmox, or use a different iso_storage"
|
|
errors=$((errors + 1))
|
|
fi
|
|
fi
|
|
;;
|
|
esac
|
|
|
|
# 2. Network bridge check (SSH only, warn-level)
|
|
if [[ "$PVE_METHOD" == "ssh" ]]; then
|
|
if ! pve_run "ip link show ${PVE_BRIDGE} 2>/dev/null" &>/dev/null; then
|
|
warn "Network bridge '${PVE_BRIDGE}' not found on Proxmox"
|
|
warn "VMs may fail to get network connectivity"
|
|
warn "Fix: update 'proxmox.bridge' in your config file"
|
|
warnings=$((warnings + 1))
|
|
fi
|
|
fi
|
|
|
|
# 3. ISO availability: check if already on Proxmox or CDN reachable
|
|
if [[ -z "$LOCAL_ISO" ]]; then
|
|
local any_iso_present=false
|
|
local iso_list=""
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
iso_list=$(pve_run "pvesm list ${PVE_ISO_STORAGE} --content iso 2>/dev/null" 2>/dev/null) || iso_list=""
|
|
;;
|
|
api)
|
|
iso_list=$(pve_api GET "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content?content=iso" 2>/dev/null) || iso_list=""
|
|
;;
|
|
esac
|
|
if echo "$iso_list" | grep -q "IncusOS_"; then
|
|
any_iso_present=true
|
|
fi
|
|
if [[ "$any_iso_present" != true ]]; then
|
|
# No ISO on Proxmox -- check if CDN is reachable
|
|
if ! curl -fsSL --connect-timeout 5 -o /dev/null "$CDN_INDEX" 2>/dev/null; then
|
|
error "No IncusOS ISO found on Proxmox and CDN is unreachable: ${CDN_INDEX}"
|
|
error "Fix: provide a local ISO with --iso FILE, or check internet connectivity"
|
|
errors=$((errors + 1))
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# 4. Resource pool check (if configured)
|
|
if [[ -n "$PVE_POOL" ]] && [[ "$DRY_RUN" != true ]]; then
|
|
local pool_ok=false
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
if pve_run "pvesh get /pools/${PVE_POOL} --output-format json 2>/dev/null" &>/dev/null; then
|
|
pool_ok=true
|
|
fi
|
|
;;
|
|
api)
|
|
if pve_api GET "/pools/${PVE_POOL}" &>/dev/null; then
|
|
pool_ok=true
|
|
fi
|
|
;;
|
|
esac
|
|
if [[ "$pool_ok" != true ]]; then
|
|
error "Resource pool '${PVE_POOL}' not found on Proxmox"
|
|
error "Create it with these commands on the Proxmox host:"
|
|
error " pveum pool add ${PVE_POOL} --comment \"IncusOS Lab VMs\""
|
|
error " pveum pool modify ${PVE_POOL} --storage ${PVE_STORAGE},${PVE_ISO_STORAGE}"
|
|
if [[ "$PVE_METHOD" == "api" ]] && [[ -n "$PVE_API_TOKEN_ID" ]]; then
|
|
local api_user
|
|
api_user=$(echo "$PVE_API_TOKEN_ID" | cut -d'!' -f1)
|
|
error " pveum aclmod /pool/${PVE_POOL} -user ${api_user} -role IncusOSDeploy"
|
|
fi
|
|
errors=$((errors + 1))
|
|
else
|
|
detail "Resource pool '${PVE_POOL}' exists"
|
|
fi
|
|
fi
|
|
|
|
# 5. genisoimage for SEED_DATA ISO 9660 generation
|
|
if ! command -v genisoimage &>/dev/null; then
|
|
error "genisoimage is required for SEED_DATA generation but not found"
|
|
error "Install with: sudo apt install genisoimage"
|
|
errors=$((errors + 1))
|
|
fi
|
|
|
|
if [[ $errors -gt 0 ]]; then
|
|
error "${errors} pre-flight check(s) failed"
|
|
exit 1
|
|
fi
|
|
|
|
if [[ $warnings -gt 0 ]]; then
|
|
success "Pre-flight checks passed with ${warnings} warning(s)"
|
|
else
|
|
success "Pre-flight checks passed"
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Connectivity check
|
|
# ---------------------------------------------------------------------------
|
|
|
|
check_connectivity() {
|
|
step "Checking Proxmox connectivity (${PVE_METHOD})"
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
if ! ssh -o BatchMode=yes -o ConnectTimeout=10 \
|
|
"${PVE_SSH_USER}@${PVE_HOST}" "pvesh get /version --output-format json" &>/dev/null; then
|
|
error "Cannot connect to Proxmox host via SSH"
|
|
error "Ensure key-based SSH auth is set up:"
|
|
error " ssh-copy-id ${PVE_SSH_USER}@${PVE_HOST}"
|
|
error "Test with: ssh ${PVE_SSH_USER}@${PVE_HOST} pvesh get /version"
|
|
exit 1
|
|
fi
|
|
local pve_version
|
|
pve_version=$(ssh -o BatchMode=yes "${PVE_SSH_USER}@${PVE_HOST}" \
|
|
"pvesh get /version --output-format json" 2>/dev/null | \
|
|
python3 -c "import json,sys; print(json.load(sys.stdin).get('version','unknown'))" 2>/dev/null) || pve_version="unknown"
|
|
success "Connected to ${PVE_HOST} (Proxmox VE ${pve_version})"
|
|
;;
|
|
api)
|
|
local url="https://${PVE_HOST}:8006/api2/json/version"
|
|
local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET:-}"
|
|
local response
|
|
if ! response=$(curl -fsSk -H "$auth" "$url" 2>/dev/null); then
|
|
error "Cannot connect to Proxmox API at https://${PVE_HOST}:8006"
|
|
error "Check that:"
|
|
error " 1. The host is reachable"
|
|
error " 2. The API token ID is correct: ${PVE_API_TOKEN_ID}"
|
|
error " 3. PROXMOX_TOKEN_SECRET is set"
|
|
exit 1
|
|
fi
|
|
local pve_version
|
|
pve_version=$(echo "$response" | \
|
|
python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('version','unknown'))" 2>/dev/null) || pve_version="unknown"
|
|
success "Connected to ${PVE_HOST} (Proxmox VE ${pve_version})"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# VMID allocation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
allocate_vmids() {
|
|
step "Allocating VMIDs"
|
|
|
|
# Get existing VMs as "vmid name" pairs
|
|
local vm_list
|
|
vm_list=$(pve_list_vms)
|
|
|
|
# Build lookup: existing IDs and name-to-VMID map
|
|
local existing_ids=""
|
|
declare -A existing_name_to_vmid
|
|
if [[ -n "$vm_list" ]]; then
|
|
while IFS=' ' read -r eid ename; do
|
|
[[ -z "$eid" ]] && continue
|
|
existing_ids="${existing_ids} ${eid}"
|
|
if [[ -n "$ename" ]]; then
|
|
existing_name_to_vmid["$ename"]="$eid"
|
|
fi
|
|
done <<< "$vm_list"
|
|
fi
|
|
|
|
local next_id="$DEF_START_VMID"
|
|
local count=${#VM_NAMES[@]}
|
|
local errors=0
|
|
local i=0
|
|
|
|
while [[ $i -lt $count ]]; do
|
|
local name="${VM_NAMES[$i]}"
|
|
|
|
# Skip if filtered
|
|
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
# Check for name collision with existing Proxmox VMs
|
|
if [[ -n "${existing_name_to_vmid[$name]+x}" ]]; then
|
|
local collision_vmid="${existing_name_to_vmid[$name]}"
|
|
if pve_vm_is_managed "$collision_vmid"; then
|
|
warn "VM '${name}' already exists at VMID ${collision_vmid} (managed by ${SCRIPT_NAME})"
|
|
warn "Use --cleanup to redeploy, or remove it manually"
|
|
# Assign the existing VMID so cleanup/skip works correctly
|
|
VM_VMIDS[$i]="$collision_vmid"
|
|
i=$((i + 1))
|
|
continue
|
|
else
|
|
error "VM name '${name}' already exists at VMID ${collision_vmid} but is NOT managed by ${SCRIPT_NAME}"
|
|
error "This VM was not created by this tool and will not be modified"
|
|
error "Fix: rename the VM in your config file, or manually remove VMID ${collision_vmid}:"
|
|
error " ssh ${PVE_SSH_USER}@${PVE_HOST} qm destroy ${collision_vmid} --purge"
|
|
errors=$((errors + 1))
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
fi
|
|
|
|
if [[ -n "${VM_VMIDS[$i]}" ]]; then
|
|
# Explicit VMID
|
|
detail "VM '${name}': VMID ${VM_VMIDS[$i]} (explicit)"
|
|
else
|
|
# Auto-assign, skipping existing
|
|
while echo "$existing_ids" | grep -qw "$next_id" 2>/dev/null; do
|
|
next_id=$((next_id + 1))
|
|
done
|
|
VM_VMIDS[$i]="$next_id"
|
|
detail "VM '${name}': VMID ${next_id} (auto-assigned)"
|
|
next_id=$((next_id + 1))
|
|
fi
|
|
|
|
i=$((i + 1))
|
|
done
|
|
|
|
if [[ $errors -gt 0 ]]; then
|
|
error "${errors} name collision(s) with unmanaged VMs -- cannot continue"
|
|
exit 1
|
|
fi
|
|
|
|
success "VMIDs allocated"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Certificate detection (reused pattern from sibling scripts)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
CERT_FILE_RESOLVED=""
|
|
|
|
detect_client_cert() {
|
|
if [[ "$NO_CERT" == true ]]; then
|
|
info "Skipping certificate injection (--no-cert)"
|
|
return
|
|
fi
|
|
|
|
if [[ -n "$CERT_FILE" ]]; then
|
|
if [[ ! -f "$CERT_FILE" ]]; then
|
|
error "Certificate file not found: ${CERT_FILE}"
|
|
exit 1
|
|
fi
|
|
if ! grep -q "BEGIN CERTIFICATE" "$CERT_FILE"; then
|
|
error "File does not appear to be a PEM certificate: ${CERT_FILE}"
|
|
exit 1
|
|
fi
|
|
CERT_FILE_RESOLVED="$CERT_FILE"
|
|
success "Certificate: ${CERT_FILE_RESOLVED}"
|
|
return
|
|
fi
|
|
|
|
step "Auto-detecting client certificate"
|
|
|
|
local cert_paths=(
|
|
"${HOME}/.config/incus/client.crt"
|
|
"${HOME}/snap/incus/common/config/client.crt"
|
|
"${HOME}/.config/lxc/client.crt"
|
|
)
|
|
|
|
for path in "${cert_paths[@]}"; do
|
|
if [[ -f "$path" ]] && grep -q "BEGIN CERTIFICATE" "$path"; then
|
|
CERT_FILE_RESOLVED="$path"
|
|
success "Certificate: ${CERT_FILE_RESOLVED}"
|
|
return
|
|
fi
|
|
done
|
|
|
|
# Try CLI fallback
|
|
if command -v incus &>/dev/null; then
|
|
local tmpfile="${WORKDIR}/client.crt"
|
|
if incus remote get-client-certificate > "$tmpfile" 2>/dev/null; then
|
|
CERT_FILE_RESOLVED="$tmpfile"
|
|
success "Certificate obtained via Incus CLI"
|
|
return
|
|
fi
|
|
fi
|
|
|
|
# Check if any VM is ops-center (which requires a cert)
|
|
local needs_cert=false
|
|
for app in "${VM_APPS[@]}"; do
|
|
if [[ "$app" == "operations-center" ]]; then
|
|
needs_cert=true
|
|
break
|
|
fi
|
|
done
|
|
|
|
if [[ "$needs_cert" == true ]]; then
|
|
error "No client certificate found"
|
|
error "Operations Center requires at least one trusted certificate for access"
|
|
error ""
|
|
error "Options:"
|
|
error " --cert FILE Provide a certificate file directly"
|
|
error " Generate one: incus remote list (creates ~/.config/incus/client.crt)"
|
|
error " --no-cert Skip certificate injection (you will be locked out!)"
|
|
exit 1
|
|
fi
|
|
|
|
warn "No client certificate found -- VMs will require manual trust setup"
|
|
warn "Use --cert FILE to provide a certificate, or --no-cert to suppress this warning"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CDN operations (reused from incusos-iso)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
LATEST_VERSION=""
|
|
|
|
fetch_latest_version() {
|
|
local channel="${DEF_CHANNEL:-stable}"
|
|
|
|
step "Fetching latest IncusOS version (channel: ${channel})"
|
|
|
|
local index_file="${WORKDIR}/cdn-index.json"
|
|
|
|
if ! curl -fsSL "$CDN_INDEX" -o "$index_file" 2>/dev/null; then
|
|
error "Failed to fetch CDN index from ${CDN_INDEX}"
|
|
error "Check your internet connection and try again"
|
|
exit 1
|
|
fi
|
|
|
|
if command -v jq &>/dev/null; then
|
|
LATEST_VERSION=$(jq -r \
|
|
--arg ch "$channel" \
|
|
'[.updates[] | select(.channels[] == $ch)] | sort_by(.version) | last | .version // empty' \
|
|
"$index_file")
|
|
else
|
|
LATEST_VERSION=$(python3 -c "
|
|
import json
|
|
data = json.load(open('${index_file}'))
|
|
versions = [u['version'] for u in data.get('updates', []) if '${channel}' in u.get('channels', [])]
|
|
if versions:
|
|
print(sorted(versions)[-1])
|
|
" 2>/dev/null)
|
|
fi
|
|
|
|
if [[ -z "$LATEST_VERSION" ]]; then
|
|
error "No version found for channel '${channel}'"
|
|
exit 1
|
|
fi
|
|
|
|
success "Latest IncusOS version: ${LATEST_VERSION}"
|
|
}
|
|
|
|
download_iso() {
|
|
local version="$1"
|
|
local output_path="$2"
|
|
local iso_filename
|
|
iso_filename=$(basename "$output_path")
|
|
|
|
# Check persistent cache first
|
|
local cached_iso="${CACHEDIR}/${iso_filename}"
|
|
if [[ -f "$cached_iso" ]]; then
|
|
success "ISO found in local cache: ${cached_iso}"
|
|
cp "$cached_iso" "$output_path"
|
|
return 0
|
|
fi
|
|
|
|
local url="${CDN_BASE}/${version}/x86_64/IncusOS_${version}.iso.gz"
|
|
local gz_path="${cached_iso}.gz"
|
|
|
|
step "Downloading IncusOS ISO"
|
|
detail "URL: ${url}"
|
|
detail "Cache directory: ${CACHEDIR}"
|
|
|
|
if ! curl -fL --progress-bar "$url" -o "$gz_path"; then
|
|
error "Failed to download ISO from ${url}"
|
|
rm -f "$gz_path"
|
|
exit 1
|
|
fi
|
|
|
|
step "Decompressing ISO"
|
|
if ! gzip -d "$gz_path"; then
|
|
local avail
|
|
avail=$(df -h --output=avail "$(dirname "$gz_path")" 2>/dev/null | tail -1 | tr -d ' ') || avail="unknown"
|
|
error "Failed to decompress ISO (${avail} available on disk)"
|
|
error "The decompressed ISO requires several GiB of disk space"
|
|
error "Free up space, or set TMPDIR to a larger filesystem:"
|
|
error " TMPDIR=/path/with/space ${SCRIPT_NAME} ${CONFIG_FILE}"
|
|
rm -f "$gz_path"
|
|
exit 1
|
|
fi
|
|
|
|
# Copy from cache to workdir
|
|
cp "$cached_iso" "$output_path"
|
|
success "ISO ready: ${output_path} (cached for future runs)"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Deployment plan summary
|
|
# ---------------------------------------------------------------------------
|
|
|
|
print_plan() {
|
|
local count=${#VM_NAMES[@]}
|
|
|
|
echo ""
|
|
echo -e "${BOLD}Deployment Plan${RESET}"
|
|
echo -e " Proxmox host: ${CYAN}${PVE_HOST}${RESET} (${PVE_METHOD})"
|
|
echo -e " Node: ${PVE_NODE}"
|
|
echo -e " Storage: ${PVE_STORAGE} (disk), ${PVE_ISO_STORAGE} (iso)"
|
|
echo -e " Bridge: ${PVE_BRIDGE}"
|
|
if [[ -n "$PVE_POOL" ]]; then
|
|
echo -e " Pool: ${CYAN}${PVE_POOL}${RESET}"
|
|
fi
|
|
if [[ -n "$CERT_FILE_RESOLVED" ]]; then
|
|
echo -e " Certificate: ${GREEN}${CERT_FILE_RESOLVED}${RESET}"
|
|
elif [[ "$NO_CERT" == true ]]; then
|
|
echo -e " Certificate: ${DIM}none (--no-cert)${RESET}"
|
|
else
|
|
echo -e " Certificate: ${YELLOW}not found${RESET}"
|
|
fi
|
|
echo ""
|
|
echo -e " ${BOLD}VMs (${count}):${RESET}"
|
|
|
|
local i=0
|
|
while [[ $i -lt $count ]]; do
|
|
local name="${VM_NAMES[$i]}"
|
|
local skip=""
|
|
|
|
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
|
|
skip=" ${DIM}(skipped)${RESET}"
|
|
fi
|
|
|
|
echo -e " ${CYAN}${name}${RESET}${skip}"
|
|
echo -e " VMID: ${VM_VMIDS[$i]} App: ${VM_APPS[$i]} Defaults: ${VM_APPLY_DEFAULTS[$i]}"
|
|
echo -e " CPU: ${VM_CORES[$i]} cores RAM: ${VM_MEMORY[$i]} MiB Disk: ${VM_DISK[$i]} GiB"
|
|
|
|
if [[ -n "${VM_CLUSTERS[$i]}" ]]; then
|
|
echo -e " Cluster: ${VM_CLUSTERS[$i]} (${VM_CLUSTER_ROLES[$i]})"
|
|
fi
|
|
|
|
i=$((i + 1))
|
|
done
|
|
echo ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Confirmation prompt
|
|
# ---------------------------------------------------------------------------
|
|
|
|
confirm() {
|
|
local message="$1"
|
|
|
|
if [[ "$YES" == true ]] || [[ "$DRY_RUN" == true ]]; then
|
|
return 0
|
|
fi
|
|
|
|
echo -n -e "${BOLD}${message} [y/N] ${RESET}"
|
|
local reply
|
|
read -r reply
|
|
if [[ ! "$reply" =~ ^[Yy]$ ]]; then
|
|
info "Aborted"
|
|
exit 0
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 1: Validate
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_validate() {
|
|
step "Phase 1: Validate"
|
|
echo ""
|
|
|
|
# Load proxmox.yaml base values (before lab config overlay)
|
|
if find_proxmox_config; then
|
|
load_proxmox_config "$PROXMOX_CONFIG_RESOLVED"
|
|
fi
|
|
|
|
parse_config
|
|
load_config
|
|
validate_config
|
|
check_connectivity
|
|
allocate_vmids
|
|
preflight_checks
|
|
detect_client_cert
|
|
print_plan
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 2: Prepare (download ISO + generate seeds)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
ISO_FILENAME=""
|
|
|
|
phase_prepare() {
|
|
step "Phase 2: Prepare"
|
|
echo ""
|
|
|
|
# Determine ISO
|
|
if [[ -n "$LOCAL_ISO" ]]; then
|
|
ISO_FILENAME=$(basename "$LOCAL_ISO")
|
|
info "Using local ISO: ${LOCAL_ISO}"
|
|
if [[ "$DRY_RUN" != true ]]; then
|
|
cp "$LOCAL_ISO" "${WORKDIR}/${ISO_FILENAME}"
|
|
fi
|
|
elif [[ "$DRY_RUN" == true ]]; then
|
|
ISO_FILENAME="IncusOS_latest.iso"
|
|
info "[dry-run] Would fetch latest version from CDN and download ISO"
|
|
else
|
|
# Check if ISO already exists on Proxmox
|
|
fetch_latest_version
|
|
ISO_FILENAME="IncusOS_${LATEST_VERSION}.iso"
|
|
|
|
if pve_iso_exists "$ISO_FILENAME"; then
|
|
success "ISO already on Proxmox: ${ISO_FILENAME} (reusing from previous deployment)"
|
|
else
|
|
download_iso "$LATEST_VERSION" "${WORKDIR}/${ISO_FILENAME}"
|
|
fi
|
|
fi
|
|
|
|
# Generate per-VM seed images
|
|
step "Generating seed images"
|
|
|
|
local count=${#VM_NAMES[@]}
|
|
local i=0
|
|
while [[ $i -lt $count ]]; do
|
|
local name="${VM_NAMES[$i]}"
|
|
|
|
# Skip if filtered
|
|
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
local app="${VM_APPS[$i]}"
|
|
local apply_defaults="${VM_APPLY_DEFAULTS[$i]}"
|
|
local seed_output="${WORKDIR}/seed-${name}.iso"
|
|
|
|
detail "Generating seed for '${name}' (${app}, defaults=${apply_defaults})"
|
|
|
|
# Build incusos-seed command
|
|
local seed_cmd=("$SEED_TOOL")
|
|
seed_cmd+=(--format iso)
|
|
seed_cmd+=(--app "$app")
|
|
seed_cmd+=(--output "$seed_output")
|
|
seed_cmd+=(--force-install)
|
|
seed_cmd+=(--force-reboot)
|
|
seed_cmd+=(--hostname "$name")
|
|
seed_cmd+=(--quiet)
|
|
|
|
if [[ "$apply_defaults" == "true" ]]; then
|
|
seed_cmd+=(--defaults)
|
|
fi
|
|
|
|
if [[ -n "$CERT_FILE_RESOLVED" ]]; then
|
|
seed_cmd+=(--cert "$CERT_FILE_RESOLVED")
|
|
elif [[ "$NO_CERT" == true ]]; then
|
|
seed_cmd+=(--no-cert)
|
|
fi
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] ${seed_cmd[*]}"
|
|
else
|
|
if ! "${seed_cmd[@]}"; then
|
|
error "Failed to generate seed for VM '${name}'"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
i=$((i + 1))
|
|
done
|
|
|
|
success "Seed images generated"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 3: Upload
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_upload() {
|
|
step "Phase 3: Upload"
|
|
echo ""
|
|
|
|
# Upload ISO (if we downloaded it)
|
|
local iso_path="${WORKDIR}/${ISO_FILENAME}"
|
|
if [[ -f "$iso_path" ]]; then
|
|
if pve_iso_exists "$ISO_FILENAME"; then
|
|
success "ISO already on Proxmox: ${ISO_FILENAME}"
|
|
else
|
|
step "Uploading ISO: ${ISO_FILENAME}"
|
|
if ! pve_upload "$iso_path" "$ISO_FILENAME"; then
|
|
error "Failed to upload ISO to Proxmox"
|
|
exit 1
|
|
fi
|
|
success "ISO uploaded: ${ISO_FILENAME}"
|
|
fi
|
|
else
|
|
detail "ISO already on Proxmox, skipping upload"
|
|
fi
|
|
|
|
# Upload seed images
|
|
local count=${#VM_NAMES[@]}
|
|
local i=0
|
|
while [[ $i -lt $count ]]; do
|
|
local name="${VM_NAMES[$i]}"
|
|
|
|
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
local seed_file="seed-${name}.iso"
|
|
local seed_path="${WORKDIR}/${seed_file}"
|
|
|
|
if [[ -f "$seed_path" ]] || [[ "$DRY_RUN" == true ]]; then
|
|
step "Uploading seed: ${seed_file}"
|
|
if ! pve_upload "$seed_path" "$seed_file"; then
|
|
error "Failed to upload seed for VM '${name}'"
|
|
exit 1
|
|
fi
|
|
success "Seed uploaded: ${seed_file}"
|
|
fi
|
|
|
|
i=$((i + 1))
|
|
done
|
|
|
|
success "All files uploaded"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 4: Create VMs
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_create() {
|
|
step "Phase 4: Create VMs"
|
|
echo ""
|
|
|
|
local count=${#VM_NAMES[@]}
|
|
local i=0
|
|
while [[ $i -lt $count ]]; do
|
|
local name="${VM_NAMES[$i]}"
|
|
|
|
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
local vmid="${VM_VMIDS[$i]}"
|
|
local cores="${VM_CORES[$i]}"
|
|
local memory="${VM_MEMORY[$i]}"
|
|
local disk="${VM_DISK[$i]}"
|
|
local seed_file="seed-${name}.iso"
|
|
|
|
step "Creating VM '${name}' (VMID ${vmid})"
|
|
|
|
# Check if VM already exists
|
|
if pve_vm_exists "$vmid"; then
|
|
if pve_vm_is_managed "$vmid"; then
|
|
warn "VM ${vmid} ('${name}') already exists (managed) -- skipping"
|
|
warn "Use --cleanup to redeploy"
|
|
else
|
|
warn "VM ${vmid} ('${name}') already exists (NOT managed by ${SCRIPT_NAME}) -- skipping"
|
|
warn "Choose a different VMID or remove it manually:"
|
|
warn " ssh ${PVE_SSH_USER}@${PVE_HOST} qm destroy ${vmid} --purge"
|
|
fi
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
detail "cores=${cores} memory=${memory}MiB disk=${disk}GiB"
|
|
detail "bios=ovmf machine=q35 cpu=host scsihw=virtio-scsi-pci"
|
|
detail "efidisk: pre-enrolled-keys=0, tpm: v2.0, balloon: 0"
|
|
detail "iso=${ISO_FILENAME} seed=${seed_file}"
|
|
|
|
if ! pve_create_vm "$vmid" "$name" "$cores" "$memory" "$disk" \
|
|
"$ISO_FILENAME" "$seed_file"; then
|
|
error "Failed to create VM '${name}' (VMID ${vmid})"
|
|
exit 1
|
|
fi
|
|
|
|
success "VM '${name}' created (VMID ${vmid})"
|
|
|
|
i=$((i + 1))
|
|
done
|
|
|
|
success "All VMs created"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Remote registration
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Register an incus remote for a deployed VM.
|
|
# Handles stale remotes: if a remote exists but points to a different IP,
|
|
# it is removed and re-added with the correct IP (DHCP may reassign addresses
|
|
# between deploys).
|
|
# Usage: register_remote <name> <ip>
|
|
register_remote() {
|
|
local name="$1"
|
|
local ip="$2"
|
|
|
|
# Only register if incus client is available and a cert was injected
|
|
if ! command -v incus &>/dev/null; then
|
|
return
|
|
fi
|
|
if [[ -z "$CERT_FILE_RESOLVED" ]]; then
|
|
return
|
|
fi
|
|
|
|
# Check if remote already exists
|
|
local existing_url=""
|
|
existing_url=$(incus remote list --format csv 2>/dev/null | \
|
|
awk -F',' -v name="$name" '$1 == name {print $2; exit}') || true
|
|
|
|
if [[ -n "$existing_url" ]]; then
|
|
# Remote exists -- check if the IP matches
|
|
if echo "$existing_url" | grep -q "${ip}"; then
|
|
detail "Remote '${name}' already configured at ${ip}"
|
|
return
|
|
fi
|
|
|
|
# Stale remote -- IP changed (DHCP reassignment between deploys)
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] Would update stale remote '${name}': ${existing_url} -> ${ip}"
|
|
return
|
|
fi
|
|
|
|
warn "Remote '${name}' points to ${existing_url} but VM is at ${ip} -- updating"
|
|
# Switch away first -- can't remove the current/default remote
|
|
local current_remote=""
|
|
current_remote=$(incus remote get-default 2>/dev/null) || true
|
|
if [[ "$current_remote" == "$name" ]]; then
|
|
incus remote switch local 2>/dev/null || true
|
|
fi
|
|
incus remote remove "$name" 2>/dev/null || true
|
|
fi
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] Would add incus remote '${name}' at ${ip}"
|
|
return
|
|
fi
|
|
|
|
# Wait for port 8443 to become reachable (up to 30s)
|
|
local attempt=0
|
|
while [[ $attempt -lt 6 ]]; do
|
|
if curl -sk --connect-timeout 3 "https://${ip}:8443" &>/dev/null; then
|
|
break
|
|
fi
|
|
sleep 5
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
if incus remote add "$name" "$ip" --accept-certificate --quiet 2>/dev/null; then
|
|
success "Remote '${name}' added (${ip})"
|
|
else
|
|
warn "Could not add remote '${name}' at ${ip} -- add manually:"
|
|
warn " incus remote add ${name} ${ip} --accept-certificate"
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 5: Install
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_install() {
|
|
step "Phase 5: Install"
|
|
echo ""
|
|
|
|
local count=${#VM_NAMES[@]}
|
|
local i=0
|
|
while [[ $i -lt $count ]]; do
|
|
local name="${VM_NAMES[$i]}"
|
|
|
|
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
local vmid="${VM_VMIDS[$i]}"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
step "Starting VM '${name}' (VMID ${vmid})"
|
|
info "[dry-run] Would start VM, swap boot order, wait for install, clean up"
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
# Idempotency: check current VM state before acting
|
|
if pve_vm_exists "$vmid"; then
|
|
local cur_status
|
|
cur_status=$(pve_vm_status "$vmid") || cur_status="unknown"
|
|
local has_media=true
|
|
pve_vm_has_install_media "$vmid" || has_media=false
|
|
|
|
if [[ "$cur_status" == "running" ]] && [[ "$has_media" == false ]]; then
|
|
# Already installed and running — detect IP, register remote, skip
|
|
success "VM '${name}' (VMID ${vmid}) is already installed and running"
|
|
local ip
|
|
ip=$(pve_vm_ip "$vmid") || ip=""
|
|
if [[ -n "$ip" ]]; then
|
|
detail "IP: ${ip}"
|
|
register_remote "$name" "$ip"
|
|
fi
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
if [[ "$cur_status" == "stopped" ]] && [[ "$has_media" == false ]]; then
|
|
# Installed but stopped — start from disk
|
|
step "VM '${name}' is installed but stopped — starting from disk"
|
|
if pve_start_vm "$vmid"; then
|
|
sleep 30
|
|
local ip
|
|
ip=$(pve_vm_ip "$vmid") || ip=""
|
|
if [[ -n "$ip" ]]; then
|
|
success "VM '${name}' started at ${ip}"
|
|
register_remote "$name" "$ip"
|
|
else
|
|
success "VM '${name}' started from disk"
|
|
fi
|
|
else
|
|
error "Failed to start VM '${name}' from disk"
|
|
fi
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
if [[ "$cur_status" == "running" ]] && [[ "$has_media" == true ]]; then
|
|
# Mid-install or stuck — fall through to blockstat monitoring
|
|
info "VM '${name}' is running with install media — monitoring installation"
|
|
fi
|
|
|
|
# stopped + has_media → needs install, fall through
|
|
fi
|
|
|
|
step "Starting VM '${name}' (VMID ${vmid})"
|
|
|
|
# 1. Start VM (boots from ISO, reads SEED_DATA, installs)
|
|
if ! pve_start_vm "$vmid"; then
|
|
error "Failed to start VM '${name}' (VMID ${vmid})"
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
success "VM '${name}' started"
|
|
|
|
# 2. Wait for installation to complete by monitoring disk writes
|
|
#
|
|
# Install flow: boot ISO -> read SEED_DATA -> install to disk -> reboot
|
|
# (force_reboot in seed). A guest-level reboot does NOT reset the QEMU
|
|
# VM uptime, so we cannot detect reboot that way. Instead we monitor
|
|
# scsi0 write bytes via blockstat:
|
|
# - During install: wr_bytes increases steadily
|
|
# - After install + reboot: wr_bytes stops (VM is idle at post-reboot
|
|
# screen or "install media detected" error)
|
|
#
|
|
# When wr_bytes has been stable for STABLE_COUNT consecutive polls,
|
|
# installation is complete. We then stop the VM, remove install media,
|
|
# and start cleanly from disk.
|
|
step "Waiting for '${name}' to complete installation"
|
|
info "Monitoring disk write activity (polling every 5s, timeout: 10 minutes)"
|
|
|
|
local timeout=600
|
|
local interval=5
|
|
local elapsed=0
|
|
local install_done=false
|
|
local prev_wr_bytes=0
|
|
local writes_started=false
|
|
local stable_count=0
|
|
local STABLE_THRESHOLD=3 # 3 consecutive polls with no change = 15s idle
|
|
|
|
while [[ $elapsed -lt $timeout ]]; do
|
|
sleep "$interval"
|
|
elapsed=$((elapsed + interval))
|
|
|
|
local status
|
|
status=$(pve_vm_status "$vmid") || status="unknown"
|
|
|
|
case "$status" in
|
|
stopped)
|
|
detail "VM stopped [${elapsed}s]"
|
|
install_done=true
|
|
break
|
|
;;
|
|
running)
|
|
local wr_bytes
|
|
wr_bytes=$(pve_vm_disk_writes "$vmid") || wr_bytes=0
|
|
|
|
if [[ $wr_bytes -gt $prev_wr_bytes ]]; then
|
|
# Disk writes are happening -- install in progress
|
|
writes_started=true
|
|
stable_count=0
|
|
local wr_mb=$(( wr_bytes / 1048576 ))
|
|
# Show progress every 15s
|
|
if [[ $((elapsed % 15)) -eq 0 ]]; then
|
|
info "Installing... (${wr_mb} MiB written) [${elapsed}s]"
|
|
fi
|
|
elif [[ "$writes_started" == true ]]; then
|
|
# Writes have stopped after being active
|
|
stable_count=$((stable_count + 1))
|
|
if [[ $stable_count -ge $STABLE_THRESHOLD ]]; then
|
|
local wr_mb=$(( wr_bytes / 1048576 ))
|
|
info "Disk activity stopped (${wr_mb} MiB written total) [${elapsed}s]"
|
|
install_done=true
|
|
break
|
|
fi
|
|
fi
|
|
|
|
prev_wr_bytes=$wr_bytes
|
|
;;
|
|
*)
|
|
detail "VM status: ${status} [${elapsed}s]"
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [[ "$install_done" != true ]]; then
|
|
warn "VM '${name}' did not complete installation within ${timeout}s"
|
|
warn "Check the VM console on Proxmox for details"
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
# 3. Stop the VM (it's either at "remove install media" prompt or
|
|
# post-reboot "install media detected" error -- either way idle).
|
|
local cur_status
|
|
cur_status=$(pve_vm_status "$vmid") || cur_status="unknown"
|
|
if [[ "$cur_status" == "running" ]]; then
|
|
info "Stopping VM to remove install media..."
|
|
pve_stop_vm "$vmid" || true
|
|
sleep 5
|
|
fi
|
|
|
|
# 4. Remove install media and set clean boot order
|
|
step "Removing install media from '${name}'"
|
|
pve_set_vm "$vmid" "--delete ide2 --delete ide3 --boot order=scsi0" || true
|
|
success "Install media detached, boot order set to disk-only"
|
|
|
|
# 5. Start VM from disk
|
|
step "Starting '${name}' from disk"
|
|
if ! pve_start_vm "$vmid"; then
|
|
error "Failed to start VM '${name}' from disk"
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
# 5. Wait for services to come up
|
|
info "Waiting for services to start..."
|
|
sleep 30
|
|
|
|
# 6. Try to detect IP
|
|
local ip
|
|
ip=$(pve_vm_ip "$vmid") || ip=""
|
|
|
|
echo ""
|
|
if [[ -n "$ip" ]]; then
|
|
success "VM '${name}' installed and running at ${ip}"
|
|
register_remote "$name" "$ip"
|
|
else
|
|
success "VM '${name}' installed and running"
|
|
info "Check Proxmox console for IP address"
|
|
fi
|
|
echo ""
|
|
|
|
i=$((i + 1))
|
|
done
|
|
|
|
success "Installation phase complete"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Status check
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_status() {
|
|
step "Deployment Status"
|
|
echo ""
|
|
|
|
local count=${#VM_NAMES[@]}
|
|
local i=0
|
|
while [[ $i -lt $count ]]; do
|
|
local name="${VM_NAMES[$i]}"
|
|
|
|
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
local vmid="${VM_VMIDS[$i]}"
|
|
local app="${VM_APPS[$i]}"
|
|
|
|
echo -e " ${BOLD}${name}${RESET} (VMID ${vmid}, app: ${app})"
|
|
|
|
# Proxmox state
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
echo -e " Proxmox: ${DIM}[dry-run]${RESET}"
|
|
i=$((i + 1))
|
|
echo ""
|
|
continue
|
|
fi
|
|
|
|
if ! pve_vm_exists "$vmid"; then
|
|
echo -e " Proxmox: ${RED}not found${RESET}"
|
|
i=$((i + 1))
|
|
echo ""
|
|
continue
|
|
fi
|
|
|
|
local managed=false
|
|
pve_vm_is_managed "$vmid" && managed=true
|
|
|
|
local status
|
|
status=$(pve_vm_status "$vmid") || status="unknown"
|
|
|
|
local has_media=false
|
|
pve_vm_has_install_media "$vmid" && has_media=true
|
|
|
|
local install_state="installed"
|
|
if [[ "$has_media" == true ]]; then
|
|
install_state="not installed (media attached)"
|
|
fi
|
|
|
|
local status_color="$GREEN"
|
|
if [[ "$status" != "running" ]]; then
|
|
status_color="$YELLOW"
|
|
fi
|
|
|
|
echo -e " Proxmox: exists managed: ${managed} status: ${status_color}${status}${RESET} ${install_state}"
|
|
|
|
# Network and service checks (only for running, installed VMs)
|
|
if [[ "$status" == "running" ]] && [[ "$has_media" == false ]]; then
|
|
local ip
|
|
ip=$(pve_vm_ip "$vmid") || ip=""
|
|
|
|
if [[ -n "$ip" ]]; then
|
|
local port_status="${RED}closed${RESET}"
|
|
if curl -sk --connect-timeout 5 "https://${ip}:8443" &>/dev/null; then
|
|
port_status="${GREEN}open${RESET}"
|
|
fi
|
|
echo -e " Network: ${ip} port 8443: ${port_status}"
|
|
|
|
# Incus remote check + auto-register
|
|
if command -v incus &>/dev/null; then
|
|
local remote_name=""
|
|
remote_name=$(incus remote list --format csv 2>/dev/null | \
|
|
awk -F',' -v addr="$ip" '$2 ~ addr {print $1; exit}') || true
|
|
if [[ -n "$remote_name" ]]; then
|
|
echo -e " Remote: ${GREEN}incus remote '${remote_name}' configured${RESET}"
|
|
else
|
|
register_remote "$name" "$ip"
|
|
# Re-check after registration attempt
|
|
remote_name=$(incus remote list --format csv 2>/dev/null | \
|
|
awk -F',' -v addr="$ip" '$2 ~ addr {print $1; exit}') || true
|
|
if [[ -n "$remote_name" ]]; then
|
|
echo -e " Remote: ${GREEN}incus remote '${remote_name}' configured${RESET}"
|
|
else
|
|
echo -e " Remote: ${YELLOW}no incus remote for ${ip}${RESET}"
|
|
fi
|
|
fi
|
|
fi
|
|
else
|
|
echo -e " Network: ${YELLOW}IP not detected${RESET}"
|
|
fi
|
|
fi
|
|
|
|
i=$((i + 1))
|
|
echo ""
|
|
done
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Post-deployment checks
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_post_deploy() {
|
|
step "Post-Deployment Checks"
|
|
echo ""
|
|
|
|
local count=${#VM_NAMES[@]}
|
|
local all_ok=true
|
|
local i=0
|
|
while [[ $i -lt $count ]]; do
|
|
local name="${VM_NAMES[$i]}"
|
|
|
|
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
local vmid="${VM_VMIDS[$i]}"
|
|
local app="${VM_APPS[$i]}"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] Would check '${name}' (VMID ${vmid})"
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
# Only check running, installed VMs
|
|
if ! pve_vm_exists "$vmid"; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
local status
|
|
status=$(pve_vm_status "$vmid") || status="unknown"
|
|
if [[ "$status" != "running" ]]; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
local has_media=false
|
|
pve_vm_has_install_media "$vmid" && has_media=true
|
|
if [[ "$has_media" == true ]]; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
local ip
|
|
ip=$(pve_vm_ip "$vmid") || ip=""
|
|
if [[ -z "$ip" ]]; then
|
|
warn "VM '${name}': could not detect IP address"
|
|
all_ok=false
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
# Port 8443 check
|
|
if curl -sk --connect-timeout 5 "https://${ip}:8443" &>/dev/null; then
|
|
success "VM '${name}' (${ip}): port 8443 reachable"
|
|
else
|
|
warn "VM '${name}' (${ip}): port 8443 not reachable (may still be starting)"
|
|
all_ok=false
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
# App-specific checks
|
|
if [[ "$app" == "incus" ]] && command -v incus &>/dev/null; then
|
|
local remote_name=""
|
|
remote_name=$(incus remote list --format csv 2>/dev/null | \
|
|
awk -F',' -v addr="$ip" '$2 ~ addr {print $1; exit}') || true
|
|
if [[ -n "$remote_name" ]]; then
|
|
if incus info "${remote_name}:" &>/dev/null; then
|
|
success "VM '${name}': incus remote '${remote_name}' connected"
|
|
else
|
|
warn "VM '${name}': incus remote '${remote_name}' exists but connection failed"
|
|
all_ok=false
|
|
fi
|
|
else
|
|
# Auto-register the remote
|
|
register_remote "$name" "$ip"
|
|
# Verify
|
|
remote_name=$(incus remote list --format csv 2>/dev/null | \
|
|
awk -F',' -v addr="$ip" '$2 ~ addr {print $1; exit}') || true
|
|
if [[ -n "$remote_name" ]]; then
|
|
success "VM '${name}': incus remote '${remote_name}' connected"
|
|
else
|
|
info "VM '${name}': no incus remote configured"
|
|
info " Add with: incus remote add ${name} ${ip} --accept-certificate"
|
|
fi
|
|
fi
|
|
elif [[ "$app" == "operations-center" ]]; then
|
|
success "VM '${name}': Operations Center at https://${ip}:8443"
|
|
info " Import PKCS#12 cert in browser for web UI access:"
|
|
info " openssl pkcs12 -export -inkey ~/.config/incus/client.key -in ~/.config/incus/client.crt -out client.pfx"
|
|
fi
|
|
|
|
i=$((i + 1))
|
|
done
|
|
|
|
if [[ "$all_ok" == true ]] && [[ "$DRY_RUN" != true ]]; then
|
|
echo ""
|
|
success "All post-deployment checks passed"
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cleanup mode
|
|
# ---------------------------------------------------------------------------
|
|
|
|
do_cleanup() {
|
|
step "Cleanup: destroying VMs defined in config"
|
|
if [[ "$FORCE_CLEANUP" == true ]]; then
|
|
warn "Force cleanup enabled -- will destroy VMs even without management marker"
|
|
fi
|
|
echo ""
|
|
|
|
local count=${#VM_NAMES[@]}
|
|
local destroyed=0
|
|
local not_found=0
|
|
local skipped_unmanaged=0
|
|
local i=0
|
|
|
|
while [[ $i -lt $count ]]; do
|
|
local name="${VM_NAMES[$i]}"
|
|
|
|
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
local vmid="${VM_VMIDS[$i]}"
|
|
|
|
# 1. Check if VM exists
|
|
if ! pve_vm_exists "$vmid"; then
|
|
detail "VM '${name}' (VMID ${vmid}): does not exist"
|
|
not_found=$((not_found + 1))
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
# 2. Check if VM is managed by us
|
|
if [[ "$DRY_RUN" != true ]]; then
|
|
if ! pve_vm_is_managed "$vmid"; then
|
|
if [[ "$FORCE_CLEANUP" == true ]]; then
|
|
warn "VM '${name}' (VMID ${vmid}): NOT managed by ${SCRIPT_NAME} -- destroying anyway (--force-cleanup)"
|
|
else
|
|
warn "VM '${name}' (VMID ${vmid}): NOT managed by ${SCRIPT_NAME} -- refusing to destroy"
|
|
warn " This VM was not created by this tool. To destroy it manually:"
|
|
warn " ssh ${PVE_SSH_USER}@${PVE_HOST} qm stop ${vmid} && ssh ${PVE_SSH_USER}@${PVE_HOST} qm destroy ${vmid} --purge"
|
|
warn " Or use --force-cleanup to override this safety check"
|
|
skipped_unmanaged=$((skipped_unmanaged + 1))
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# 3. Destroy the VM
|
|
step "Destroying VM '${name}' (VMID ${vmid})"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] Would stop and destroy VM ${vmid}"
|
|
destroyed=$((destroyed + 1))
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
# Stop if running
|
|
local status
|
|
status=$(pve_vm_status "$vmid") || status="unknown"
|
|
if [[ "$status" == "running" ]]; then
|
|
info "Stopping VM..."
|
|
pve_stop_vm "$vmid" || true
|
|
sleep 5
|
|
fi
|
|
|
|
# Destroy
|
|
if pve_destroy_vm "$vmid"; then
|
|
success "VM '${name}' (VMID ${vmid}) destroyed"
|
|
destroyed=$((destroyed + 1))
|
|
else
|
|
error "Failed to destroy VM '${name}' (VMID ${vmid})"
|
|
fi
|
|
|
|
i=$((i + 1))
|
|
done
|
|
|
|
# Summary
|
|
echo ""
|
|
local summary="Cleanup complete: ${destroyed} destroyed"
|
|
if [[ $not_found -gt 0 ]]; then
|
|
summary+=", ${not_found} not found"
|
|
fi
|
|
if [[ $skipped_unmanaged -gt 0 ]]; then
|
|
summary+=", ${skipped_unmanaged} skipped (unmanaged)"
|
|
fi
|
|
success "$summary"
|
|
}
|
|
|
|
# Detect which IncusOS ISO was used by checking ide2 config on existing VMs.
|
|
# Sets DEPLOYMENT_ISO to the filename (e.g. "IncusOS_25.03.iso") or "" if unknown.
|
|
DEPLOYMENT_ISO=""
|
|
|
|
detect_deployment_iso() {
|
|
DEPLOYMENT_ISO=""
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
return
|
|
fi
|
|
|
|
local count=${#VM_NAMES[@]}
|
|
local i=0
|
|
while [[ $i -lt $count ]]; do
|
|
local vmid="${VM_VMIDS[$i]}"
|
|
if pve_vm_exists "$vmid"; then
|
|
local config=""
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
config=$(pve_run "qm config ${vmid} 2>/dev/null" 2>/dev/null) || config=""
|
|
;;
|
|
api)
|
|
config=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null) || config=""
|
|
;;
|
|
esac
|
|
|
|
if [[ -n "$config" ]]; then
|
|
local iso_name
|
|
# Extract IncusOS ISO name from ide2 line or JSON
|
|
iso_name=$(echo "$config" | python3 -c "
|
|
import sys, json, re
|
|
text = sys.stdin.read()
|
|
# Try JSON (API response)
|
|
try:
|
|
data = json.loads(text)
|
|
ide2 = data.get('data', data).get('ide2', '')
|
|
except (json.JSONDecodeError, AttributeError):
|
|
ide2 = ''
|
|
for line in text.splitlines():
|
|
if line.startswith('ide2:'):
|
|
ide2 = line.split(':', 1)[1].strip()
|
|
break
|
|
m = re.search(r'(IncusOS_[^,]+\.iso)', ide2)
|
|
if m:
|
|
print(m.group(1))
|
|
" 2>/dev/null) || iso_name=""
|
|
|
|
if [[ -n "$iso_name" ]]; then
|
|
DEPLOYMENT_ISO="$iso_name"
|
|
detail "Detected deployment ISO: ${DEPLOYMENT_ISO} (from VM ${vmid})"
|
|
return
|
|
fi
|
|
fi
|
|
fi
|
|
i=$((i + 1))
|
|
done
|
|
}
|
|
|
|
# Deep cleanup: remove seeds and the specific IncusOS ISO used by this deployment
|
|
cleanup_storage() {
|
|
step "Deep cleanup: removing seeds and deployment ISO from Proxmox storage"
|
|
|
|
local count=${#VM_NAMES[@]}
|
|
local i=0
|
|
|
|
# Remove per-VM seed ISOs
|
|
while [[ $i -lt $count ]]; do
|
|
local name="${VM_NAMES[$i]}"
|
|
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
local seed_file="seed-${name}.iso"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] Would delete ${PVE_ISO_STORAGE}:iso/${seed_file}"
|
|
else
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
pve_run "pvesm free ${PVE_ISO_STORAGE}:iso/${seed_file} 2>/dev/null" &>/dev/null || true
|
|
;;
|
|
api)
|
|
pve_api DELETE "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content/${PVE_ISO_STORAGE}:iso/${seed_file}" &>/dev/null || true
|
|
;;
|
|
esac
|
|
info "Deleted seed: ${seed_file}"
|
|
fi
|
|
|
|
i=$((i + 1))
|
|
done
|
|
|
|
# Remove only the specific IncusOS ISO used by this deployment
|
|
if [[ -n "$DEPLOYMENT_ISO" ]]; then
|
|
local volid="${PVE_ISO_STORAGE}:iso/${DEPLOYMENT_ISO}"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] Would delete ${volid}"
|
|
else
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
pve_run "pvesm free ${volid} 2>/dev/null" &>/dev/null || true
|
|
;;
|
|
api)
|
|
local encoded_volid
|
|
encoded_volid=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${volid}', safe=''))" 2>/dev/null)
|
|
pve_api DELETE "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content/${encoded_volid}" &>/dev/null || true
|
|
;;
|
|
esac
|
|
info "Deleted ISO: ${DEPLOYMENT_ISO}"
|
|
fi
|
|
else
|
|
detail "No deployment ISO detected (VMs may already be destroyed)"
|
|
fi
|
|
|
|
success "Storage cleanup complete"
|
|
}
|
|
|
|
# Aggressive cleanup: remove ALL IncusOS ISOs and seed ISOs from storage
|
|
cleanup_storage_all() {
|
|
step "Deep cleanup: removing ALL IncusOS ISOs and seeds from Proxmox storage"
|
|
|
|
local iso_list=""
|
|
if [[ "$DRY_RUN" != true ]]; then
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
iso_list=$(pve_run "pvesm list ${PVE_ISO_STORAGE} --content iso 2>/dev/null" 2>/dev/null | grep -E "(IncusOS_|seed-)" | awk '{print $1}') || iso_list=""
|
|
;;
|
|
api)
|
|
iso_list=$(pve_api GET "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content?content=iso" 2>/dev/null | \
|
|
python3 -c "
|
|
import json, sys
|
|
data = json.load(sys.stdin)
|
|
for item in data.get('data', []):
|
|
volid = item.get('volid', '')
|
|
if 'IncusOS_' in volid or 'seed-' in volid:
|
|
print(volid)
|
|
" 2>/dev/null) || iso_list=""
|
|
;;
|
|
esac
|
|
fi
|
|
|
|
if [[ -n "$iso_list" ]]; then
|
|
while IFS= read -r volid; do
|
|
[[ -z "$volid" ]] && continue
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] Would delete ${volid}"
|
|
else
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
pve_run "pvesm free ${volid} 2>/dev/null" &>/dev/null || true
|
|
;;
|
|
api)
|
|
local encoded_volid
|
|
encoded_volid=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${volid}', safe=''))" 2>/dev/null)
|
|
pve_api DELETE "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content/${encoded_volid}" &>/dev/null || true
|
|
;;
|
|
esac
|
|
info "Deleted: ${volid}"
|
|
fi
|
|
done <<< "$iso_list"
|
|
else
|
|
info "No IncusOS ISOs or seeds found in storage"
|
|
fi
|
|
|
|
success "Storage cleanup complete"
|
|
}
|
|
|
|
# Remove incus remotes. Uses VM_NAMES (config-based cleanup) or
|
|
# DESTROYED_VM_NAMES (pool-wide cleanup) depending on what's populated.
|
|
cleanup_remotes() {
|
|
step "Deep cleanup: removing incus remotes"
|
|
|
|
if ! command -v incus &>/dev/null; then
|
|
detail "incus client not installed, skipping remote cleanup"
|
|
return
|
|
fi
|
|
|
|
# Determine which name list to use
|
|
local names_to_check=()
|
|
if [[ ${#DESTROYED_VM_NAMES[@]} -gt 0 ]]; then
|
|
# Pool-wide cleanup: use names of VMs actually destroyed
|
|
names_to_check=("${DESTROYED_VM_NAMES[@]}")
|
|
elif [[ ${#VM_NAMES[@]} -gt 0 ]]; then
|
|
# Config-based cleanup
|
|
local i=0
|
|
while [[ $i -lt ${#VM_NAMES[@]} ]]; do
|
|
if [[ -n "$VM_FILTER" ]] && [[ "${VM_NAMES[$i]}" != "$VM_FILTER" ]]; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
names_to_check+=("${VM_NAMES[$i]}")
|
|
i=$((i + 1))
|
|
done
|
|
fi
|
|
|
|
if [[ ${#names_to_check[@]} -eq 0 ]]; then
|
|
detail "No VM names to check for remote cleanup"
|
|
success "Remote cleanup complete"
|
|
return
|
|
fi
|
|
|
|
for name in "${names_to_check[@]}"; do
|
|
# Check if a remote with this name exists
|
|
if incus remote list --format csv 2>/dev/null | grep -q "^${name},"; then
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] Would remove incus remote '${name}'"
|
|
else
|
|
incus remote remove "$name" 2>/dev/null || true
|
|
info "Removed incus remote: ${name}"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
success "Remote cleanup complete"
|
|
}
|
|
|
|
cleanup_local_cache() {
|
|
step "Deep cleanup: clearing local ISO cache"
|
|
|
|
local cache_base="${XDG_CACHE_HOME:-${HOME}/.cache}/${SCRIPT_NAME}/iso-cache"
|
|
if [[ -d "$cache_base" ]]; then
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] Would clear ${cache_base}"
|
|
else
|
|
rm -rf "$cache_base"
|
|
info "Cleared cache: ${cache_base}"
|
|
fi
|
|
else
|
|
detail "No local cache to clear"
|
|
fi
|
|
|
|
success "Local cache cleanup complete"
|
|
}
|
|
|
|
# Names of VMs destroyed during cleanup-all (used by cleanup_remotes)
|
|
declare -a DESTROYED_VM_NAMES=()
|
|
|
|
# Pool-wide cleanup: destroy ALL managed VMs in the resource pool
|
|
do_cleanup_all() {
|
|
step "Pool-wide cleanup: destroying all managed VMs"
|
|
echo ""
|
|
|
|
if [[ -z "$PVE_POOL" ]]; then
|
|
error "Pool-wide cleanup requires a resource pool (proxmox.pool)"
|
|
error "Set pool in proxmox.yaml or pass a config with proxmox.pool"
|
|
error "Without a pool, there is no safe way to scope which VMs to destroy"
|
|
exit 1
|
|
fi
|
|
|
|
# Get all VMs in the pool
|
|
local vm_list
|
|
vm_list=$(pve_list_vms)
|
|
|
|
if [[ -z "$vm_list" ]]; then
|
|
info "No VMs found in pool '${PVE_POOL}'"
|
|
return
|
|
fi
|
|
|
|
# Filter for managed VMs
|
|
local managed_vms=()
|
|
local managed_names=()
|
|
while IFS=' ' read -r eid ename; do
|
|
[[ -z "$eid" ]] && continue
|
|
if pve_vm_is_managed "$eid"; then
|
|
managed_vms+=("$eid")
|
|
managed_names+=("${ename:-unknown}")
|
|
fi
|
|
done <<< "$vm_list"
|
|
|
|
if [[ ${#managed_vms[@]} -eq 0 ]]; then
|
|
info "No managed VMs found in pool '${PVE_POOL}'"
|
|
return
|
|
fi
|
|
|
|
# Show what will be destroyed
|
|
echo -e " ${BOLD}Managed VMs in pool '${PVE_POOL}':${RESET}"
|
|
local idx=0
|
|
while [[ $idx -lt ${#managed_vms[@]} ]]; do
|
|
echo -e " VMID ${managed_vms[$idx]}: ${managed_names[$idx]}"
|
|
idx=$((idx + 1))
|
|
done
|
|
echo ""
|
|
|
|
# Confirm
|
|
confirm "Destroy all ${#managed_vms[@]} managed VM(s) in pool '${PVE_POOL}'?"
|
|
|
|
# Destroy each
|
|
local destroyed=0
|
|
idx=0
|
|
while [[ $idx -lt ${#managed_vms[@]} ]]; do
|
|
local vmid="${managed_vms[$idx]}"
|
|
local name="${managed_names[$idx]}"
|
|
|
|
step "Destroying VM '${name}' (VMID ${vmid})"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] Would stop and destroy VM ${vmid}"
|
|
destroyed=$((destroyed + 1))
|
|
DESTROYED_VM_NAMES+=("$name")
|
|
idx=$((idx + 1))
|
|
continue
|
|
fi
|
|
|
|
local status
|
|
status=$(pve_vm_status "$vmid") || status="unknown"
|
|
if [[ "$status" == "running" ]]; then
|
|
info "Stopping VM..."
|
|
pve_stop_vm "$vmid" || true
|
|
sleep 5
|
|
fi
|
|
|
|
if pve_destroy_vm "$vmid"; then
|
|
success "VM '${name}' (VMID ${vmid}) destroyed"
|
|
destroyed=$((destroyed + 1))
|
|
DESTROYED_VM_NAMES+=("$name")
|
|
else
|
|
error "Failed to destroy VM '${name}' (VMID ${vmid})"
|
|
fi
|
|
|
|
idx=$((idx + 1))
|
|
done
|
|
|
|
echo ""
|
|
success "Pool cleanup complete: ${destroyed} VM(s) destroyed"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Reconcile existing deployments
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Check if VMs from config already exist and offer interactive options.
|
|
# Only called for PHASE=all, non-cleanup, non-dry-run runs.
|
|
# Returns 0 to continue with fresh deploy, or exits.
|
|
reconcile_existing() {
|
|
local count=${#VM_NAMES[@]}
|
|
local existing=0
|
|
local running=0
|
|
local installed=0
|
|
local i=0
|
|
|
|
while [[ $i -lt $count ]]; do
|
|
local name="${VM_NAMES[$i]}"
|
|
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
local vmid="${VM_VMIDS[$i]}"
|
|
if pve_vm_exists "$vmid"; then
|
|
existing=$((existing + 1))
|
|
|
|
local status
|
|
status=$(pve_vm_status "$vmid") || status="unknown"
|
|
if [[ "$status" == "running" ]]; then
|
|
running=$((running + 1))
|
|
fi
|
|
|
|
local has_media=false
|
|
pve_vm_has_install_media "$vmid" && has_media=true
|
|
if [[ "$has_media" == false ]]; then
|
|
installed=$((installed + 1))
|
|
fi
|
|
fi
|
|
|
|
i=$((i + 1))
|
|
done
|
|
|
|
# No existing VMs — fresh deploy, proceed normally
|
|
if [[ $existing -eq 0 ]]; then
|
|
return 0
|
|
fi
|
|
|
|
# Count of VMs we're deploying (respecting filter)
|
|
local target=0
|
|
for name in "${VM_NAMES[@]}"; do
|
|
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
|
|
continue
|
|
fi
|
|
target=$((target + 1))
|
|
done
|
|
|
|
echo ""
|
|
echo -e "${BOLD}Existing deployment detected${RESET}"
|
|
echo -e " ${existing}/${target} VMs exist (${running} running, ${installed} installed)"
|
|
echo ""
|
|
|
|
if [[ "$existing" -lt "$target" ]]; then
|
|
local incomplete=$((target - existing))
|
|
echo -e " ${YELLOW}Partial deployment:${RESET} ${incomplete} VM(s) still need to be created/installed"
|
|
fi
|
|
|
|
# With --yes, default to safe option (post-deploy checks)
|
|
if [[ "$YES" == true ]]; then
|
|
info "Existing VMs detected with --yes; running post-deployment checks"
|
|
echo ""
|
|
phase_status
|
|
phase_post_deploy
|
|
exit 0
|
|
fi
|
|
|
|
echo -e " Options:"
|
|
echo -e " ${BOLD}1)${RESET} Run status checks and post-deployment verification"
|
|
echo -e " ${BOLD}2)${RESET} Continue deployment (skip existing, install incomplete VMs)"
|
|
echo -e " ${BOLD}3)${RESET} Destroy and redeploy cleanly"
|
|
echo -e " ${BOLD}4)${RESET} Abort"
|
|
echo ""
|
|
echo -n -e " ${BOLD}Choose [1/2/3/4]: ${RESET}"
|
|
local choice
|
|
read -r choice
|
|
|
|
case "$choice" in
|
|
1)
|
|
phase_status
|
|
phase_post_deploy
|
|
exit 0
|
|
;;
|
|
2)
|
|
info "Continuing deployment (existing VMs will be skipped)"
|
|
return 0
|
|
;;
|
|
3)
|
|
warn "Destroying existing VMs before redeployment"
|
|
do_cleanup
|
|
echo ""
|
|
return 0
|
|
;;
|
|
4|*)
|
|
info "Aborted"
|
|
exit 0
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main
|
|
# ---------------------------------------------------------------------------
|
|
|
|
main() {
|
|
setup_colors
|
|
parse_args "$@"
|
|
validate_args
|
|
|
|
# Handle doctor mode early (before config parsing)
|
|
if [[ "$DOCTOR" == true ]]; then
|
|
run_doctor
|
|
return
|
|
fi
|
|
|
|
if [[ "$QUIET" != true ]]; then
|
|
echo -e "${BOLD}${SCRIPT_NAME}${RESET} v${VERSION}"
|
|
echo ""
|
|
fi
|
|
|
|
setup_workdir
|
|
|
|
# Handle cleanup-all without a config file (only needs proxmox.yaml)
|
|
if [[ "$CLEANUP_ALL" == true ]] && [[ -z "$CONFIG_FILE" ]]; then
|
|
if ! find_proxmox_config; then
|
|
error "No proxmox.yaml found and no config file specified"
|
|
error "Create proxmox.yaml or use --proxmox FILE"
|
|
exit 1
|
|
fi
|
|
load_proxmox_config "$PROXMOX_CONFIG_RESOLVED"
|
|
|
|
# Apply defaults for anything still unset
|
|
[[ -z "$PVE_NODE" ]] && PVE_NODE="pve"
|
|
[[ -z "$PVE_STORAGE" ]] && PVE_STORAGE="local-lvm"
|
|
[[ -z "$PVE_ISO_STORAGE" ]] && PVE_ISO_STORAGE="local"
|
|
[[ -z "$PVE_BRIDGE" ]] && PVE_BRIDGE="vmbr0"
|
|
[[ -z "$PVE_SSH_USER" ]] && PVE_SSH_USER="root"
|
|
[[ -z "$PVE_METHOD" ]] && PVE_METHOD="ssh"
|
|
[[ -n "$METHOD" ]] && PVE_METHOD="$METHOD"
|
|
[[ -n "$HOST_OVERRIDE" ]] && PVE_HOST="$HOST_OVERRIDE"
|
|
|
|
if [[ -z "$PVE_HOST" ]]; then
|
|
error "proxmox.yaml must contain 'host:'"
|
|
exit 1
|
|
fi
|
|
|
|
do_cleanup_all
|
|
if [[ "$DEEP_CLEANUP" == true ]]; then
|
|
echo ""
|
|
cleanup_storage_all
|
|
cleanup_remotes
|
|
cleanup_local_cache
|
|
fi
|
|
return
|
|
fi
|
|
|
|
if [[ "$DRY_RUN" != true ]]; then
|
|
check_dependencies
|
|
fi
|
|
echo ""
|
|
|
|
# Always run validate first
|
|
phase_validate
|
|
|
|
# Handle cleanup-all mode (with config file for additional context)
|
|
if [[ "$CLEANUP_ALL" == true ]]; then
|
|
do_cleanup_all
|
|
if [[ "$DEEP_CLEANUP" == true ]]; then
|
|
echo ""
|
|
cleanup_storage_all
|
|
cleanup_remotes
|
|
cleanup_local_cache
|
|
fi
|
|
return
|
|
fi
|
|
|
|
# Handle cleanup mode
|
|
if [[ "$CLEANUP" == true ]]; then
|
|
if [[ "$DEEP_CLEANUP" == true ]]; then
|
|
# Detect which ISO was used before destroying VMs
|
|
detect_deployment_iso
|
|
fi
|
|
confirm "Destroy VMs listed above?"
|
|
do_cleanup
|
|
if [[ "$DEEP_CLEANUP" == true ]]; then
|
|
echo ""
|
|
cleanup_storage
|
|
cleanup_remotes
|
|
cleanup_local_cache
|
|
fi
|
|
return
|
|
fi
|
|
|
|
# Reconcile existing deployments (interactive menu on re-runs)
|
|
if [[ "$PHASE" == "all" ]] && [[ "$DRY_RUN" != true ]]; then
|
|
reconcile_existing
|
|
fi
|
|
|
|
# Confirm before proceeding
|
|
if [[ "$PHASE" == "all" ]]; then
|
|
confirm "Proceed with deployment?"
|
|
fi
|
|
|
|
# Run requested phases
|
|
case "$PHASE" in
|
|
validate)
|
|
# Already done above
|
|
;;
|
|
prepare)
|
|
phase_prepare
|
|
;;
|
|
upload)
|
|
phase_prepare
|
|
phase_upload
|
|
;;
|
|
create)
|
|
phase_prepare
|
|
phase_upload
|
|
phase_create
|
|
;;
|
|
install)
|
|
phase_prepare
|
|
phase_upload
|
|
phase_create
|
|
phase_install
|
|
;;
|
|
status)
|
|
phase_status
|
|
phase_post_deploy
|
|
;;
|
|
all)
|
|
phase_prepare
|
|
echo ""
|
|
phase_upload
|
|
echo ""
|
|
phase_create
|
|
echo ""
|
|
phase_install
|
|
echo ""
|
|
phase_post_deploy
|
|
;;
|
|
esac
|
|
|
|
echo ""
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "Dry run complete -- no changes were made"
|
|
else
|
|
success "Done"
|
|
fi
|
|
}
|
|
|
|
main "$@"
|