4894 lines
165 KiB
Bash
Executable File
4894 lines
165 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# incusos-proxmox - Declarative IncusOS deployment on Proxmox VE
|
|
#
|
|
# Reads a YAML configuration file defining VMs and deploys them on a Proxmox
|
|
# host with IncusOS-correct settings (UEFI, TPM 2.0, VirtIO-SCSI, etc.).
|
|
# Generates per-VM seed images, uploads ISOs, creates VMs, and boots them
|
|
# through automated installation.
|
|
#
|
|
# Usage: incusos-proxmox [OPTIONS] CONFIG_FILE
|
|
# Run 'incusos-proxmox --help' for full usage information.
|
|
|
|
set -euo pipefail
|
|
|
|
readonly VERSION="0.1.0"
|
|
readonly SCRIPT_NAME="incusos-proxmox"
|
|
readonly CDN_INDEX="https://images.linuxcontainers.org/os/index.json"
|
|
readonly CDN_BASE="https://images.linuxcontainers.org/os"
|
|
readonly MIN_DISK_GIB=50
|
|
readonly MIN_MEMORY_MIB=4096
|
|
readonly MANAGED_MARKER="[incusos-lab:managed]"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Auto-load env file (secrets like PROXMOX_TOKEN_SECRET)
|
|
# ---------------------------------------------------------------------------
|
|
# Searches for an 'env' file: first next to the script, then walking up from
|
|
# the current directory. Sources it if found and not already loaded. This is
|
|
# a convenience -- users can still export variables manually or source the
|
|
# file themselves.
|
|
|
|
load_env_file() {
|
|
# Skip if the secret is already set (user exported it, or env already sourced)
|
|
if [[ -n "${PROXMOX_TOKEN_SECRET:-}" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
local script_real
|
|
script_real=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|
|
|
# Check: script directory's parent (repo root when script is in incusos/)
|
|
if [[ -f "${script_real}/../env" ]]; then
|
|
# shellcheck source=/dev/null
|
|
source "${script_real}/../env"
|
|
return 0
|
|
fi
|
|
|
|
# Check: script directory itself
|
|
if [[ -f "${script_real}/env" ]]; then
|
|
# shellcheck source=/dev/null
|
|
source "${script_real}/env"
|
|
return 0
|
|
fi
|
|
|
|
# Walk up from cwd
|
|
local dir="$PWD"
|
|
while [[ "$dir" != "/" ]]; do
|
|
if [[ -f "${dir}/env" ]]; then
|
|
# shellcheck source=/dev/null
|
|
source "${dir}/env"
|
|
return 0
|
|
fi
|
|
dir=$(dirname "$dir")
|
|
done
|
|
|
|
# No env file found -- not an error; require_api_auth() will catch
|
|
# missing secrets later if API method is used.
|
|
return 0
|
|
}
|
|
|
|
load_env_file
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Color and formatting
|
|
# ---------------------------------------------------------------------------
|
|
|
|
setup_colors() {
|
|
if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[0;33m'
|
|
BLUE='\033[0;34m'
|
|
CYAN='\033[0;36m'
|
|
BOLD='\033[1m'
|
|
DIM='\033[2m'
|
|
RESET='\033[0m'
|
|
else
|
|
RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' RESET=''
|
|
fi
|
|
}
|
|
|
|
info() { [[ "$QUIET" == true ]] && return; echo -e "${BLUE}[info]${RESET} $*"; }
|
|
success() { [[ "$QUIET" == true ]] && return; echo -e "${GREEN}[ok]${RESET} $*"; }
|
|
warn() { echo -e "${YELLOW}[warn]${RESET} $*" >&2; }
|
|
error() { echo -e "${RED}[error]${RESET} $*" >&2; }
|
|
step() { [[ "$QUIET" == true ]] && return; echo -e "${CYAN}[step]${RESET} ${BOLD}$*${RESET}"; }
|
|
detail() { [[ "$VERBOSE" != true ]] && return; echo -e "${DIM} $*${RESET}"; }
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Help
|
|
# ---------------------------------------------------------------------------
|
|
|
|
usage() {
|
|
cat <<EOF
|
|
${BOLD}${SCRIPT_NAME}${RESET} v${VERSION} - Declarative IncusOS deployment on Proxmox VE
|
|
|
|
${BOLD}USAGE${RESET}
|
|
${SCRIPT_NAME} [OPTIONS] CONFIG_FILE
|
|
|
|
${BOLD}DESCRIPTION${RESET}
|
|
Reads a YAML configuration file defining VMs and deploys them on a Proxmox
|
|
host with IncusOS-correct settings. Handles seed generation, ISO upload, VM
|
|
creation, and automated installation.
|
|
|
|
IncusOS requires specific Proxmox VM settings:
|
|
- UEFI firmware without pre-enrolled Secure Boot keys
|
|
- TPM 2.0 virtual device
|
|
- VirtIO-SCSI controller (not VirtIO-blk)
|
|
- CPU type 'host' (x86_64_v3 instruction set requirement)
|
|
- Minimum 50 GiB disk, minimum 4 GiB RAM
|
|
|
|
This script ensures all settings are correct, preventing install failures.
|
|
|
|
${BOLD}OPTIONS${RESET}
|
|
${BOLD}-p, --phase${RESET} PHASE Run a specific phase:
|
|
${CYAN}validate${RESET} -- parse config, check connectivity
|
|
${CYAN}prepare${RESET} -- download ISO, generate seeds
|
|
${CYAN}upload${RESET} -- upload ISO and seeds to Proxmox
|
|
${CYAN}create${RESET} -- create VMs with correct settings
|
|
${CYAN}install${RESET} -- start VMs and monitor installation
|
|
${CYAN}status${RESET} -- check deployment status
|
|
${CYAN}all${RESET} -- run all phases (default)
|
|
${BOLD}--status${RESET} Shortcut for --phase status
|
|
${BOLD}-m, --method${RESET} METHOD Connection method: ${CYAN}ssh${RESET} (default), ${CYAN}api${RESET}
|
|
${BOLD}-H, --host${RESET} HOST Override Proxmox host from config
|
|
${BOLD}-i, --iso${RESET} FILE Use local ISO file (skip CDN download)
|
|
${BOLD}-c, --cert${RESET} FILE Client certificate PEM file
|
|
${BOLD}--no-cert${RESET} Skip certificate injection
|
|
${BOLD}--vm${RESET} NAME Operate on a single VM only
|
|
${BOLD}-y, --yes${RESET} Skip confirmation prompts
|
|
${BOLD}--doctor${RESET} Check tool versions and environment health
|
|
${BOLD}--cleanup${RESET} Destroy VMs defined in config (managed only)
|
|
${BOLD}--cleanup --deep${RESET} Also remove ISOs, seeds, remotes, and local cache
|
|
${BOLD}--cleanup-all${RESET} Destroy ALL managed VMs in the pool
|
|
${BOLD}--proxmox${RESET} FILE Proxmox connection config (default: auto-discover)
|
|
${BOLD}--force-cleanup${RESET} Destroy VMs even without management marker
|
|
${BOLD}--lab-up${RESET} Start all stopped VMs from config (boot from disk)
|
|
${BOLD}--lab-down${RESET} Stop all running VMs from config (Proxmox stop)
|
|
${BOLD}--resources${RESET} Show Proxmox host resource usage (RAM, storage)
|
|
${BOLD}--labs${RESET} List all deployed labs (scan pool for managed VMs)
|
|
${BOLD}--dry-run${RESET} Preview actions without executing
|
|
${BOLD}-v, --verbose${RESET} Show detailed output (tool paths, API calls)
|
|
${BOLD}-q, --quiet${RESET} Suppress informational output
|
|
${BOLD}-h, --help${RESET} Show this help message
|
|
${BOLD}-V, --version${RESET} Show version
|
|
|
|
${BOLD}PROXMOX CONNECTION CONFIG${RESET}
|
|
Proxmox connection settings can be stored in a separate proxmox.yaml file
|
|
to avoid duplicating credentials across lab configs. Auto-discovered in:
|
|
script directory, then current directory. Override with --proxmox FILE.
|
|
|
|
${BOLD}proxmox.yaml${RESET} (flat structure, no wrapping key):
|
|
host: 192.168.1.10 # Proxmox host (required)
|
|
method: api # ssh or api (default: ssh)
|
|
api_token_id: user!token # API token (for api method)
|
|
node: pve # Node name (default: pve)
|
|
storage: local-lvm # VM disk storage (default: local-lvm)
|
|
iso_storage: local # ISO storage (default: local)
|
|
bridge: vmbr0 # Network bridge (default: vmbr0)
|
|
pool: IncusLab # Resource pool (optional)
|
|
ssh_user: root # SSH user (default: root)
|
|
|
|
Merge priority: proxmox.yaml → lab config proxmox: section → CLI flags
|
|
|
|
${BOLD}CONFIG FILE${RESET}
|
|
YAML file defining VM specifications (and optional Proxmox overrides).
|
|
See examples/proxmox-lab.yaml for a full example.
|
|
|
|
${BOLD}proxmox:${RESET} # Optional: overrides proxmox.yaml
|
|
host: 192.168.1.10 # Proxmox host
|
|
storage: local-zfs # Override storage for this deployment
|
|
|
|
${BOLD}defaults:${RESET}
|
|
cores: 4 # Default CPU cores
|
|
memory: 8192 # Default RAM in MiB (min: 4096)
|
|
disk: 64 # Default disk in GiB (min: 50)
|
|
channel: stable # Update channel
|
|
start_vmid: 200 # Auto-assign VMIDs from here
|
|
|
|
${BOLD}vms:${RESET}
|
|
- name: my-vm # VM name (required)
|
|
app: incus # incus or operations-center (required)
|
|
apply_defaults: true # Apply IncusOS defaults
|
|
cores: 4 # Override default cores
|
|
memory: 8192 # Override default memory
|
|
disk: 64 # Override default disk
|
|
vmid: 200 # Explicit VMID (optional)
|
|
|
|
${BOLD}EXAMPLES${RESET}
|
|
# Preview a full deployment
|
|
${SCRIPT_NAME} --dry-run examples/proxmox-lab.yaml
|
|
|
|
# Validate config and check Proxmox connectivity
|
|
${SCRIPT_NAME} --phase validate examples/proxmox-lab.yaml
|
|
|
|
# Deploy everything (with confirmation prompts)
|
|
${SCRIPT_NAME} examples/proxmox-lab.yaml
|
|
|
|
# Deploy without prompts
|
|
${SCRIPT_NAME} --yes examples/proxmox-lab.yaml
|
|
|
|
# Deploy a single VM from the config
|
|
${SCRIPT_NAME} --vm ops-center examples/proxmox-lab.yaml
|
|
|
|
# Use a local ISO instead of downloading
|
|
${SCRIPT_NAME} --iso IncusOS_20250201.iso examples/proxmox-lab.yaml
|
|
|
|
# Check deployment status
|
|
${SCRIPT_NAME} --status examples/proxmox-lab.yaml
|
|
|
|
# Destroy all VMs defined in config
|
|
${SCRIPT_NAME} --cleanup examples/proxmox-lab.yaml
|
|
|
|
# Deep cleanup: VMs + the IncusOS ISO used + seeds + remotes + cache
|
|
${SCRIPT_NAME} --cleanup --deep examples/proxmox-lab.yaml
|
|
|
|
# Destroy ALL managed VMs in the pool (uses proxmox.yaml, no config needed)
|
|
${SCRIPT_NAME} --cleanup-all
|
|
${SCRIPT_NAME} --cleanup-all --deep # also remove all ISOs + seeds
|
|
|
|
# Check tool versions and environment health
|
|
${SCRIPT_NAME} --doctor
|
|
${SCRIPT_NAME} --doctor examples/proxmox-lab.yaml # also check Proxmox
|
|
|
|
# Verbose output (show dependency paths, API calls, per-file actions)
|
|
${SCRIPT_NAME} --doctor -v
|
|
|
|
# Use API method with token
|
|
export PROXMOX_TOKEN_SECRET="xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
|
|
${SCRIPT_NAME} --method api examples/proxmox-lab.yaml
|
|
|
|
${BOLD}PROXMOX SETUP${RESET}
|
|
${BOLD}SSH method (default):${RESET}
|
|
Requires key-based SSH authentication to the Proxmox host.
|
|
Test with: ssh root@<host> pvesh get /version
|
|
|
|
${BOLD}API method:${RESET}
|
|
Requires an API token. Set up minimal privileges:
|
|
|
|
pveum role add IncusOSDeployer --privs "VM.Allocate VM.Config.Disk \\
|
|
VM.Config.CPU VM.Config.Memory VM.Config.Network VM.Config.CDROM \\
|
|
VM.Config.Options VM.Config.HWType VM.PowerMgmt VM.Audit \\
|
|
Datastore.AllocateSpace Datastore.AllocateTemplate \\
|
|
Datastore.Audit SDN.Use Sys.Audit"
|
|
pveum user add automation@pve
|
|
pveum aclmod / -user automation@pve -role IncusOSDeployer
|
|
pveum user token add automation@pve deploy --privsep 0
|
|
|
|
For resource pool isolation (recommended):
|
|
pveum pool add IncusLab --comment "IncusOS Lab VMs"
|
|
pveum pool modify IncusLab --storage local-lvm,local
|
|
pveum aclmod /pool/IncusLab -user automation@pve -role IncusOSDeployer
|
|
|
|
Set the token secret:
|
|
export PROXMOX_TOKEN_SECRET="<secret-from-above>"
|
|
|
|
And in your config file:
|
|
proxmox:
|
|
method: api
|
|
api_token_id: automation@pve!deploy
|
|
|
|
${BOLD}PREREQUISITES${RESET}
|
|
- python3 with PyYAML (python3-yaml / py3-yaml)
|
|
- jq (for JSON field access; falls back to python3)
|
|
- ssh + scp (for SSH method)
|
|
- curl (for API method and CDN downloads)
|
|
- incusos-seed sibling script (auto-detected)
|
|
- Bash 4+ (associative arrays)
|
|
|
|
EOF
|
|
exit 0
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Argument parsing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Defaults
|
|
PHASE="all"
|
|
METHOD=""
|
|
HOST_OVERRIDE=""
|
|
LOCAL_ISO=""
|
|
CERT_FILE=""
|
|
NO_CERT=false
|
|
VM_FILTER=""
|
|
YES=false
|
|
DOCTOR=false
|
|
CLEANUP=false
|
|
CLEANUP_ALL=false
|
|
DEEP_CLEANUP=false
|
|
FORCE_CLEANUP=false
|
|
LAB_UP=false
|
|
LAB_DOWN=false
|
|
RESOURCES=false
|
|
LIST_LABS=false
|
|
DRY_RUN=false
|
|
QUIET=false
|
|
VERBOSE=false
|
|
PROXMOX_CONFIG=""
|
|
CONFIG_FILE=""
|
|
|
|
parse_args() {
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
-p|--phase)
|
|
PHASE="${2:?'--phase requires a value'}"
|
|
shift 2
|
|
;;
|
|
-m|--method)
|
|
METHOD="${2:?'--method requires a value'}"
|
|
shift 2
|
|
;;
|
|
-H|--host)
|
|
HOST_OVERRIDE="${2:?'--host requires a value'}"
|
|
shift 2
|
|
;;
|
|
-i|--iso)
|
|
LOCAL_ISO="${2:?'--iso requires a file path'}"
|
|
shift 2
|
|
;;
|
|
-c|--cert)
|
|
CERT_FILE="${2:?'--cert requires a file path'}"
|
|
shift 2
|
|
;;
|
|
--no-cert)
|
|
NO_CERT=true
|
|
shift
|
|
;;
|
|
--vm)
|
|
VM_FILTER="${2:?'--vm requires a name'}"
|
|
shift 2
|
|
;;
|
|
-y|--yes)
|
|
YES=true
|
|
shift
|
|
;;
|
|
--status)
|
|
PHASE="status"
|
|
shift
|
|
;;
|
|
--doctor)
|
|
DOCTOR=true
|
|
shift
|
|
;;
|
|
--cleanup)
|
|
CLEANUP=true
|
|
shift
|
|
;;
|
|
--deep)
|
|
DEEP_CLEANUP=true
|
|
shift
|
|
;;
|
|
--cleanup-all)
|
|
CLEANUP_ALL=true
|
|
shift
|
|
;;
|
|
--force-cleanup)
|
|
CLEANUP=true
|
|
FORCE_CLEANUP=true
|
|
shift
|
|
;;
|
|
--lab-up)
|
|
LAB_UP=true
|
|
shift
|
|
;;
|
|
--lab-down)
|
|
LAB_DOWN=true
|
|
shift
|
|
;;
|
|
--resources)
|
|
RESOURCES=true
|
|
shift
|
|
;;
|
|
--labs)
|
|
LIST_LABS=true
|
|
shift
|
|
;;
|
|
--proxmox)
|
|
PROXMOX_CONFIG="${2:?'--proxmox requires a file path'}"
|
|
shift 2
|
|
;;
|
|
--dry-run)
|
|
DRY_RUN=true
|
|
shift
|
|
;;
|
|
-v|--verbose)
|
|
VERBOSE=true
|
|
shift
|
|
;;
|
|
-q|--quiet)
|
|
QUIET=true
|
|
shift
|
|
;;
|
|
-h|--help)
|
|
usage
|
|
;;
|
|
-V|--version)
|
|
echo "${SCRIPT_NAME} v${VERSION}"
|
|
exit 0
|
|
;;
|
|
-*)
|
|
error "Unknown option: $1"
|
|
echo "Run '${SCRIPT_NAME} --help' for usage information." >&2
|
|
exit 1
|
|
;;
|
|
*)
|
|
if [[ -z "$CONFIG_FILE" ]]; then
|
|
CONFIG_FILE="$1"
|
|
else
|
|
error "Unexpected argument: $1 (config file already set to ${CONFIG_FILE})"
|
|
echo "Run '${SCRIPT_NAME} --help' for usage information." >&2
|
|
exit 1
|
|
fi
|
|
shift
|
|
;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Validation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
validate_args() {
|
|
# Universal checks (apply regardless of mode)
|
|
if [[ "$VERBOSE" == true ]] && [[ "$QUIET" == true ]]; then
|
|
error "Cannot use both --verbose and --quiet"
|
|
exit 1
|
|
fi
|
|
|
|
# --doctor can run without a config file
|
|
if [[ "$DOCTOR" == true ]] && [[ -z "$CONFIG_FILE" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
# --cleanup-all, --labs, --resources can run without a config file (uses proxmox.yaml)
|
|
if [[ "$CLEANUP_ALL" == true ]] && [[ -z "$CONFIG_FILE" ]]; then
|
|
return 0
|
|
fi
|
|
if [[ "$LIST_LABS" == true ]] && [[ -z "$CONFIG_FILE" ]]; then
|
|
return 0
|
|
fi
|
|
if [[ "$RESOURCES" == true ]] && [[ -z "$CONFIG_FILE" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
if [[ -z "$CONFIG_FILE" ]]; then
|
|
error "No configuration file specified"
|
|
echo "Usage: ${SCRIPT_NAME} [OPTIONS] CONFIG_FILE" >&2
|
|
echo "Run '${SCRIPT_NAME} --help' for usage information." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ ! -f "$CONFIG_FILE" ]]; then
|
|
error "Configuration file not found: ${CONFIG_FILE}"
|
|
exit 1
|
|
fi
|
|
|
|
# --deep only makes sense with --cleanup or --cleanup-all
|
|
if [[ "$DEEP_CLEANUP" == true ]] && [[ "$CLEANUP" != true ]] && [[ "$CLEANUP_ALL" != true ]]; then
|
|
error "--deep can only be used with --cleanup or --cleanup-all"
|
|
exit 1
|
|
fi
|
|
|
|
case "$PHASE" in
|
|
validate|prepare|upload|create|install|status|all) ;;
|
|
*)
|
|
error "Invalid phase: ${PHASE}"
|
|
error "Supported: validate, prepare, upload, create, install, status, all"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
if [[ -n "$METHOD" ]]; then
|
|
case "$METHOD" in
|
|
ssh|api) ;;
|
|
*)
|
|
error "Invalid method: ${METHOD}"
|
|
error "Supported: ssh, api"
|
|
exit 1
|
|
;;
|
|
esac
|
|
fi
|
|
|
|
if [[ -n "$CERT_FILE" ]] && [[ "$NO_CERT" == true ]]; then
|
|
error "Cannot use both --cert and --no-cert"
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -n "$CERT_FILE" ]] && [[ ! -f "$CERT_FILE" ]]; then
|
|
error "Certificate file not found: ${CERT_FILE}"
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -n "$LOCAL_ISO" ]] && [[ ! -f "$LOCAL_ISO" ]]; then
|
|
error "ISO file not found: ${LOCAL_ISO}"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Dependency checks
|
|
# ---------------------------------------------------------------------------
|
|
|
|
SEED_TOOL=""
|
|
|
|
check_dependencies() {
|
|
step "Checking dependencies"
|
|
|
|
local missing=false
|
|
|
|
# python3
|
|
if ! command -v python3 &>/dev/null; then
|
|
error "python3 is required but not found"
|
|
error "Install Python 3 for your platform (https://www.python.org/)"
|
|
missing=true
|
|
else
|
|
detail "python3: $(command -v python3)"
|
|
fi
|
|
|
|
# python3-yaml (optional -- basic YAML fallback built-in)
|
|
if command -v python3 &>/dev/null; then
|
|
if python3 -c "import yaml" 2>/dev/null; then
|
|
detail "PyYAML: available"
|
|
else
|
|
detail "PyYAML: not found (using built-in basic parser)"
|
|
fi
|
|
fi
|
|
|
|
# jq (optional, python3 fallback)
|
|
if command -v jq &>/dev/null; then
|
|
detail "jq: $(command -v jq)"
|
|
else
|
|
detail "jq: not found (using python3 fallback)"
|
|
fi
|
|
|
|
# curl
|
|
if ! command -v curl &>/dev/null; then
|
|
error "curl is required but not found"
|
|
missing=true
|
|
else
|
|
detail "curl: $(command -v curl)"
|
|
fi
|
|
|
|
# ssh/scp (for SSH method)
|
|
if [[ "${PVE_METHOD:-ssh}" == "ssh" ]]; then
|
|
if ! command -v ssh &>/dev/null; then
|
|
error "ssh is required for SSH method but not found"
|
|
missing=true
|
|
fi
|
|
if ! command -v scp &>/dev/null; then
|
|
error "scp is required for SSH method but not found"
|
|
missing=true
|
|
fi
|
|
fi
|
|
|
|
# incusos-seed sibling script
|
|
local script_dir
|
|
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
SEED_TOOL="${script_dir}/incusos-seed"
|
|
if [[ ! -x "$SEED_TOOL" ]]; then
|
|
error "incusos-seed not found at: ${SEED_TOOL}"
|
|
error "The incusos-seed script must be in the same directory as ${SCRIPT_NAME}"
|
|
missing=true
|
|
else
|
|
detail "incusos-seed: ${SEED_TOOL}"
|
|
fi
|
|
|
|
# bash version
|
|
if [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then
|
|
error "Bash 4+ is required (found ${BASH_VERSION})"
|
|
missing=true
|
|
else
|
|
detail "bash: ${BASH_VERSION}"
|
|
fi
|
|
|
|
if [[ "$missing" == true ]]; then
|
|
error "Missing dependencies -- cannot continue"
|
|
exit 1
|
|
fi
|
|
|
|
success "All dependencies satisfied"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Doctor mode
|
|
# ---------------------------------------------------------------------------
|
|
|
|
run_doctor() {
|
|
echo -e "${BOLD}${SCRIPT_NAME} doctor${RESET}"
|
|
echo ""
|
|
|
|
local script_dir
|
|
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
# -- Platform section --
|
|
echo -e " ${BOLD}Platform:${RESET}"
|
|
echo -e " OS: $(uname -s) $(uname -m)"
|
|
echo -e " Bash: ${BASH_VERSION}"
|
|
echo ""
|
|
|
|
# -- Tools section --
|
|
echo -e " ${BOLD}Tools:${RESET}"
|
|
|
|
# incus client
|
|
if command -v incus &>/dev/null; then
|
|
local incus_ver
|
|
incus_ver=$(incus version 2>/dev/null | sed -n 's/.*Client version: \([0-9.]*\).*/\1/p')
|
|
[[ -z "$incus_ver" ]] && incus_ver="unknown"
|
|
echo -e " incus client: ${GREEN}${incus_ver} ✓${RESET}"
|
|
else
|
|
echo -e " incus client: ${YELLOW}not installed${RESET}"
|
|
fi
|
|
|
|
# flasher-tool
|
|
if command -v flasher-tool &>/dev/null; then
|
|
local ft_ver
|
|
ft_ver=$(flasher-tool --version 2>/dev/null | head -1 | sed 's/^flasher-tool version //' || echo "installed")
|
|
if [[ -z "$ft_ver" ]] || [[ "$ft_ver" == *"unknown"* ]]; then
|
|
ft_ver="dev (installed)"
|
|
fi
|
|
echo -e " flasher-tool: ${GREEN}${ft_ver} ✓${RESET}"
|
|
else
|
|
echo -e " flasher-tool: ${YELLOW}not installed${RESET}"
|
|
fi
|
|
|
|
# operations-center CLI
|
|
if command -v operations-center &>/dev/null; then
|
|
local oc_ver
|
|
oc_ver=$(operations-center --version 2>/dev/null | head -1 || echo "installed")
|
|
echo -e " operations-center CLI: ${GREEN}${oc_ver} ✓${RESET}"
|
|
else
|
|
echo -e " operations-center CLI: ${DIM}not installed (optional)${RESET}"
|
|
fi
|
|
|
|
# python3
|
|
if command -v python3 &>/dev/null; then
|
|
local py_ver
|
|
py_ver=$(python3 --version 2>/dev/null | awk '{print $2}')
|
|
echo -e " python3: ${GREEN}${py_ver} ✓${RESET}"
|
|
else
|
|
echo -e " python3: ${RED}not installed (required)${RESET}"
|
|
fi
|
|
|
|
# curl
|
|
if command -v curl &>/dev/null; then
|
|
local curl_ver
|
|
curl_ver=$(curl --version 2>/dev/null | head -1 | awk '{print $2}')
|
|
echo -e " curl: ${GREEN}${curl_ver} ✓${RESET}"
|
|
else
|
|
echo -e " curl: ${RED}not installed (required)${RESET}"
|
|
fi
|
|
|
|
# ISO creation tool (genisoimage on Linux, mkisofs on macOS)
|
|
if command -v genisoimage &>/dev/null; then
|
|
echo -e " genisoimage: ${GREEN}installed ✓${RESET}"
|
|
elif command -v mkisofs &>/dev/null; then
|
|
echo -e " mkisofs: ${GREEN}installed ✓${RESET} (cdrtools)"
|
|
else
|
|
case "$(uname -s)" in
|
|
Darwin) echo -e " mkisofs: ${RED}not installed (brew install cdrtools)${RESET}" ;;
|
|
*) echo -e " genisoimage: ${RED}not installed (required for seed ISOs)${RESET}" ;;
|
|
esac
|
|
fi
|
|
|
|
# jq
|
|
if command -v jq &>/dev/null; then
|
|
local jq_ver
|
|
jq_ver=$(jq --version 2>/dev/null | sed 's/^jq-//')
|
|
echo -e " jq: ${GREEN}${jq_ver} ✓${RESET}"
|
|
else
|
|
echo -e " jq: ${YELLOW}not installed (python3 fallback used)${RESET}"
|
|
fi
|
|
|
|
# incusos-seed sibling
|
|
if [[ -x "${script_dir}/incusos-seed" ]]; then
|
|
echo -e " incusos-seed: ${GREEN}found ✓${RESET}"
|
|
else
|
|
echo -e " incusos-seed: ${RED}not found at ${script_dir}/incusos-seed${RESET}"
|
|
fi
|
|
|
|
# -- IncusOS section --
|
|
echo ""
|
|
echo -e " ${BOLD}IncusOS:${RESET}"
|
|
|
|
local cdn_latest=""
|
|
local cdn_index
|
|
cdn_index=$(mktemp)
|
|
if curl -fsSL --connect-timeout 5 "$CDN_INDEX" -o "$cdn_index" 2>/dev/null; then
|
|
if command -v jq &>/dev/null; then
|
|
cdn_latest=$(jq -r '[.updates[] | select(.channels[] == "stable")] | sort_by(.version) | last | .version // empty' "$cdn_index" 2>/dev/null)
|
|
else
|
|
cdn_latest=$(python3 -c "
|
|
import json
|
|
data = json.load(open('${cdn_index}'))
|
|
versions = [u['version'] for u in data.get('updates', []) if 'stable' in u.get('channels', [])]
|
|
if versions:
|
|
print(sorted(versions)[-1])
|
|
" 2>/dev/null)
|
|
fi
|
|
fi
|
|
rm -f "$cdn_index"
|
|
|
|
if [[ -n "$cdn_latest" ]]; then
|
|
echo -e " Latest stable: ${cdn_latest}"
|
|
else
|
|
echo -e " Latest stable: ${YELLOW}could not reach CDN${RESET}"
|
|
fi
|
|
|
|
# Check local cache
|
|
local cache_base="${XDG_CACHE_HOME:-${HOME}/.cache}/${SCRIPT_NAME}/iso-cache"
|
|
if [[ -d "$cache_base" ]]; then
|
|
local cached_iso
|
|
cached_iso=$(ls -1 "${cache_base}"/IncusOS_*.iso 2>/dev/null | sort | tail -1 || true)
|
|
if [[ -n "$cached_iso" ]]; then
|
|
local cached_name
|
|
cached_name=$(basename "$cached_iso" .iso | sed 's/^IncusOS_//')
|
|
if [[ -n "$cdn_latest" ]] && [[ "$cached_name" == "$cdn_latest" ]]; then
|
|
echo -e " Cached ISO: ${GREEN}${cached_name} (up to date)${RESET}"
|
|
else
|
|
echo -e " Cached ISO: ${YELLOW}${cached_name}${RESET}"
|
|
fi
|
|
else
|
|
echo -e " Cached ISO: ${DIM}none${RESET}"
|
|
fi
|
|
else
|
|
echo -e " Cached ISO: ${DIM}none${RESET}"
|
|
fi
|
|
|
|
# -- Config section --
|
|
echo ""
|
|
echo -e " ${BOLD}Config:${RESET}"
|
|
if find_proxmox_config; then
|
|
echo -e " proxmox.yaml: ${GREEN}${PROXMOX_CONFIG_RESOLVED} ✓${RESET}"
|
|
else
|
|
echo -e " proxmox.yaml: ${DIM}not found (optional)${RESET}"
|
|
fi
|
|
if [[ -n "$CONFIG_FILE" ]] && [[ -f "$CONFIG_FILE" ]]; then
|
|
echo -e " Lab config: ${CONFIG_FILE}"
|
|
fi
|
|
|
|
# -- Proxmox section (if config file or proxmox.yaml provides connection info) --
|
|
echo ""
|
|
if [[ -n "$CONFIG_FILE" ]] && [[ -f "$CONFIG_FILE" ]]; then
|
|
echo -e " ${BOLD}Proxmox:${RESET}"
|
|
|
|
# Minimal config parsing for doctor mode
|
|
local doctor_workdir
|
|
doctor_workdir=$(mktemp -d "${TMPDIR:-/tmp}/${SCRIPT_NAME}-doctor-XXXXXX")
|
|
WORKDIR="$doctor_workdir"
|
|
CONFIG_JSON="${doctor_workdir}/config.json"
|
|
|
|
if python3 -c "
|
|
import json, sys
|
|
try:
|
|
import yaml
|
|
with open(sys.argv[1]) as f:
|
|
data = yaml.safe_load(f)
|
|
except ImportError:
|
|
# Minimal inline parse -- just need proxmox section
|
|
data = {}
|
|
with open(sys.argv[1]) as f:
|
|
import re
|
|
lines = f.readlines()
|
|
processed = []
|
|
for raw in lines:
|
|
s = raw.rstrip()
|
|
if not s or s.lstrip().startswith('#'):
|
|
continue
|
|
indent = len(s) - len(s.lstrip())
|
|
content = s.lstrip()
|
|
processed.append((indent, content))
|
|
result = {}
|
|
section = None
|
|
for indent, content in processed:
|
|
m = re.match(r'^([^:]+?):\s*(.*?)$', content)
|
|
if m:
|
|
key = m.group(1).strip()
|
|
val = m.group(2).strip()
|
|
if not val:
|
|
section = key
|
|
result[section] = {}
|
|
elif section:
|
|
if val.startswith('\"') or val.startswith(\"'\"):
|
|
val = val[1:-1]
|
|
result[section][key] = val
|
|
data = result
|
|
json.dump(data, sys.stdout, indent=2)
|
|
" "$CONFIG_FILE" > "$CONFIG_JSON" 2>/dev/null; then
|
|
local doc_host doc_method doc_token_id doc_pool
|
|
|
|
doc_host=$(json_get "$CONFIG_JSON" ".proxmox.host" "")
|
|
doc_method=$(json_get "$CONFIG_JSON" ".proxmox.method" "ssh")
|
|
doc_token_id=$(json_get "$CONFIG_JSON" ".proxmox.api_token_id" "")
|
|
doc_pool=$(json_get "$CONFIG_JSON" ".proxmox.pool" "")
|
|
|
|
if [[ -z "$doc_host" ]]; then
|
|
echo -e " Host: ${RED}not set in config${RESET}"
|
|
else
|
|
echo -e " Host: ${doc_host} (${doc_method})"
|
|
|
|
# Connectivity check
|
|
local pve_ok=false
|
|
if [[ "$doc_method" == "api" ]]; then
|
|
if [[ -n "$doc_token_id" ]] && [[ -n "${PROXMOX_TOKEN_SECRET:-}" ]]; then
|
|
local auth="Authorization: PVEAPIToken=${doc_token_id}=${PROXMOX_TOKEN_SECRET}"
|
|
local pve_ver
|
|
pve_ver=$(curl -fsSk --connect-timeout 5 -H "$auth" \
|
|
"https://${doc_host}:8006/api2/json/version" 2>/dev/null | \
|
|
python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('version',''))" 2>/dev/null) || pve_ver=""
|
|
if [[ -n "$pve_ver" ]]; then
|
|
echo -e " Connectivity: ${GREEN}connected (PVE ${pve_ver}) ✓${RESET}"
|
|
pve_ok=true
|
|
else
|
|
echo -e " Connectivity: ${RED}API connection failed${RESET}"
|
|
fi
|
|
else
|
|
echo -e " Connectivity: ${YELLOW}API token not configured or PROXMOX_TOKEN_SECRET not set${RESET}"
|
|
fi
|
|
else
|
|
local ssh_user
|
|
ssh_user=$(json_get "$CONFIG_JSON" ".proxmox.ssh_user" "root")
|
|
if ssh -o BatchMode=yes -o ConnectTimeout=5 "${ssh_user}@${doc_host}" "true" &>/dev/null; then
|
|
echo -e " Connectivity: ${GREEN}SSH connected ✓${RESET}"
|
|
pve_ok=true
|
|
else
|
|
echo -e " Connectivity: ${RED}SSH connection failed${RESET}"
|
|
fi
|
|
fi
|
|
|
|
# Pool check (API only, if connected)
|
|
if [[ -n "$doc_pool" ]]; then
|
|
if [[ "$pve_ok" == true ]] && [[ "$doc_method" == "api" ]]; then
|
|
local auth="Authorization: PVEAPIToken=${doc_token_id}=${PROXMOX_TOKEN_SECRET}"
|
|
if curl -fsSk --connect-timeout 5 -H "$auth" \
|
|
"https://${doc_host}:8006/api2/json/pools/${doc_pool}" &>/dev/null; then
|
|
echo -e " Pool: ${GREEN}${doc_pool} exists ✓${RESET}"
|
|
else
|
|
echo -e " Pool: ${RED}${doc_pool} not found${RESET}"
|
|
fi
|
|
else
|
|
echo -e " Pool: ${doc_pool} (not verified)"
|
|
fi
|
|
fi
|
|
fi
|
|
else
|
|
echo -e " ${RED}Failed to parse config file${RESET}"
|
|
fi
|
|
|
|
rm -rf "$doctor_workdir"
|
|
elif [[ -n "$PROXMOX_CONFIG_RESOLVED" ]]; then
|
|
# Use proxmox.yaml for connectivity check (no lab config provided)
|
|
echo -e " ${BOLD}Proxmox:${RESET} ${DIM}(from proxmox.yaml)${RESET}"
|
|
|
|
local doctor_workdir
|
|
doctor_workdir=$(mktemp -d "${TMPDIR:-/tmp}/${SCRIPT_NAME}-doctor-XXXXXX")
|
|
WORKDIR="$doctor_workdir"
|
|
|
|
load_proxmox_config "$PROXMOX_CONFIG_RESOLVED"
|
|
[[ -z "$PVE_NODE" ]] && PVE_NODE="pve"
|
|
[[ -z "$PVE_METHOD" ]] && PVE_METHOD="ssh"
|
|
[[ -z "$PVE_SSH_USER" ]] && PVE_SSH_USER="root"
|
|
[[ -n "$METHOD" ]] && PVE_METHOD="$METHOD"
|
|
[[ -n "$HOST_OVERRIDE" ]] && PVE_HOST="$HOST_OVERRIDE"
|
|
|
|
if [[ -z "$PVE_HOST" ]]; then
|
|
echo -e " Host: ${RED}not set in proxmox.yaml${RESET}"
|
|
else
|
|
echo -e " Host: ${PVE_HOST} (${PVE_METHOD})"
|
|
local pve_ok=false
|
|
if [[ "$PVE_METHOD" == "api" ]]; then
|
|
if [[ -n "$PVE_API_TOKEN_ID" ]] && [[ -n "${PROXMOX_TOKEN_SECRET:-}" ]]; then
|
|
local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}"
|
|
local pve_ver
|
|
pve_ver=$(curl -fsSk --connect-timeout 5 -H "$auth" \
|
|
"https://${PVE_HOST}:8006/api2/json/version" 2>/dev/null | \
|
|
python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('version',''))" 2>/dev/null) || pve_ver=""
|
|
if [[ -n "$pve_ver" ]]; then
|
|
echo -e " Connectivity: ${GREEN}connected (PVE ${pve_ver}) ✓${RESET}"
|
|
pve_ok=true
|
|
else
|
|
echo -e " Connectivity: ${RED}API connection failed${RESET}"
|
|
fi
|
|
else
|
|
echo -e " Connectivity: ${YELLOW}API token not configured or PROXMOX_TOKEN_SECRET not set${RESET}"
|
|
fi
|
|
else
|
|
if ssh -o BatchMode=yes -o ConnectTimeout=5 "${PVE_SSH_USER}@${PVE_HOST}" "true" &>/dev/null; then
|
|
echo -e " Connectivity: ${GREEN}SSH connected ✓${RESET}"
|
|
pve_ok=true
|
|
else
|
|
echo -e " Connectivity: ${RED}SSH connection failed${RESET}"
|
|
fi
|
|
fi
|
|
if [[ -n "$PVE_POOL" ]]; then
|
|
if [[ "$pve_ok" == true ]] && [[ "$PVE_METHOD" == "api" ]]; then
|
|
local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}"
|
|
if curl -fsSk --connect-timeout 5 -H "$auth" \
|
|
"https://${PVE_HOST}:8006/api2/json/pools/${PVE_POOL}" &>/dev/null; then
|
|
echo -e " Pool: ${GREEN}${PVE_POOL} exists ✓${RESET}"
|
|
else
|
|
echo -e " Pool: ${RED}${PVE_POOL} not found${RESET}"
|
|
fi
|
|
else
|
|
echo -e " Pool: ${PVE_POOL} (not verified)"
|
|
fi
|
|
fi
|
|
fi
|
|
rm -rf "$doctor_workdir"
|
|
else
|
|
echo -e " ${BOLD}Proxmox:${RESET} ${DIM}(no config file or proxmox.yaml found)${RESET}"
|
|
fi
|
|
|
|
echo ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Temporary directory management
|
|
# ---------------------------------------------------------------------------
|
|
|
|
WORKDIR=""
|
|
CACHEDIR=""
|
|
|
|
setup_workdir() {
|
|
# Use persistent storage for the work directory -- the ISO download +
|
|
# decompression can exceed 4 GiB, which overflows tmpfs on most systems.
|
|
# Honour TMPDIR if explicitly set; otherwise use ~/.cache on real disk.
|
|
local base="${TMPDIR:-${XDG_CACHE_HOME:-${HOME}/.cache}/${SCRIPT_NAME}}"
|
|
mkdir -p "$base"
|
|
|
|
# Persistent cache for ISO downloads (survives across runs)
|
|
CACHEDIR="${base}/iso-cache"
|
|
mkdir -p "$CACHEDIR"
|
|
|
|
WORKDIR=$(mktemp -d "${base}/${SCRIPT_NAME}-XXXXXX")
|
|
trap 'rm -rf "$WORKDIR"' EXIT
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# YAML parsing via python3
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Convert YAML config to JSON, stored in a file for repeated access
|
|
CONFIG_JSON=""
|
|
|
|
parse_config() {
|
|
step "Parsing configuration: ${CONFIG_FILE}"
|
|
|
|
CONFIG_JSON="${WORKDIR}/config.json"
|
|
|
|
# Try PyYAML first, fall back to a basic pure-Python parser
|
|
if ! python3 -c "
|
|
import json, sys, re
|
|
|
|
def _parse_value(v):
|
|
v = v.strip()
|
|
# Strip inline comments (but not inside quotes)
|
|
if not v.startswith(('\"', \"'\")):
|
|
m = re.match(r'^(.*?)\s+#.*$', v)
|
|
if m:
|
|
v = m.group(1).rstrip()
|
|
if not v:
|
|
return None
|
|
if (v.startswith('\"') and v.endswith('\"')) or (v.startswith(\"'\") and v.endswith(\"'\")):
|
|
return v[1:-1]
|
|
if v == 'true':
|
|
return True
|
|
if v == 'false':
|
|
return False
|
|
if v == 'null' or v == '~':
|
|
return None
|
|
try:
|
|
return int(v)
|
|
except ValueError:
|
|
pass
|
|
try:
|
|
return float(v)
|
|
except ValueError:
|
|
pass
|
|
return v
|
|
|
|
def parse_yaml_basic(path):
|
|
\"\"\"Minimal YAML parser for simple config files.
|
|
Handles: mappings, lists of mappings, lists of scalars, inline comments.
|
|
Does NOT handle: anchors, multi-line strings, flow style, merge keys.\"\"\"
|
|
with open(path) as f:
|
|
lines = f.readlines()
|
|
|
|
# Pre-process: strip comments and blank lines, track indent + content
|
|
processed = []
|
|
for raw in lines:
|
|
s = raw.rstrip()
|
|
if not s or s.lstrip().startswith('#'):
|
|
continue
|
|
indent = len(s) - len(s.lstrip())
|
|
content = s.lstrip()
|
|
processed.append((indent, content))
|
|
|
|
def parse_block(idx, base_indent):
|
|
\"\"\"Parse a block at the given indentation level, return (result, next_idx).\"\"\"
|
|
if idx >= len(processed):
|
|
return None, idx
|
|
|
|
indent, content = processed[idx]
|
|
|
|
# Detect if this block is a list or mapping
|
|
if content.startswith('- '):
|
|
return parse_list(idx, indent)
|
|
else:
|
|
return parse_mapping(idx, indent)
|
|
|
|
def parse_mapping(idx, base_indent):
|
|
result = {}
|
|
while idx < len(processed):
|
|
indent, content = processed[idx]
|
|
if indent < base_indent:
|
|
break
|
|
if indent > base_indent:
|
|
break
|
|
if content.startswith('- '):
|
|
break
|
|
|
|
# Must be key: value
|
|
m = re.match(r'^([^:]+?):\s*(.*?)$', content)
|
|
if not m:
|
|
idx += 1
|
|
continue
|
|
|
|
key = m.group(1).strip()
|
|
val_str = m.group(2).strip()
|
|
|
|
# Strip inline comment from value
|
|
if val_str and not val_str.startswith(('\"', \"'\")):
|
|
cm = re.match(r'^(.*?)\s+#.*$', val_str)
|
|
if cm:
|
|
val_str = cm.group(1).rstrip()
|
|
|
|
if val_str and not val_str.startswith('#'):
|
|
result[key] = _parse_value(val_str)
|
|
idx += 1
|
|
else:
|
|
# Value is a nested block
|
|
idx += 1
|
|
if idx < len(processed) and processed[idx][0] > base_indent:
|
|
child_indent = processed[idx][0]
|
|
child, idx = parse_block(idx, child_indent)
|
|
result[key] = child if child is not None else {}
|
|
else:
|
|
result[key] = {}
|
|
|
|
return result, idx
|
|
|
|
def parse_list(idx, base_indent):
|
|
result = []
|
|
while idx < len(processed):
|
|
indent, content = processed[idx]
|
|
if indent < base_indent:
|
|
break
|
|
if indent > base_indent:
|
|
break
|
|
if not content.startswith('- '):
|
|
break
|
|
|
|
item_str = content[2:].strip()
|
|
# Check if it's a mapping item: '- key: value'
|
|
m = re.match(r'^([^:]+?):\s*(.*?)$', item_str)
|
|
if m:
|
|
# It's a mapping -- parse this item and continuation lines
|
|
obj = {}
|
|
key = m.group(1).strip()
|
|
val_str = m.group(2).strip()
|
|
if val_str and not val_str.startswith('#'):
|
|
if not val_str.startswith(('\"', \"'\")):
|
|
cm = re.match(r'^(.*?)\s+#.*$', val_str)
|
|
if cm:
|
|
val_str = cm.group(1).rstrip()
|
|
obj[key] = _parse_value(val_str)
|
|
else:
|
|
idx += 1
|
|
if idx < len(processed) and processed[idx][0] > indent:
|
|
child_indent = processed[idx][0]
|
|
child, idx = parse_block(idx, child_indent)
|
|
obj[key] = child if child is not None else {}
|
|
else:
|
|
obj[key] = {}
|
|
# Parse remaining keys at item indent + 2
|
|
item_indent = indent + 2
|
|
while idx < len(processed) and processed[idx][0] == item_indent:
|
|
ic = processed[idx][1]
|
|
if ic.startswith('- '):
|
|
break
|
|
im = re.match(r'^([^:]+?):\s*(.*?)$', ic)
|
|
if im:
|
|
ik = im.group(1).strip()
|
|
iv = im.group(2).strip()
|
|
if iv and not iv.startswith('#'):
|
|
if not iv.startswith(('\"', \"'\")):
|
|
icm = re.match(r'^(.*?)\s+#.*$', iv)
|
|
if icm:
|
|
iv = icm.group(1).rstrip()
|
|
obj[ik] = _parse_value(iv)
|
|
idx += 1
|
|
else:
|
|
idx += 1
|
|
if idx < len(processed) and processed[idx][0] > item_indent:
|
|
child_indent = processed[idx][0]
|
|
child, idx = parse_block(idx, child_indent)
|
|
obj[ik] = child if child is not None else {}
|
|
else:
|
|
obj[ik] = {}
|
|
else:
|
|
idx += 1
|
|
result.append(obj)
|
|
continue
|
|
|
|
idx += 1
|
|
# Parse remaining keys of this list item
|
|
item_indent = indent + 2
|
|
while idx < len(processed) and processed[idx][0] >= item_indent:
|
|
if processed[idx][0] == item_indent:
|
|
ic = processed[idx][1]
|
|
if ic.startswith('- '):
|
|
break
|
|
im = re.match(r'^([^:]+?):\s*(.*?)$', ic)
|
|
if im:
|
|
ik = im.group(1).strip()
|
|
iv = im.group(2).strip()
|
|
if iv and not iv.startswith('#'):
|
|
if not iv.startswith(('\"', \"'\")):
|
|
icm = re.match(r'^(.*?)\s+#.*$', iv)
|
|
if icm:
|
|
iv = icm.group(1).rstrip()
|
|
obj[ik] = _parse_value(iv)
|
|
idx += 1
|
|
else:
|
|
idx += 1
|
|
if idx < len(processed) and processed[idx][0] > item_indent:
|
|
child_indent = processed[idx][0]
|
|
child, idx = parse_block(idx, child_indent)
|
|
obj[ik] = child if child is not None else {}
|
|
else:
|
|
obj[ik] = {}
|
|
else:
|
|
idx += 1
|
|
else:
|
|
# Deeper nesting handled by recursive calls above
|
|
break
|
|
result.append(obj)
|
|
else:
|
|
# Scalar list item
|
|
result.append(_parse_value(item_str))
|
|
idx += 1
|
|
|
|
return result, idx
|
|
|
|
result, _ = parse_block(0, 0)
|
|
return result if result is not None else {}
|
|
|
|
try:
|
|
import yaml
|
|
with open(sys.argv[1]) as f:
|
|
data = yaml.safe_load(f)
|
|
except ImportError:
|
|
data = parse_yaml_basic(sys.argv[1])
|
|
|
|
json.dump(data, sys.stdout, indent=2)
|
|
" "$CONFIG_FILE" > "$CONFIG_JSON" 2>/dev/null; then
|
|
error "Failed to parse YAML configuration: ${CONFIG_FILE}"
|
|
error "Check that the file is valid YAML"
|
|
error "For complex YAML features, install PyYAML: pip3 install pyyaml"
|
|
exit 1
|
|
fi
|
|
|
|
success "Configuration parsed"
|
|
}
|
|
|
|
# JSON field accessor -- uses jq if available, falls back to python3
|
|
json_get() {
|
|
local file="$1"
|
|
local query="$2"
|
|
local default="${3:-}"
|
|
|
|
local result
|
|
if command -v jq &>/dev/null; then
|
|
result=$(jq -r "$query // empty" "$file" 2>/dev/null) || result=""
|
|
else
|
|
result=$(python3 -c "
|
|
import json, sys
|
|
data = json.load(open(sys.argv[1]))
|
|
path = sys.argv[2]
|
|
# Simple jq-like path traversal for common patterns
|
|
if path.startswith('.'):
|
|
path = path[1:]
|
|
parts = path.replace('[]','').split('.')
|
|
obj = data
|
|
for p in parts:
|
|
if p == '':
|
|
continue
|
|
if isinstance(obj, dict):
|
|
obj = obj.get(p)
|
|
elif isinstance(obj, list) and p.isdigit():
|
|
obj = obj[int(p)]
|
|
else:
|
|
obj = None
|
|
if obj is None:
|
|
break
|
|
if obj is None:
|
|
print('')
|
|
elif isinstance(obj, bool):
|
|
print('true' if obj else 'false')
|
|
elif isinstance(obj, (dict, list)):
|
|
json.dump(obj, sys.stdout)
|
|
else:
|
|
print(obj)
|
|
" "$file" "$query" 2>/dev/null) || result=""
|
|
fi
|
|
|
|
if [[ -z "$result" ]] && [[ -n "$default" ]]; then
|
|
echo "$default"
|
|
else
|
|
echo "$result"
|
|
fi
|
|
}
|
|
|
|
# Get the number of VMs in config
|
|
vm_count() {
|
|
if command -v jq &>/dev/null; then
|
|
jq -r '.vms | length' "$CONFIG_JSON" 2>/dev/null || echo "0"
|
|
else
|
|
python3 -c "
|
|
import json
|
|
data = json.load(open('$CONFIG_JSON'))
|
|
print(len(data.get('vms', [])))
|
|
" 2>/dev/null || echo "0"
|
|
fi
|
|
}
|
|
|
|
# Get a field from a specific VM by index
|
|
vm_get() {
|
|
local index="$1"
|
|
local field="$2"
|
|
local default="${3:-}"
|
|
|
|
local result
|
|
if command -v jq &>/dev/null; then
|
|
result=$(jq -r ".vms[${index}].${field} // empty" "$CONFIG_JSON" 2>/dev/null) || result=""
|
|
else
|
|
result=$(python3 -c "
|
|
import json
|
|
data = json.load(open('$CONFIG_JSON'))
|
|
vm = data.get('vms', [])[${index}]
|
|
val = vm.get('${field}')
|
|
if val is None:
|
|
print('')
|
|
elif isinstance(val, bool):
|
|
print('true' if val else 'false')
|
|
else:
|
|
print(val)
|
|
" 2>/dev/null) || result=""
|
|
fi
|
|
|
|
if [[ -z "$result" ]] && [[ -n "$default" ]]; then
|
|
echo "$default"
|
|
else
|
|
echo "$result"
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Configuration loading
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Proxmox connection settings
|
|
PVE_HOST=""
|
|
PVE_NODE=""
|
|
PVE_STORAGE=""
|
|
PVE_ISO_STORAGE=""
|
|
PVE_BRIDGE=""
|
|
PVE_METHOD=""
|
|
PVE_SSH_USER=""
|
|
PVE_API_TOKEN_ID=""
|
|
PVE_POOL=""
|
|
|
|
# Default VM settings
|
|
DEF_CORES=""
|
|
DEF_MEMORY=""
|
|
DEF_DISK=""
|
|
DEF_CHANNEL=""
|
|
DEF_START_VMID=""
|
|
|
|
# VM arrays (indexed by position)
|
|
declare -a VM_NAMES=()
|
|
declare -a VM_APPS=()
|
|
declare -a VM_APPLY_DEFAULTS=()
|
|
declare -a VM_CORES=()
|
|
declare -a VM_MEMORY=()
|
|
declare -a VM_DISK=()
|
|
declare -a VM_VMIDS=()
|
|
declare -a VM_CLUSTERS=()
|
|
declare -a VM_CLUSTER_ROLES=()
|
|
|
|
# Path to the resolved proxmox connection config file (set by find_proxmox_config)
|
|
PROXMOX_CONFIG_RESOLVED=""
|
|
|
|
# Auto-discover proxmox.yaml: --proxmox flag > script directory > cwd
|
|
find_proxmox_config() {
|
|
if [[ -n "$PROXMOX_CONFIG" ]]; then
|
|
if [[ ! -f "$PROXMOX_CONFIG" ]]; then
|
|
error "Proxmox config file not found: ${PROXMOX_CONFIG}"
|
|
exit 1
|
|
fi
|
|
PROXMOX_CONFIG_RESOLVED="$PROXMOX_CONFIG"
|
|
detail "Using proxmox config: ${PROXMOX_CONFIG_RESOLVED} (--proxmox)"
|
|
return 0
|
|
fi
|
|
|
|
local script_dir
|
|
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
if [[ -f "${script_dir}/proxmox.yaml" ]]; then
|
|
PROXMOX_CONFIG_RESOLVED="${script_dir}/proxmox.yaml"
|
|
detail "Using proxmox config: ${PROXMOX_CONFIG_RESOLVED} (auto-discovered)"
|
|
return 0
|
|
fi
|
|
|
|
if [[ -f "proxmox.yaml" ]]; then
|
|
PROXMOX_CONFIG_RESOLVED="$(pwd)/proxmox.yaml"
|
|
detail "Using proxmox config: ${PROXMOX_CONFIG_RESOLVED} (auto-discovered)"
|
|
return 0
|
|
fi
|
|
|
|
# No proxmox.yaml found -- caller decides if this is an error
|
|
return 1
|
|
}
|
|
|
|
# Parse proxmox.yaml (flat key: value structure) and set PVE_* globals as base values
|
|
load_proxmox_config() {
|
|
local pxconfig="$1"
|
|
|
|
local px_json
|
|
px_json=$(mktemp "${WORKDIR}/proxmox-config-XXXXXX.json")
|
|
|
|
if ! python3 -c "
|
|
import json, sys, re
|
|
|
|
result = {}
|
|
with open(sys.argv[1]) as f:
|
|
for line in f:
|
|
s = line.strip()
|
|
if not s or s.startswith('#'):
|
|
continue
|
|
m = re.match(r'^([^:]+?):\s*(.*?)$', s)
|
|
if m:
|
|
key = m.group(1).strip()
|
|
val = m.group(2).strip()
|
|
# Strip inline comments
|
|
if val and not val.startswith(('\"', \"'\")):
|
|
cm = re.match(r'^(.*?)\s+#.*$', val)
|
|
if cm:
|
|
val = cm.group(1).rstrip()
|
|
if (val.startswith('\"') and val.endswith('\"')) or (val.startswith(\"'\") and val.endswith(\"'\")):
|
|
val = val[1:-1]
|
|
result[key] = val
|
|
|
|
json.dump(result, sys.stdout, indent=2)
|
|
" "$pxconfig" > "$px_json" 2>/dev/null; then
|
|
error "Failed to parse proxmox config: ${pxconfig}"
|
|
exit 1
|
|
fi
|
|
|
|
# Set base values from proxmox.yaml (lab config overlay and CLI flags applied later)
|
|
local val
|
|
val=$(json_get "$px_json" ".host" ""); [[ -n "$val" ]] && PVE_HOST="$val"
|
|
val=$(json_get "$px_json" ".node" ""); [[ -n "$val" ]] && PVE_NODE="$val"
|
|
val=$(json_get "$px_json" ".storage" ""); [[ -n "$val" ]] && PVE_STORAGE="$val"
|
|
val=$(json_get "$px_json" ".iso_storage" ""); [[ -n "$val" ]] && PVE_ISO_STORAGE="$val"
|
|
val=$(json_get "$px_json" ".bridge" ""); [[ -n "$val" ]] && PVE_BRIDGE="$val"
|
|
val=$(json_get "$px_json" ".ssh_user" ""); [[ -n "$val" ]] && PVE_SSH_USER="$val"
|
|
val=$(json_get "$px_json" ".api_token_id" ""); [[ -n "$val" ]] && PVE_API_TOKEN_ID="$val"
|
|
val=$(json_get "$px_json" ".pool" ""); [[ -n "$val" ]] && PVE_POOL="$val"
|
|
val=$(json_get "$px_json" ".method" ""); [[ -n "$val" ]] && PVE_METHOD="$val"
|
|
|
|
rm -f "$px_json"
|
|
}
|
|
|
|
load_config() {
|
|
step "Loading configuration values"
|
|
|
|
# Merge priority: proxmox.yaml (base) → lab config proxmox: (overlay) → CLI flags
|
|
# Step 1: proxmox.yaml base values were already loaded by the caller via
|
|
# load_proxmox_config(). PVE_* globals may already be set.
|
|
|
|
# Step 2: overlay from lab config proxmox: section (if present)
|
|
local val
|
|
val=$(json_get "$CONFIG_JSON" ".proxmox.host" ""); [[ -n "$val" ]] && PVE_HOST="$val"
|
|
val=$(json_get "$CONFIG_JSON" ".proxmox.node" ""); [[ -n "$val" ]] && PVE_NODE="$val"
|
|
val=$(json_get "$CONFIG_JSON" ".proxmox.storage" ""); [[ -n "$val" ]] && PVE_STORAGE="$val"
|
|
val=$(json_get "$CONFIG_JSON" ".proxmox.iso_storage" ""); [[ -n "$val" ]] && PVE_ISO_STORAGE="$val"
|
|
val=$(json_get "$CONFIG_JSON" ".proxmox.bridge" ""); [[ -n "$val" ]] && PVE_BRIDGE="$val"
|
|
val=$(json_get "$CONFIG_JSON" ".proxmox.ssh_user" ""); [[ -n "$val" ]] && PVE_SSH_USER="$val"
|
|
val=$(json_get "$CONFIG_JSON" ".proxmox.api_token_id" ""); [[ -n "$val" ]] && PVE_API_TOKEN_ID="$val"
|
|
val=$(json_get "$CONFIG_JSON" ".proxmox.pool" ""); [[ -n "$val" ]] && PVE_POOL="$val"
|
|
val=$(json_get "$CONFIG_JSON" ".proxmox.method" ""); [[ -n "$val" ]] && PVE_METHOD="$val"
|
|
|
|
# Step 3: apply defaults for anything still unset
|
|
[[ -z "$PVE_NODE" ]] && PVE_NODE="pve"
|
|
[[ -z "$PVE_STORAGE" ]] && PVE_STORAGE="local-lvm"
|
|
[[ -z "$PVE_ISO_STORAGE" ]] && PVE_ISO_STORAGE="local"
|
|
[[ -z "$PVE_BRIDGE" ]] && PVE_BRIDGE="vmbr0"
|
|
[[ -z "$PVE_SSH_USER" ]] && PVE_SSH_USER="root"
|
|
[[ -z "$PVE_METHOD" ]] && PVE_METHOD="ssh"
|
|
|
|
# Step 4: CLI flags override everything
|
|
if [[ -n "$METHOD" ]]; then
|
|
PVE_METHOD="$METHOD"
|
|
fi
|
|
if [[ -n "$HOST_OVERRIDE" ]]; then
|
|
PVE_HOST="$HOST_OVERRIDE"
|
|
fi
|
|
|
|
# Default VM settings
|
|
DEF_CORES=$(json_get "$CONFIG_JSON" ".defaults.cores" "4")
|
|
DEF_MEMORY=$(json_get "$CONFIG_JSON" ".defaults.memory" "8192")
|
|
DEF_DISK=$(json_get "$CONFIG_JSON" ".defaults.disk" "64")
|
|
DEF_CHANNEL=$(json_get "$CONFIG_JSON" ".defaults.channel" "stable")
|
|
DEF_START_VMID=$(json_get "$CONFIG_JSON" ".defaults.start_vmid" "200")
|
|
|
|
# Load VMs
|
|
local count
|
|
count=$(vm_count)
|
|
|
|
local i=0
|
|
while [[ $i -lt $count ]]; do
|
|
VM_NAMES+=("$(vm_get "$i" "name" "")")
|
|
VM_APPS+=("$(vm_get "$i" "app" "incus")")
|
|
VM_APPLY_DEFAULTS+=("$(vm_get "$i" "apply_defaults" "false")")
|
|
VM_CORES+=("$(vm_get "$i" "cores" "$DEF_CORES")")
|
|
VM_MEMORY+=("$(vm_get "$i" "memory" "$DEF_MEMORY")")
|
|
VM_DISK+=("$(vm_get "$i" "disk" "$DEF_DISK")")
|
|
VM_VMIDS+=("$(vm_get "$i" "vmid" "")")
|
|
VM_CLUSTERS+=("$(vm_get "$i" "cluster" "")")
|
|
VM_CLUSTER_ROLES+=("$(vm_get "$i" "cluster_role" "")")
|
|
|
|
i=$((i + 1))
|
|
done
|
|
|
|
success "Loaded ${count} VM definition(s)"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Configuration validation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
validate_config() {
|
|
step "Validating configuration"
|
|
|
|
local errors=0
|
|
|
|
# Required: Proxmox host
|
|
if [[ -z "$PVE_HOST" ]]; then
|
|
error "proxmox.host is required"
|
|
errors=$((errors + 1))
|
|
fi
|
|
|
|
# Method validation
|
|
case "$PVE_METHOD" in
|
|
ssh|api) ;;
|
|
*)
|
|
error "Invalid proxmox.method: ${PVE_METHOD} (must be ssh or api)"
|
|
errors=$((errors + 1))
|
|
;;
|
|
esac
|
|
|
|
# API method requires token
|
|
if [[ "$PVE_METHOD" == "api" ]]; then
|
|
if [[ -z "$PVE_API_TOKEN_ID" ]]; then
|
|
error "proxmox.api_token_id is required for API method"
|
|
error "Example: automation@pve!deploy"
|
|
errors=$((errors + 1))
|
|
fi
|
|
if [[ -z "${PROXMOX_TOKEN_SECRET:-}" ]]; then
|
|
error "PROXMOX_TOKEN_SECRET is not set (no env file found in repo tree)"
|
|
error "Create an env file or export directly (see --help)"
|
|
errors=$((errors + 1))
|
|
fi
|
|
fi
|
|
|
|
# Must have at least one VM
|
|
local count=${#VM_NAMES[@]}
|
|
if [[ $count -eq 0 ]]; then
|
|
error "No VMs defined in configuration"
|
|
errors=$((errors + 1))
|
|
fi
|
|
|
|
# Validate each VM
|
|
local seen_names=""
|
|
local cluster_inits=""
|
|
local i=0
|
|
while [[ $i -lt $count ]]; do
|
|
local name="${VM_NAMES[$i]}"
|
|
local app="${VM_APPS[$i]}"
|
|
local memory="${VM_MEMORY[$i]}"
|
|
local disk="${VM_DISK[$i]}"
|
|
local cluster="${VM_CLUSTERS[$i]}"
|
|
local role="${VM_CLUSTER_ROLES[$i]}"
|
|
|
|
# Name required
|
|
if [[ -z "$name" ]]; then
|
|
error "VM #$((i + 1)): name is required"
|
|
errors=$((errors + 1))
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
# Unique names
|
|
if echo "$seen_names" | grep -qw "$name"; then
|
|
error "VM '${name}': duplicate VM name"
|
|
errors=$((errors + 1))
|
|
fi
|
|
seen_names="${seen_names} ${name}"
|
|
|
|
# Valid app
|
|
case "$app" in
|
|
incus|operations-center) ;;
|
|
*)
|
|
error "VM '${name}': invalid app '${app}' (must be incus or operations-center)"
|
|
errors=$((errors + 1))
|
|
;;
|
|
esac
|
|
|
|
# Minimum memory
|
|
if [[ "$memory" -lt "$MIN_MEMORY_MIB" ]] 2>/dev/null; then
|
|
error "VM '${name}': memory ${memory} MiB is below minimum ${MIN_MEMORY_MIB} MiB"
|
|
errors=$((errors + 1))
|
|
fi
|
|
|
|
# Minimum disk
|
|
if [[ "$disk" -lt "$MIN_DISK_GIB" ]] 2>/dev/null; then
|
|
error "VM '${name}': disk ${disk} GiB is below minimum ${MIN_DISK_GIB} GiB"
|
|
errors=$((errors + 1))
|
|
fi
|
|
|
|
# Cluster role validation
|
|
if [[ -n "$role" ]] && [[ -z "$cluster" ]]; then
|
|
error "VM '${name}': cluster_role '${role}' set but no cluster name specified"
|
|
errors=$((errors + 1))
|
|
fi
|
|
if [[ -n "$role" ]]; then
|
|
case "$role" in
|
|
init|join) ;;
|
|
*)
|
|
error "VM '${name}': invalid cluster_role '${role}' (must be init or join)"
|
|
errors=$((errors + 1))
|
|
;;
|
|
esac
|
|
fi
|
|
|
|
# Track cluster init nodes
|
|
if [[ -n "$cluster" ]] && [[ "$role" == "init" ]]; then
|
|
if echo "$cluster_inits" | grep -qw "$cluster"; then
|
|
error "Cluster '${cluster}': multiple init nodes defined (only one allowed)"
|
|
errors=$((errors + 1))
|
|
fi
|
|
cluster_inits="${cluster_inits} ${cluster}"
|
|
fi
|
|
|
|
i=$((i + 1))
|
|
done
|
|
|
|
# VM filter validation
|
|
if [[ -n "$VM_FILTER" ]]; then
|
|
local found=false
|
|
for name in "${VM_NAMES[@]}"; do
|
|
if [[ "$name" == "$VM_FILTER" ]]; then
|
|
found=true
|
|
break
|
|
fi
|
|
done
|
|
if [[ "$found" != true ]]; then
|
|
error "VM '${VM_FILTER}' not found in configuration"
|
|
error "Available VMs: ${VM_NAMES[*]}"
|
|
errors=$((errors + 1))
|
|
fi
|
|
fi
|
|
|
|
if [[ $errors -gt 0 ]]; then
|
|
error "${errors} validation error(s) found"
|
|
exit 1
|
|
fi
|
|
|
|
success "Configuration valid"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Proxmox transport layer
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Execute a command on the Proxmox host
|
|
pve_run() {
|
|
local cmd="$1"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] ${PVE_METHOD}: ${cmd}"
|
|
return 0
|
|
fi
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
ssh -o BatchMode=yes -o ConnectTimeout=10 \
|
|
"${PVE_SSH_USER}@${PVE_HOST}" "$cmd"
|
|
;;
|
|
api)
|
|
error "API execution of arbitrary commands is not supported"
|
|
error "Use specific API functions instead"
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Validate API auth prerequisites and connectivity.
|
|
# Call this before any code path that uses pve_api() outside of phase_validate().
|
|
# Exits with error if credentials are missing or API is unreachable.
|
|
require_api_auth() {
|
|
if [[ "$PVE_METHOD" != "api" ]]; then
|
|
return 0
|
|
fi
|
|
if [[ -z "$PVE_API_TOKEN_ID" ]]; then
|
|
error "API token ID is required for API method"
|
|
error "Set proxmox.api_token_id in proxmox.yaml"
|
|
exit 1
|
|
fi
|
|
if [[ -z "${PROXMOX_TOKEN_SECRET:-}" ]]; then
|
|
error "PROXMOX_TOKEN_SECRET is not set (no env file found in repo tree)"
|
|
error "Create an env file: echo 'export PROXMOX_TOKEN_SECRET=\"<secret>\"' > env"
|
|
error "Or export directly: export PROXMOX_TOKEN_SECRET=\"<your-token-secret>\""
|
|
exit 1
|
|
fi
|
|
# Verify we can actually reach the API with these credentials
|
|
local url="https://${PVE_HOST}:8006/api2/json/version"
|
|
local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}"
|
|
if ! curl -fsSk --connect-timeout 5 -H "$auth" "$url" &>/dev/null; then
|
|
error "Cannot authenticate to Proxmox API at https://${PVE_HOST}:8006"
|
|
error "Check that the host is reachable, token ID is correct, and secret is valid"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# Execute a Proxmox API call (API method)
|
|
# Usage: pve_api METHOD ENDPOINT [DATA] [json]
|
|
# Pass "json" as 4th arg to send data as application/json instead of form-encoded.
|
|
pve_api() {
|
|
local method="$1"
|
|
local endpoint="$2"
|
|
local data="${3:-}"
|
|
local content_type="${4:-form}"
|
|
|
|
local url="https://${PVE_HOST}:8006/api2/json${endpoint}"
|
|
local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET:-}"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] API ${method} ${endpoint}"
|
|
if [[ -n "$data" ]]; then
|
|
detail "[dry-run] data: ${data}"
|
|
fi
|
|
return 0
|
|
fi
|
|
|
|
local curl_args=(-fsSk -H "$auth")
|
|
if [[ "$content_type" == "json" ]]; then
|
|
curl_args+=(-H "Content-Type: application/json")
|
|
fi
|
|
|
|
case "$method" in
|
|
GET)
|
|
curl "${curl_args[@]}" "$url"
|
|
;;
|
|
POST)
|
|
if [[ -n "$data" ]]; then
|
|
curl "${curl_args[@]}" -X POST -d "$data" "$url"
|
|
else
|
|
curl "${curl_args[@]}" -X POST "$url"
|
|
fi
|
|
;;
|
|
PUT)
|
|
if [[ -n "$data" ]]; then
|
|
curl "${curl_args[@]}" -X PUT -d "$data" "$url"
|
|
else
|
|
curl "${curl_args[@]}" -X PUT "$url"
|
|
fi
|
|
;;
|
|
DELETE)
|
|
curl "${curl_args[@]}" -X DELETE "$url"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Upload a file to Proxmox ISO storage
|
|
pve_upload() {
|
|
local local_path="$1"
|
|
local remote_filename="$2"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] upload: ${local_path} -> ${PVE_ISO_STORAGE}:iso/${remote_filename}"
|
|
return 0
|
|
fi
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
local iso_dir
|
|
# Query the actual path of the ISO storage
|
|
iso_dir=$(pve_run "pvesm path ${PVE_ISO_STORAGE}:iso/test 2>/dev/null" | sed 's|/test$||') || true
|
|
if [[ -z "$iso_dir" ]]; then
|
|
# Common default paths
|
|
iso_dir="/var/lib/vz/template/iso"
|
|
fi
|
|
scp -o BatchMode=yes -o ConnectTimeout=10 \
|
|
"$local_path" "${PVE_SSH_USER}@${PVE_HOST}:${iso_dir}/${remote_filename}"
|
|
;;
|
|
api)
|
|
local upload_response upload_rc
|
|
upload_response=$(curl -sSk \
|
|
-H "Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET:-}" \
|
|
-X POST \
|
|
-F "content=iso" \
|
|
-F "filename=@${local_path};filename=${remote_filename}" \
|
|
"https://${PVE_HOST}:8006/api2/json/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/upload" \
|
|
-w "\n%{http_code}" 2>&1) || true
|
|
local http_code
|
|
http_code=$(echo "$upload_response" | tail -1)
|
|
local body
|
|
body=$(echo "$upload_response" | sed '$d')
|
|
|
|
if [[ "$http_code" -ge 200 ]] && [[ "$http_code" -lt 300 ]] 2>/dev/null; then
|
|
return 0
|
|
else
|
|
error "Upload failed (HTTP ${http_code})"
|
|
# Extract error message from JSON response
|
|
local api_error
|
|
api_error=$(echo "$body" | python3 -c "
|
|
import json, sys
|
|
try:
|
|
data = json.load(sys.stdin)
|
|
errors = data.get('errors', {})
|
|
if errors:
|
|
for k, v in errors.items():
|
|
print(f' {k}: {v}')
|
|
elif data.get('message'):
|
|
print(f' {data[\"message\"]}')
|
|
else:
|
|
print(json.dumps(data, indent=2))
|
|
except:
|
|
print(sys.stdin.read() if hasattr(sys.stdin, 'read') else '')
|
|
" 2>/dev/null) || api_error="$body"
|
|
if [[ -n "$api_error" ]]; then
|
|
error "Proxmox API response:"
|
|
echo "$api_error" | while IFS= read -r line; do
|
|
error " ${line}"
|
|
done
|
|
fi
|
|
return 1
|
|
fi
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Check if an ISO exists on Proxmox storage
|
|
pve_iso_exists() {
|
|
local filename="$1"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
return 1
|
|
fi
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
pve_run "pvesm list ${PVE_ISO_STORAGE} --content iso 2>/dev/null" | grep -q "$filename"
|
|
;;
|
|
api)
|
|
pve_api GET "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content?content=iso" | \
|
|
grep -q "$filename"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Get list of existing VMIDs
|
|
pve_list_vmids() {
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
echo ""
|
|
return 0
|
|
fi
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
pve_run "qm list 2>/dev/null" | awk 'NR>1 {print $1}' || true
|
|
;;
|
|
api)
|
|
pve_api GET "/nodes/${PVE_NODE}/qemu" 2>/dev/null | \
|
|
python3 -c "
|
|
import json, sys
|
|
data = json.load(sys.stdin)
|
|
for vm in data.get('data', []):
|
|
print(vm.get('vmid', ''))
|
|
" 2>/dev/null || true
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Create a VM via qm create
|
|
pve_create_vm() {
|
|
local vmid="$1"
|
|
local name="$2"
|
|
local cores="$3"
|
|
local memory="$4"
|
|
local disk_gib="$5"
|
|
local iso_filename="$6"
|
|
local seed_filename="$7"
|
|
|
|
local description
|
|
description=$(build_vm_description)
|
|
|
|
local cmd="qm create ${vmid}"
|
|
cmd+=" --name ${name}"
|
|
cmd+=" --description '${description}'"
|
|
cmd+=" --ostype l26"
|
|
cmd+=" --bios ovmf"
|
|
cmd+=" --machine q35"
|
|
cmd+=" --efidisk0 ${PVE_STORAGE}:0,efitype=4m,pre-enrolled-keys=0"
|
|
cmd+=" --tpmstate0 ${PVE_STORAGE}:1,version=v2.0"
|
|
cmd+=" --cpu host"
|
|
cmd+=" --cores ${cores}"
|
|
cmd+=" --memory ${memory}"
|
|
cmd+=" --balloon 0"
|
|
cmd+=" --scsihw virtio-scsi-pci"
|
|
cmd+=" --scsi0 ${PVE_STORAGE}:${disk_gib},iothread=1"
|
|
cmd+=" --ide2 ${PVE_ISO_STORAGE}:iso/${iso_filename},media=cdrom"
|
|
cmd+=" --ide3 ${PVE_ISO_STORAGE}:iso/${seed_filename},media=cdrom"
|
|
cmd+=" --net0 virtio,bridge=${PVE_BRIDGE}"
|
|
cmd+=" --boot order=ide2\\;scsi0"
|
|
cmd+=" --agent 1"
|
|
if [[ -n "$PVE_POOL" ]]; then
|
|
cmd+=" --pool ${PVE_POOL}"
|
|
fi
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] ${cmd}"
|
|
return 0
|
|
fi
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
pve_run "$cmd"
|
|
;;
|
|
api)
|
|
# Build JSON payload -- the Proxmox API requires JSON for VM
|
|
# creation because comma-separated disk/net specs are misinterpreted
|
|
# when sent as form-encoded data.
|
|
local json_payload
|
|
json_payload=$(python3 -c "
|
|
import json
|
|
payload = {
|
|
'vmid': ${vmid},
|
|
'name': '${name}',
|
|
'description': '''${description}''',
|
|
'ostype': 'l26',
|
|
'bios': 'ovmf',
|
|
'machine': 'q35',
|
|
'efidisk0': '${PVE_STORAGE}:0,efitype=4m,pre-enrolled-keys=0',
|
|
'tpmstate0': '${PVE_STORAGE}:1,version=v2.0',
|
|
'cpu': 'host',
|
|
'cores': ${cores},
|
|
'memory': ${memory},
|
|
'balloon': 0,
|
|
'scsihw': 'virtio-scsi-pci',
|
|
'scsi0': '${PVE_STORAGE}:${disk_gib},iothread=1',
|
|
'ide2': '${PVE_ISO_STORAGE}:iso/${iso_filename},media=cdrom',
|
|
'ide3': '${PVE_ISO_STORAGE}:iso/${seed_filename},media=cdrom',
|
|
'net0': 'virtio,bridge=${PVE_BRIDGE}',
|
|
'boot': 'order=ide2;scsi0',
|
|
'agent': 1,
|
|
}
|
|
pool = '${PVE_POOL}'
|
|
if pool:
|
|
payload['pool'] = pool
|
|
print(json.dumps(payload))
|
|
")
|
|
local create_response
|
|
create_response=$(pve_api POST "/nodes/${PVE_NODE}/qemu" "$json_payload" json)
|
|
# VM creation is async -- the API returns a UPID. Wait for the
|
|
# task to finish before returning, otherwise the caller may try
|
|
# to start or configure the VM while it's still being created.
|
|
local upid
|
|
upid=$(echo "$create_response" | python3 -c "import json,sys; print(json.load(sys.stdin).get('data',''))" 2>/dev/null) || upid=""
|
|
if [[ -n "$upid" ]]; then
|
|
local encoded_upid
|
|
encoded_upid=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${upid}', safe=''))" 2>/dev/null)
|
|
local wait_elapsed=0
|
|
while [[ $wait_elapsed -lt 60 ]]; do
|
|
sleep 2
|
|
wait_elapsed=$((wait_elapsed + 2))
|
|
local task_status
|
|
task_status=$(pve_api GET "/nodes/${PVE_NODE}/tasks/${encoded_upid}/status" 2>/dev/null | \
|
|
python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status',''))" 2>/dev/null) || task_status=""
|
|
if [[ "$task_status" == "stopped" ]]; then
|
|
break
|
|
fi
|
|
done
|
|
fi
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Start a VM
|
|
pve_start_vm() {
|
|
local vmid="$1"
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh) pve_run "qm start ${vmid}" ;;
|
|
api) pve_api POST "/nodes/${PVE_NODE}/qemu/${vmid}/status/start" >/dev/null ;;
|
|
esac
|
|
}
|
|
|
|
# Stop a VM
|
|
pve_stop_vm() {
|
|
local vmid="$1"
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh) pve_run "qm stop ${vmid}" ;;
|
|
api) pve_api POST "/nodes/${PVE_NODE}/qemu/${vmid}/status/stop" >/dev/null ;;
|
|
esac
|
|
}
|
|
|
|
# Destroy a VM (waits for async task to complete)
|
|
pve_destroy_vm() {
|
|
local vmid="$1"
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh) pve_run "qm destroy ${vmid} --purge" ;;
|
|
api)
|
|
local destroy_response
|
|
destroy_response=$(pve_api DELETE "/nodes/${PVE_NODE}/qemu/${vmid}?purge=1&destroy-unreferenced-disks=1" 2>/dev/null) || return 1
|
|
# VM destruction is async -- wait for the UPID task to finish
|
|
# to avoid VMID/disk reuse issues in rapid destroy+create cycles.
|
|
local upid
|
|
upid=$(echo "$destroy_response" | python3 -c "import json,sys; print(json.load(sys.stdin).get('data',''))" 2>/dev/null) || upid=""
|
|
if [[ -n "$upid" ]]; then
|
|
local encoded_upid
|
|
encoded_upid=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${upid}', safe=''))" 2>/dev/null)
|
|
local wait_elapsed=0
|
|
while [[ $wait_elapsed -lt 30 ]]; do
|
|
sleep 2
|
|
wait_elapsed=$((wait_elapsed + 2))
|
|
local task_status
|
|
task_status=$(pve_api GET "/nodes/${PVE_NODE}/tasks/${encoded_upid}/status" 2>/dev/null | \
|
|
python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status',''))" 2>/dev/null) || task_status=""
|
|
if [[ "$task_status" == "stopped" ]]; then
|
|
break
|
|
fi
|
|
done
|
|
fi
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Set VM configuration
|
|
pve_set_vm() {
|
|
local vmid="$1"
|
|
shift
|
|
local opts="$*"
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh) pve_run "qm set ${vmid} ${opts}" ;;
|
|
api)
|
|
# Convert CLI-style opts (--key value) to JSON.
|
|
# JSON avoids form-encoding ambiguity with values containing = and ;
|
|
# Duplicate keys (e.g. --delete ide2 --delete ide3) are merged with commas.
|
|
# Shell escapes (e.g. \;) are stripped since the API doesn't need them.
|
|
local json_payload
|
|
json_payload=$(echo "$opts" | python3 -c "
|
|
import json, sys
|
|
tokens = sys.stdin.read().split()
|
|
obj = {}
|
|
i = 0
|
|
while i < len(tokens):
|
|
t = tokens[i]
|
|
if t.startswith('--'):
|
|
key = t.lstrip('-')
|
|
if i + 1 < len(tokens) and not tokens[i+1].startswith('--'):
|
|
val = tokens[i+1].replace(chr(92), '')
|
|
i += 2
|
|
else:
|
|
val = ''
|
|
i += 1
|
|
if key in obj and obj[key]:
|
|
obj[key] = obj[key] + ',' + val
|
|
else:
|
|
obj[key] = val
|
|
else:
|
|
i += 1
|
|
print(json.dumps(obj))
|
|
" 2>/dev/null)
|
|
pve_api PUT "/nodes/${PVE_NODE}/qemu/${vmid}/config" "$json_payload" json >/dev/null
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Get VM status
|
|
pve_vm_status() {
|
|
local vmid="$1"
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
pve_run "qm status ${vmid}" | awk '{print $2}'
|
|
;;
|
|
api)
|
|
pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" | \
|
|
python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status','unknown'))" 2>/dev/null || echo "unknown"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Get VM uptime (seconds); returns 0 if stopped or unknown
|
|
pve_vm_uptime() {
|
|
local vmid="$1"
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
pve_run "qm status ${vmid} --verbose" | awk '/^uptime:/{print $2}'
|
|
;;
|
|
api)
|
|
pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" | \
|
|
python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('uptime',0))" 2>/dev/null || echo "0"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Get cumulative write bytes for scsi0 (from QEMU blockstat).
|
|
# Returns 0 if unavailable. Only works with api method.
|
|
pve_vm_disk_writes() {
|
|
local vmid="$1"
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
echo "0"
|
|
;;
|
|
api)
|
|
pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" | \
|
|
python3 -c "
|
|
import json,sys
|
|
d = json.load(sys.stdin).get('data',{})
|
|
bs = d.get('blockstat',{}).get('scsi0',{})
|
|
print(bs.get('wr_bytes',0))
|
|
" 2>/dev/null || echo "0"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Check if a VMID exists
|
|
pve_vm_exists() {
|
|
local vmid="$1"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
return 1
|
|
fi
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
pve_run "qm status ${vmid}" &>/dev/null
|
|
;;
|
|
api)
|
|
pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" &>/dev/null
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Get VM's IP address (best-effort via QEMU guest agent)
|
|
pve_vm_ip() {
|
|
local vmid="$1"
|
|
|
|
# Method 1: Try QEMU guest agent (works if agent is installed)
|
|
local ip=""
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
ip=$(pve_run "qm guest cmd ${vmid} network-get-interfaces 2>/dev/null" | \
|
|
python3 -c "
|
|
import json, sys
|
|
try:
|
|
data = json.load(sys.stdin)
|
|
for iface in data:
|
|
if iface.get('name') == 'lo':
|
|
continue
|
|
for addr in iface.get('ip-addresses', []):
|
|
if addr.get('ip-address-type') == 'ipv4':
|
|
print(addr['ip-address'])
|
|
sys.exit(0)
|
|
except:
|
|
pass
|
|
" 2>/dev/null) || true
|
|
;;
|
|
api)
|
|
ip=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/agent/network-get-interfaces" 2>/dev/null | \
|
|
python3 -c "
|
|
import json, sys
|
|
try:
|
|
data = json.load(sys.stdin).get('data', {}).get('result', [])
|
|
for iface in data:
|
|
if iface.get('name') == 'lo':
|
|
continue
|
|
for addr in iface.get('ip-addresses', []):
|
|
if addr.get('ip-address-type') == 'ipv4':
|
|
print(addr['ip-address'])
|
|
sys.exit(0)
|
|
except:
|
|
pass
|
|
" 2>/dev/null) || true
|
|
;;
|
|
esac
|
|
|
|
if [[ -n "$ip" ]]; then
|
|
echo "$ip"
|
|
return 0
|
|
fi
|
|
|
|
# Method 2: ARP-based lookup using VM's MAC address.
|
|
# Works without guest agent (e.g. IncusOS which is immutable).
|
|
# Get MAC from VM config, flush stale ARP, ping sweep, look up.
|
|
local mac=""
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
mac=$(pve_run "qm config ${vmid}" 2>/dev/null | \
|
|
awk -F'=' '/^net0:/{print $1}' | grep -oiE '[0-9a-f:]{17}') || true
|
|
;;
|
|
api)
|
|
mac=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null | \
|
|
python3 -c "
|
|
import json, sys, re
|
|
try:
|
|
cfg = json.load(sys.stdin).get('data', {})
|
|
net0 = cfg.get('net0', '')
|
|
m = re.search(r'([0-9A-Fa-f:]{17})', net0)
|
|
if m:
|
|
print(m.group(1).lower())
|
|
except:
|
|
pass
|
|
" 2>/dev/null) || true
|
|
;;
|
|
esac
|
|
|
|
if [[ -z "$mac" ]]; then
|
|
return 1
|
|
fi
|
|
|
|
# Populate ARP table to find this MAC's IP address.
|
|
# Strategy: first check existing ARP, then try broadcast ping, then
|
|
# fall back to a parallel ping sweep of the local subnet.
|
|
ip neigh flush nud stale 2>/dev/null || true
|
|
|
|
local bridge_subnet
|
|
bridge_subnet=$(ip -4 addr show | awk '/inet / && !/127\.0\.0/ {print $2; exit}')
|
|
|
|
# Try broadcast ping first (fast, but many hosts ignore it)
|
|
if [[ -n "$bridge_subnet" ]]; then
|
|
local broadcast
|
|
broadcast=$(python3 -c "import ipaddress; print(ipaddress.ip_interface('${bridge_subnet}').network.broadcast_address)" 2>/dev/null) || true
|
|
if [[ -n "$broadcast" ]]; then
|
|
ping -c 2 -W 1 -b "$broadcast" &>/dev/null || true
|
|
sleep 1
|
|
fi
|
|
fi
|
|
|
|
# Check ARP table
|
|
ip=$(ip neigh show | awk -v mac="$mac" 'tolower($0) ~ tolower(mac) && (/REACHABLE/ || /DELAY/ || /STALE/) {print $1; exit}') || true
|
|
|
|
if [[ -n "$ip" ]] && ping -c 1 -W 1 "$ip" &>/dev/null; then
|
|
echo "$ip"
|
|
return 0
|
|
fi
|
|
|
|
# Broadcast didn't work — parallel ping sweep of the subnet
|
|
if [[ -n "$bridge_subnet" ]]; then
|
|
local net_prefix
|
|
net_prefix=$(python3 -c "
|
|
import ipaddress
|
|
net = ipaddress.ip_interface('${bridge_subnet}').network
|
|
for h in net.hosts():
|
|
print(h)
|
|
" 2>/dev/null)
|
|
if [[ -n "$net_prefix" ]]; then
|
|
# Ping all hosts in parallel (background, 1s timeout each)
|
|
while IFS= read -r host_ip; do
|
|
ping -c 1 -W 1 "$host_ip" &>/dev/null &
|
|
done <<< "$net_prefix"
|
|
wait
|
|
sleep 1
|
|
|
|
# Re-check ARP table
|
|
ip=$(ip neigh show | awk -v mac="$mac" 'tolower($0) ~ tolower(mac) && (/REACHABLE/ || /DELAY/) {print $1; exit}') || true
|
|
|
|
if [[ -n "$ip" ]] && ping -c 1 -W 1 "$ip" &>/dev/null; then
|
|
echo "$ip"
|
|
return 0
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
return 1
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# VM management marker
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Build the description string for a managed VM
|
|
build_vm_description() {
|
|
local config_basename
|
|
config_basename=$(basename "$CONFIG_FILE")
|
|
local timestamp
|
|
timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
local marker="${MANAGED_MARKER} config=${config_basename} created=${timestamp} by=${SCRIPT_NAME}/${VERSION}"
|
|
if [[ -n "$PVE_POOL" ]]; then
|
|
marker+=" pool=${PVE_POOL}"
|
|
fi
|
|
echo "$marker"
|
|
}
|
|
|
|
# Check if a VM has our management marker in its description
|
|
pve_vm_is_managed() {
|
|
local vmid="$1"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
return 1
|
|
fi
|
|
|
|
local config_output=""
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
config_output=$(pve_run "qm config ${vmid}" 2>/dev/null) || return 1
|
|
;;
|
|
api)
|
|
config_output=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null) || return 1
|
|
;;
|
|
esac
|
|
|
|
echo "$config_output" | grep -qF "$MANAGED_MARKER"
|
|
}
|
|
|
|
# Check if a VM still has install media (ide2) attached.
|
|
# Returns 0 if install media is present, 1 if not (already installed).
|
|
pve_vm_has_install_media() {
|
|
local vmid="$1"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
return 1
|
|
fi
|
|
|
|
local config_output=""
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
config_output=$(pve_run "qm config ${vmid}" 2>/dev/null) || return 1
|
|
;;
|
|
api)
|
|
config_output=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null) || return 1
|
|
;;
|
|
esac
|
|
|
|
echo "$config_output" | grep -q "ide2"
|
|
}
|
|
|
|
# List all VMs as "vmid name" pairs (one per line)
|
|
# Returns non-zero and prints error message on API/SSH failure.
|
|
pve_list_vms() {
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
echo ""
|
|
return 0
|
|
fi
|
|
|
|
local raw_response
|
|
|
|
# When a resource pool is configured, list only pool members (type=qemu)
|
|
if [[ -n "$PVE_POOL" ]]; then
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
if ! raw_response=$(pve_run "pvesh get /pools/${PVE_POOL} --output-format json 2>/dev/null" 2>/dev/null); then
|
|
echo "[error] Failed to list VMs via SSH" >&2
|
|
return 1
|
|
fi
|
|
echo "$raw_response" | python3 -c "
|
|
import json, sys
|
|
data = json.load(sys.stdin)
|
|
for m in data.get('members', []):
|
|
if m.get('type') == 'qemu':
|
|
print('{} {}'.format(m.get('vmid', ''), m.get('name', '')))
|
|
" 2>/dev/null || true
|
|
;;
|
|
api)
|
|
if ! raw_response=$(pve_api GET "/pools/${PVE_POOL}" 2>/dev/null); then
|
|
echo "[error] Failed to list VMs from pool '${PVE_POOL}' via API (auth failure or pool not found)" >&2
|
|
return 1
|
|
fi
|
|
echo "$raw_response" | python3 -c "
|
|
import json, sys
|
|
data = json.load(sys.stdin).get('data', {})
|
|
for m in data.get('members', []):
|
|
if m.get('type') == 'qemu':
|
|
print('{} {}'.format(m.get('vmid', ''), m.get('name', '')))
|
|
" 2>/dev/null || true
|
|
;;
|
|
esac
|
|
return 0
|
|
fi
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
pve_run "qm list 2>/dev/null" | awk 'NR>1 {print $1, $2}' || true
|
|
;;
|
|
api)
|
|
if ! raw_response=$(pve_api GET "/nodes/${PVE_NODE}/qemu" 2>/dev/null); then
|
|
echo "[error] Failed to list VMs via API (auth failure or node not found)" >&2
|
|
return 1
|
|
fi
|
|
echo "$raw_response" | python3 -c "
|
|
import json, sys
|
|
data = json.load(sys.stdin)
|
|
for vm in data.get('data', []):
|
|
print('{} {}'.format(vm.get('vmid', ''), vm.get('name', '')))
|
|
" 2>/dev/null || true
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Pre-flight checks
|
|
# ---------------------------------------------------------------------------
|
|
|
|
preflight_checks() {
|
|
step "Running pre-flight checks"
|
|
|
|
local errors=0
|
|
local warnings=0
|
|
|
|
# 1. Check that storage pools exist on Proxmox
|
|
if [[ "$DRY_RUN" != true ]]; then
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
if ! pve_run "pvesm status 2>/dev/null" | awk 'NR>1 {print \$1}' | grep -qw "$PVE_STORAGE"; then
|
|
error "Storage pool '${PVE_STORAGE}' not found on Proxmox"
|
|
error "Available pools: $(pve_run "pvesm status 2>/dev/null" | awk 'NR>1 {print \$1}' | tr '\n' ' ')"
|
|
error "Fix: update 'proxmox.storage' in your config file"
|
|
errors=$((errors + 1))
|
|
fi
|
|
if ! pve_run "pvesm status 2>/dev/null" | awk 'NR>1 {print \$1}' | grep -qw "$PVE_ISO_STORAGE"; then
|
|
error "ISO storage '${PVE_ISO_STORAGE}' not found on Proxmox"
|
|
error "Available pools: $(pve_run "pvesm status 2>/dev/null" | awk 'NR>1 {print \$1}' | tr '\n' ' ')"
|
|
error "Fix: update 'proxmox.iso_storage' in your config file"
|
|
errors=$((errors + 1))
|
|
fi
|
|
;;
|
|
api)
|
|
local storage_json
|
|
storage_json=$(pve_api GET "/nodes/${PVE_NODE}/storage" 2>/dev/null) || storage_json=""
|
|
local storage_list
|
|
storage_list=$(echo "$storage_json" | python3 -c "
|
|
import json, sys
|
|
data = json.load(sys.stdin)
|
|
for s in data.get('data', []):
|
|
print(s.get('storage', ''))
|
|
" 2>/dev/null) || storage_list=""
|
|
if ! echo "$storage_list" | grep -qw "$PVE_STORAGE"; then
|
|
error "Storage pool '${PVE_STORAGE}' not found on Proxmox"
|
|
error "Available pools: $(echo "$storage_list" | tr '\n' ' ')"
|
|
error "Fix: update 'proxmox.storage' in your config file"
|
|
errors=$((errors + 1))
|
|
fi
|
|
if ! echo "$storage_list" | grep -qw "$PVE_ISO_STORAGE"; then
|
|
error "ISO storage '${PVE_ISO_STORAGE}' not found on Proxmox"
|
|
error "Available pools: $(echo "$storage_list" | tr '\n' ' ')"
|
|
error "Fix: update 'proxmox.iso_storage' in your config file"
|
|
errors=$((errors + 1))
|
|
else
|
|
# Check that ISO storage accepts 'iso' content type
|
|
local iso_content
|
|
iso_content=$(echo "$storage_json" | python3 -c "
|
|
import json, sys
|
|
data = json.load(sys.stdin)
|
|
for s in data.get('data', []):
|
|
if s.get('storage') == '${PVE_ISO_STORAGE}':
|
|
print(s.get('content', ''))
|
|
break
|
|
" 2>/dev/null) || iso_content=""
|
|
if [[ -n "$iso_content" ]] && ! echo "$iso_content" | grep -q "iso"; then
|
|
error "Storage '${PVE_ISO_STORAGE}' does not accept ISO content (content types: ${iso_content})"
|
|
error "Fix: enable 'iso' content on '${PVE_ISO_STORAGE}' in Proxmox, or use a different iso_storage"
|
|
errors=$((errors + 1))
|
|
fi
|
|
fi
|
|
;;
|
|
esac
|
|
|
|
# 2. Network bridge check (SSH only, warn-level)
|
|
if [[ "$PVE_METHOD" == "ssh" ]]; then
|
|
if ! pve_run "ip link show ${PVE_BRIDGE} 2>/dev/null" &>/dev/null; then
|
|
warn "Network bridge '${PVE_BRIDGE}' not found on Proxmox"
|
|
warn "VMs may fail to get network connectivity"
|
|
warn "Fix: update 'proxmox.bridge' in your config file"
|
|
warnings=$((warnings + 1))
|
|
fi
|
|
fi
|
|
|
|
# 3. ISO availability: check if already on Proxmox or CDN reachable
|
|
if [[ -z "$LOCAL_ISO" ]]; then
|
|
local any_iso_present=false
|
|
local iso_list=""
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
iso_list=$(pve_run "pvesm list ${PVE_ISO_STORAGE} --content iso 2>/dev/null" 2>/dev/null) || iso_list=""
|
|
;;
|
|
api)
|
|
iso_list=$(pve_api GET "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content?content=iso" 2>/dev/null) || iso_list=""
|
|
;;
|
|
esac
|
|
if echo "$iso_list" | grep -q "IncusOS_"; then
|
|
any_iso_present=true
|
|
fi
|
|
if [[ "$any_iso_present" != true ]]; then
|
|
# No ISO on Proxmox -- check if CDN is reachable
|
|
if ! curl -fsSL --connect-timeout 5 -o /dev/null "$CDN_INDEX" 2>/dev/null; then
|
|
error "No IncusOS ISO found on Proxmox and CDN is unreachable: ${CDN_INDEX}"
|
|
error "Fix: provide a local ISO with --iso FILE, or check internet connectivity"
|
|
errors=$((errors + 1))
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# 4. Resource pool check (if configured)
|
|
if [[ -n "$PVE_POOL" ]] && [[ "$DRY_RUN" != true ]]; then
|
|
local pool_ok=false
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
if pve_run "pvesh get /pools/${PVE_POOL} --output-format json 2>/dev/null" &>/dev/null; then
|
|
pool_ok=true
|
|
fi
|
|
;;
|
|
api)
|
|
if pve_api GET "/pools/${PVE_POOL}" &>/dev/null; then
|
|
pool_ok=true
|
|
fi
|
|
;;
|
|
esac
|
|
if [[ "$pool_ok" != true ]]; then
|
|
error "Resource pool '${PVE_POOL}' not found on Proxmox"
|
|
error "Create it with these commands on the Proxmox host:"
|
|
error " pveum pool add ${PVE_POOL} --comment \"IncusOS Lab VMs\""
|
|
error " pveum pool modify ${PVE_POOL} --storage ${PVE_STORAGE},${PVE_ISO_STORAGE}"
|
|
if [[ "$PVE_METHOD" == "api" ]] && [[ -n "$PVE_API_TOKEN_ID" ]]; then
|
|
local api_user
|
|
api_user=$(echo "$PVE_API_TOKEN_ID" | cut -d'!' -f1)
|
|
error " pveum aclmod /pool/${PVE_POOL} -user ${api_user} -role IncusOSDeployer"
|
|
fi
|
|
errors=$((errors + 1))
|
|
else
|
|
detail "Resource pool '${PVE_POOL}' exists"
|
|
fi
|
|
fi
|
|
|
|
# 5. ISO creation tool for SEED_DATA ISO 9660 generation
|
|
if ! command -v genisoimage &>/dev/null && ! command -v mkisofs &>/dev/null; then
|
|
error "An ISO creation tool is required for SEED_DATA generation but not found"
|
|
case "$(uname -s)" in
|
|
Darwin) error "Install with: brew install cdrtools" ;;
|
|
*) error "Install with: sudo apt install genisoimage" ;;
|
|
esac
|
|
errors=$((errors + 1))
|
|
fi
|
|
|
|
# 6. Host resource feasibility (API method only)
|
|
if [[ "$PVE_METHOD" == "api" ]] && [[ "$DRY_RUN" != true ]] && [[ ${#VM_MEMORY[@]} -gt 0 ]]; then
|
|
local total_ram_mib=0
|
|
local total_disk_gib=0
|
|
local total_cores=0
|
|
local ri=0
|
|
while [[ $ri -lt ${#VM_MEMORY[@]} ]]; do
|
|
total_ram_mib=$((total_ram_mib + VM_MEMORY[$ri]))
|
|
total_disk_gib=$((total_disk_gib + VM_DISK[$ri]))
|
|
total_cores=$((total_cores + VM_CORES[$ri]))
|
|
ri=$((ri + 1))
|
|
done
|
|
|
|
local res_node_status
|
|
res_node_status=$(pve_api GET "/nodes/${PVE_NODE}/status" 2>/dev/null) || res_node_status=""
|
|
if [[ -n "$res_node_status" ]]; then
|
|
local resource_lines
|
|
resource_lines=$(python3 -c "
|
|
import json, sys
|
|
data = json.loads('''${res_node_status}''').get('data', {})
|
|
mem = data.get('memory', {})
|
|
mem_total = mem.get('total', 0) / (1024**3)
|
|
mem_used = mem.get('used', 0) / (1024**3)
|
|
mem_free = mem_total - mem_used
|
|
cpu_count = data.get('cpuinfo', {}).get('cpus', 0)
|
|
req_ram = ${total_ram_mib} / 1024
|
|
req_cores = ${total_cores}
|
|
|
|
# RAM check
|
|
if mem_free > 0:
|
|
pct = req_ram / mem_free * 100
|
|
if req_ram > mem_free:
|
|
print(f'error|RAM: {req_ram:.1f} GiB requested, {mem_free:.1f} GiB free -- not enough RAM')
|
|
elif pct > 70:
|
|
print(f'warn|RAM: {req_ram:.1f} GiB requested, {mem_free:.1f} GiB free ({pct:.0f}% of free) -- tight fit')
|
|
else:
|
|
print(f'ok|RAM: {req_ram:.1f} GiB requested, {mem_free:.1f} GiB free ({pct:.0f}% of free)')
|
|
|
|
# CPU (info only)
|
|
print(f'ok|CPU: {req_cores} cores requested, {cpu_count} available')
|
|
" 2>/dev/null) || resource_lines=""
|
|
|
|
local line
|
|
while IFS= read -r line; do
|
|
[[ -z "$line" ]] && continue
|
|
local level="${line%%|*}"
|
|
local msg="${line#*|}"
|
|
case "$level" in
|
|
error)
|
|
error "$msg"
|
|
errors=$((errors + 1))
|
|
;;
|
|
warn)
|
|
warn "$msg"
|
|
warnings=$((warnings + 1))
|
|
;;
|
|
ok)
|
|
detail "$msg"
|
|
;;
|
|
esac
|
|
done <<< "$resource_lines"
|
|
fi
|
|
fi
|
|
|
|
if [[ $errors -gt 0 ]]; then
|
|
error "${errors} pre-flight check(s) failed"
|
|
exit 1
|
|
fi
|
|
|
|
if [[ $warnings -gt 0 ]]; then
|
|
success "Pre-flight checks passed with ${warnings} warning(s)"
|
|
else
|
|
success "Pre-flight checks passed"
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Connectivity check
|
|
# ---------------------------------------------------------------------------
|
|
|
|
check_connectivity() {
|
|
step "Checking Proxmox connectivity (${PVE_METHOD})"
|
|
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
if ! ssh -o BatchMode=yes -o ConnectTimeout=10 \
|
|
"${PVE_SSH_USER}@${PVE_HOST}" "pvesh get /version --output-format json" &>/dev/null; then
|
|
error "Cannot connect to Proxmox host via SSH"
|
|
error "Ensure key-based SSH auth is set up:"
|
|
error " ssh-copy-id ${PVE_SSH_USER}@${PVE_HOST}"
|
|
error "Test with: ssh ${PVE_SSH_USER}@${PVE_HOST} pvesh get /version"
|
|
exit 1
|
|
fi
|
|
local pve_version
|
|
pve_version=$(ssh -o BatchMode=yes "${PVE_SSH_USER}@${PVE_HOST}" \
|
|
"pvesh get /version --output-format json" 2>/dev/null | \
|
|
python3 -c "import json,sys; print(json.load(sys.stdin).get('version','unknown'))" 2>/dev/null) || pve_version="unknown"
|
|
success "Connected to ${PVE_HOST} (Proxmox VE ${pve_version})"
|
|
;;
|
|
api)
|
|
local url="https://${PVE_HOST}:8006/api2/json/version"
|
|
local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET:-}"
|
|
local response
|
|
if ! response=$(curl -fsSk -H "$auth" "$url" 2>/dev/null); then
|
|
error "Cannot connect to Proxmox API at https://${PVE_HOST}:8006"
|
|
error "Check that:"
|
|
error " 1. The host is reachable"
|
|
error " 2. The API token ID is correct: ${PVE_API_TOKEN_ID}"
|
|
error " 3. PROXMOX_TOKEN_SECRET is set"
|
|
exit 1
|
|
fi
|
|
local pve_version
|
|
pve_version=$(echo "$response" | \
|
|
python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('version','unknown'))" 2>/dev/null) || pve_version="unknown"
|
|
success "Connected to ${PVE_HOST} (Proxmox VE ${pve_version})"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# VMID allocation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
allocate_vmids() {
|
|
step "Allocating VMIDs"
|
|
|
|
# Get existing VMs as "vmid name" pairs
|
|
local vm_list
|
|
vm_list=$(pve_list_vms)
|
|
|
|
# Build lookup: existing IDs and name-to-VMID map
|
|
local existing_ids=""
|
|
declare -A existing_name_to_vmid
|
|
if [[ -n "$vm_list" ]]; then
|
|
while IFS=' ' read -r eid ename; do
|
|
[[ -z "$eid" ]] && continue
|
|
existing_ids="${existing_ids} ${eid}"
|
|
if [[ -n "$ename" ]]; then
|
|
existing_name_to_vmid["$ename"]="$eid"
|
|
fi
|
|
done <<< "$vm_list"
|
|
fi
|
|
|
|
local next_id="$DEF_START_VMID"
|
|
local count=${#VM_NAMES[@]}
|
|
local errors=0
|
|
local i=0
|
|
|
|
while [[ $i -lt $count ]]; do
|
|
local name="${VM_NAMES[$i]}"
|
|
|
|
# Skip if filtered
|
|
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
# Check for name collision with existing Proxmox VMs
|
|
if [[ -n "${existing_name_to_vmid[$name]+x}" ]]; then
|
|
local collision_vmid="${existing_name_to_vmid[$name]}"
|
|
if pve_vm_is_managed "$collision_vmid"; then
|
|
warn "VM '${name}' already exists at VMID ${collision_vmid} (managed by ${SCRIPT_NAME})"
|
|
warn "Use --cleanup to redeploy, or remove it manually"
|
|
# Assign the existing VMID so cleanup/skip works correctly
|
|
VM_VMIDS[$i]="$collision_vmid"
|
|
i=$((i + 1))
|
|
continue
|
|
else
|
|
error "VM name '${name}' already exists at VMID ${collision_vmid} but is NOT managed by ${SCRIPT_NAME}"
|
|
error "This VM was not created by this tool and will not be modified"
|
|
error "Fix: rename the VM in your config file, or manually remove VMID ${collision_vmid}:"
|
|
error " ssh ${PVE_SSH_USER}@${PVE_HOST} qm destroy ${collision_vmid} --purge"
|
|
errors=$((errors + 1))
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
fi
|
|
|
|
if [[ -n "${VM_VMIDS[$i]}" ]]; then
|
|
# Explicit VMID
|
|
detail "VM '${name}': VMID ${VM_VMIDS[$i]} (explicit)"
|
|
else
|
|
# Auto-assign, skipping existing
|
|
while echo "$existing_ids" | grep -qw "$next_id" 2>/dev/null; do
|
|
next_id=$((next_id + 1))
|
|
done
|
|
VM_VMIDS[$i]="$next_id"
|
|
detail "VM '${name}': VMID ${next_id} (auto-assigned)"
|
|
next_id=$((next_id + 1))
|
|
fi
|
|
|
|
i=$((i + 1))
|
|
done
|
|
|
|
if [[ $errors -gt 0 ]]; then
|
|
error "${errors} name collision(s) with unmanaged VMs -- cannot continue"
|
|
exit 1
|
|
fi
|
|
|
|
success "VMIDs allocated"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Certificate detection (reused pattern from sibling scripts)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
CERT_FILE_RESOLVED=""
|
|
|
|
detect_client_cert() {
|
|
if [[ "$NO_CERT" == true ]]; then
|
|
info "Skipping certificate injection (--no-cert)"
|
|
return
|
|
fi
|
|
|
|
if [[ -n "$CERT_FILE" ]]; then
|
|
if [[ ! -f "$CERT_FILE" ]]; then
|
|
error "Certificate file not found: ${CERT_FILE}"
|
|
exit 1
|
|
fi
|
|
if ! grep -q "BEGIN CERTIFICATE" "$CERT_FILE"; then
|
|
error "File does not appear to be a PEM certificate: ${CERT_FILE}"
|
|
exit 1
|
|
fi
|
|
CERT_FILE_RESOLVED="$CERT_FILE"
|
|
success "Certificate: ${CERT_FILE_RESOLVED}"
|
|
return
|
|
fi
|
|
|
|
step "Auto-detecting client certificate"
|
|
|
|
local cert_paths=(
|
|
"${HOME}/.config/incus/client.crt"
|
|
"${HOME}/snap/incus/common/config/client.crt"
|
|
"${HOME}/.config/lxc/client.crt"
|
|
)
|
|
|
|
for path in "${cert_paths[@]}"; do
|
|
if [[ -f "$path" ]] && grep -q "BEGIN CERTIFICATE" "$path"; then
|
|
CERT_FILE_RESOLVED="$path"
|
|
success "Certificate: ${CERT_FILE_RESOLVED}"
|
|
return
|
|
fi
|
|
done
|
|
|
|
# Try CLI fallback
|
|
if command -v incus &>/dev/null; then
|
|
local tmpfile="${WORKDIR}/client.crt"
|
|
if incus remote get-client-certificate > "$tmpfile" 2>/dev/null; then
|
|
CERT_FILE_RESOLVED="$tmpfile"
|
|
success "Certificate obtained via Incus CLI"
|
|
return
|
|
fi
|
|
fi
|
|
|
|
# Check if any VM is ops-center (which requires a cert)
|
|
local needs_cert=false
|
|
for app in "${VM_APPS[@]}"; do
|
|
if [[ "$app" == "operations-center" ]]; then
|
|
needs_cert=true
|
|
break
|
|
fi
|
|
done
|
|
|
|
if [[ "$needs_cert" == true ]]; then
|
|
error "No client certificate found"
|
|
error "Operations Center requires at least one trusted certificate for access"
|
|
error ""
|
|
error "Options:"
|
|
error " --cert FILE Provide a certificate file directly"
|
|
error " Generate one: incus remote list (creates ~/.config/incus/client.crt)"
|
|
error " --no-cert Skip certificate injection (you will be locked out!)"
|
|
exit 1
|
|
fi
|
|
|
|
warn "No client certificate found -- VMs will require manual trust setup"
|
|
warn "Use --cert FILE to provide a certificate, or --no-cert to suppress this warning"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CDN operations (reused from incusos-iso)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
LATEST_VERSION=""
|
|
|
|
fetch_latest_version() {
|
|
local channel="${DEF_CHANNEL:-stable}"
|
|
|
|
step "Fetching latest IncusOS version (channel: ${channel})"
|
|
|
|
local index_file="${WORKDIR}/cdn-index.json"
|
|
|
|
if ! curl -fsSL "$CDN_INDEX" -o "$index_file" 2>/dev/null; then
|
|
error "Failed to fetch CDN index from ${CDN_INDEX}"
|
|
error "Check your internet connection and try again"
|
|
exit 1
|
|
fi
|
|
|
|
if command -v jq &>/dev/null; then
|
|
LATEST_VERSION=$(jq -r \
|
|
--arg ch "$channel" \
|
|
'[.updates[] | select(.channels[] == $ch)] | sort_by(.version) | last | .version // empty' \
|
|
"$index_file")
|
|
else
|
|
LATEST_VERSION=$(python3 -c "
|
|
import json
|
|
data = json.load(open('${index_file}'))
|
|
versions = [u['version'] for u in data.get('updates', []) if '${channel}' in u.get('channels', [])]
|
|
if versions:
|
|
print(sorted(versions)[-1])
|
|
" 2>/dev/null)
|
|
fi
|
|
|
|
if [[ -z "$LATEST_VERSION" ]]; then
|
|
error "No version found for channel '${channel}'"
|
|
exit 1
|
|
fi
|
|
|
|
success "Latest IncusOS version: ${LATEST_VERSION}"
|
|
}
|
|
|
|
download_iso() {
|
|
local version="$1"
|
|
local output_path="$2"
|
|
local iso_filename
|
|
iso_filename=$(basename "$output_path")
|
|
|
|
# Check persistent cache first
|
|
local cached_iso="${CACHEDIR}/${iso_filename}"
|
|
if [[ -f "$cached_iso" ]]; then
|
|
success "ISO found in local cache: ${cached_iso}"
|
|
cp "$cached_iso" "$output_path"
|
|
return 0
|
|
fi
|
|
|
|
local url="${CDN_BASE}/${version}/x86_64/IncusOS_${version}.iso.gz"
|
|
local gz_path="${cached_iso}.gz"
|
|
|
|
step "Downloading IncusOS ISO"
|
|
detail "URL: ${url}"
|
|
detail "Cache directory: ${CACHEDIR}"
|
|
|
|
if ! curl -fL --progress-bar "$url" -o "$gz_path"; then
|
|
error "Failed to download ISO from ${url}"
|
|
rm -f "$gz_path"
|
|
exit 1
|
|
fi
|
|
|
|
step "Decompressing ISO"
|
|
if ! gzip -d "$gz_path"; then
|
|
local avail
|
|
avail=$(df -h --output=avail "$(dirname "$gz_path")" 2>/dev/null | tail -1 | tr -d ' ') || avail="unknown"
|
|
error "Failed to decompress ISO (${avail} available on disk)"
|
|
error "The decompressed ISO requires several GiB of disk space"
|
|
error "Free up space, or set TMPDIR to a larger filesystem:"
|
|
error " TMPDIR=/path/with/space ${SCRIPT_NAME} ${CONFIG_FILE}"
|
|
rm -f "$gz_path"
|
|
exit 1
|
|
fi
|
|
|
|
# Copy from cache to workdir
|
|
cp "$cached_iso" "$output_path"
|
|
success "ISO ready: ${output_path} (cached for future runs)"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Deployment plan summary
|
|
# ---------------------------------------------------------------------------
|
|
|
|
print_plan() {
|
|
local count=${#VM_NAMES[@]}
|
|
|
|
echo ""
|
|
echo -e "${BOLD}Deployment Plan${RESET}"
|
|
echo -e " Proxmox host: ${CYAN}${PVE_HOST}${RESET} (${PVE_METHOD})"
|
|
echo -e " Node: ${PVE_NODE}"
|
|
echo -e " Storage: ${PVE_STORAGE} (disk), ${PVE_ISO_STORAGE} (iso)"
|
|
echo -e " Bridge: ${PVE_BRIDGE}"
|
|
if [[ -n "$PVE_POOL" ]]; then
|
|
echo -e " Pool: ${CYAN}${PVE_POOL}${RESET}"
|
|
fi
|
|
if [[ -n "$CERT_FILE_RESOLVED" ]]; then
|
|
echo -e " Certificate: ${GREEN}${CERT_FILE_RESOLVED}${RESET}"
|
|
elif [[ "$NO_CERT" == true ]]; then
|
|
echo -e " Certificate: ${DIM}none (--no-cert)${RESET}"
|
|
else
|
|
echo -e " Certificate: ${YELLOW}not found${RESET}"
|
|
fi
|
|
echo ""
|
|
echo -e " ${BOLD}VMs (${count}):${RESET}"
|
|
|
|
local i=0
|
|
while [[ $i -lt $count ]]; do
|
|
local name="${VM_NAMES[$i]}"
|
|
local skip=""
|
|
|
|
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
|
|
skip=" ${DIM}(skipped)${RESET}"
|
|
fi
|
|
|
|
echo -e " ${CYAN}${name}${RESET}${skip}"
|
|
echo -e " VMID: ${VM_VMIDS[$i]} App: ${VM_APPS[$i]} Defaults: ${VM_APPLY_DEFAULTS[$i]}"
|
|
echo -e " CPU: ${VM_CORES[$i]} cores RAM: ${VM_MEMORY[$i]} MiB Disk: ${VM_DISK[$i]} GiB"
|
|
|
|
if [[ -n "${VM_CLUSTERS[$i]}" ]]; then
|
|
echo -e " Cluster: ${VM_CLUSTERS[$i]} (${VM_CLUSTER_ROLES[$i]})"
|
|
fi
|
|
|
|
i=$((i + 1))
|
|
done
|
|
echo ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Confirmation prompt
|
|
# ---------------------------------------------------------------------------
|
|
|
|
confirm() {
|
|
local message="$1"
|
|
|
|
if [[ "$YES" == true ]] || [[ "$DRY_RUN" == true ]]; then
|
|
return 0
|
|
fi
|
|
|
|
echo -n -e "${BOLD}${message} [y/N] ${RESET}"
|
|
local reply
|
|
read -r reply
|
|
if [[ ! "$reply" =~ ^[Yy]$ ]]; then
|
|
info "Aborted"
|
|
exit 0
|
|
fi
|
|
}
|
|
|
|
# Like confirm(), but returns 0/1 instead of exiting on "no"
|
|
confirm_yn() {
|
|
local message="$1"
|
|
|
|
if [[ "$YES" == true ]]; then
|
|
return 0
|
|
fi
|
|
|
|
echo -n -e "${BOLD}${message} [y/N] ${RESET}"
|
|
local reply
|
|
read -r reply
|
|
[[ "$reply" =~ ^[Yy]$ ]]
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 1: Validate
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_validate() {
|
|
step "Phase 1: Validate"
|
|
echo ""
|
|
|
|
# Load proxmox.yaml base values (before lab config overlay)
|
|
if find_proxmox_config; then
|
|
load_proxmox_config "$PROXMOX_CONFIG_RESOLVED"
|
|
fi
|
|
|
|
parse_config
|
|
load_config
|
|
validate_config
|
|
check_connectivity
|
|
allocate_vmids
|
|
preflight_checks
|
|
detect_client_cert
|
|
print_plan
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 2: Prepare (download ISO + generate seeds)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
ISO_FILENAME=""
|
|
|
|
phase_prepare() {
|
|
step "Phase 2: Prepare"
|
|
echo ""
|
|
|
|
# Determine ISO
|
|
if [[ -n "$LOCAL_ISO" ]]; then
|
|
ISO_FILENAME=$(basename "$LOCAL_ISO")
|
|
info "Using local ISO: ${LOCAL_ISO}"
|
|
if [[ "$DRY_RUN" != true ]]; then
|
|
cp "$LOCAL_ISO" "${WORKDIR}/${ISO_FILENAME}"
|
|
fi
|
|
elif [[ "$DRY_RUN" == true ]]; then
|
|
ISO_FILENAME="IncusOS_latest.iso"
|
|
info "[dry-run] Would fetch latest version from CDN and download ISO"
|
|
else
|
|
# Check if ISO already exists on Proxmox
|
|
fetch_latest_version
|
|
ISO_FILENAME="IncusOS_${LATEST_VERSION}.iso"
|
|
|
|
if pve_iso_exists "$ISO_FILENAME"; then
|
|
success "ISO already on Proxmox: ${ISO_FILENAME} (reusing from previous deployment)"
|
|
else
|
|
download_iso "$LATEST_VERSION" "${WORKDIR}/${ISO_FILENAME}"
|
|
fi
|
|
fi
|
|
|
|
# Generate per-VM seed images
|
|
step "Generating seed images"
|
|
|
|
local count=${#VM_NAMES[@]}
|
|
local i=0
|
|
while [[ $i -lt $count ]]; do
|
|
local name="${VM_NAMES[$i]}"
|
|
|
|
# Skip if filtered
|
|
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
local app="${VM_APPS[$i]}"
|
|
local apply_defaults="${VM_APPLY_DEFAULTS[$i]}"
|
|
local seed_output="${WORKDIR}/seed-${name}.iso"
|
|
|
|
detail "Generating seed for '${name}' (${app}, defaults=${apply_defaults})"
|
|
|
|
# Build incusos-seed command
|
|
local seed_cmd=("$SEED_TOOL")
|
|
seed_cmd+=(--format iso)
|
|
seed_cmd+=(--app "$app")
|
|
seed_cmd+=(--output "$seed_output")
|
|
seed_cmd+=(--force-install)
|
|
seed_cmd+=(--force-reboot)
|
|
seed_cmd+=(--hostname "$name")
|
|
seed_cmd+=(--quiet)
|
|
|
|
if [[ "$apply_defaults" == "true" ]]; then
|
|
seed_cmd+=(--defaults)
|
|
fi
|
|
|
|
if [[ -n "$CERT_FILE_RESOLVED" ]]; then
|
|
seed_cmd+=(--cert "$CERT_FILE_RESOLVED")
|
|
elif [[ "$NO_CERT" == true ]]; then
|
|
seed_cmd+=(--no-cert)
|
|
fi
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] ${seed_cmd[*]}"
|
|
else
|
|
if ! "${seed_cmd[@]}"; then
|
|
error "Failed to generate seed for VM '${name}'"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
i=$((i + 1))
|
|
done
|
|
|
|
success "Seed images generated"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 3: Upload
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_upload() {
|
|
step "Phase 3: Upload"
|
|
echo ""
|
|
|
|
# Upload ISO (if we downloaded it)
|
|
local iso_path="${WORKDIR}/${ISO_FILENAME}"
|
|
if [[ -f "$iso_path" ]]; then
|
|
if pve_iso_exists "$ISO_FILENAME"; then
|
|
success "ISO already on Proxmox: ${ISO_FILENAME}"
|
|
else
|
|
step "Uploading ISO: ${ISO_FILENAME}"
|
|
if ! pve_upload "$iso_path" "$ISO_FILENAME"; then
|
|
error "Failed to upload ISO to Proxmox"
|
|
exit 1
|
|
fi
|
|
success "ISO uploaded: ${ISO_FILENAME}"
|
|
fi
|
|
else
|
|
detail "ISO already on Proxmox, skipping upload"
|
|
fi
|
|
|
|
# Upload seed images
|
|
local count=${#VM_NAMES[@]}
|
|
local i=0
|
|
while [[ $i -lt $count ]]; do
|
|
local name="${VM_NAMES[$i]}"
|
|
|
|
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
local seed_file="seed-${name}.iso"
|
|
local seed_path="${WORKDIR}/${seed_file}"
|
|
|
|
if [[ -f "$seed_path" ]] || [[ "$DRY_RUN" == true ]]; then
|
|
step "Uploading seed: ${seed_file}"
|
|
if ! pve_upload "$seed_path" "$seed_file"; then
|
|
error "Failed to upload seed for VM '${name}'"
|
|
exit 1
|
|
fi
|
|
success "Seed uploaded: ${seed_file}"
|
|
fi
|
|
|
|
i=$((i + 1))
|
|
done
|
|
|
|
success "All files uploaded"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 4: Create VMs
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_create() {
|
|
step "Phase 4: Create VMs"
|
|
echo ""
|
|
|
|
local count=${#VM_NAMES[@]}
|
|
local i=0
|
|
while [[ $i -lt $count ]]; do
|
|
local name="${VM_NAMES[$i]}"
|
|
|
|
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
local vmid="${VM_VMIDS[$i]}"
|
|
local cores="${VM_CORES[$i]}"
|
|
local memory="${VM_MEMORY[$i]}"
|
|
local disk="${VM_DISK[$i]}"
|
|
local seed_file="seed-${name}.iso"
|
|
|
|
step "Creating VM '${name}' (VMID ${vmid})"
|
|
|
|
# Check if VM already exists
|
|
if pve_vm_exists "$vmid"; then
|
|
if pve_vm_is_managed "$vmid"; then
|
|
warn "VM ${vmid} ('${name}') already exists (managed) -- skipping"
|
|
warn "Use --cleanup to redeploy"
|
|
else
|
|
warn "VM ${vmid} ('${name}') already exists (NOT managed by ${SCRIPT_NAME}) -- skipping"
|
|
warn "Choose a different VMID or remove it manually:"
|
|
warn " ssh ${PVE_SSH_USER}@${PVE_HOST} qm destroy ${vmid} --purge"
|
|
fi
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
detail "cores=${cores} memory=${memory}MiB disk=${disk}GiB"
|
|
detail "bios=ovmf machine=q35 cpu=host scsihw=virtio-scsi-pci"
|
|
detail "efidisk: pre-enrolled-keys=0, tpm: v2.0, balloon: 0"
|
|
detail "iso=${ISO_FILENAME} seed=${seed_file}"
|
|
|
|
if ! pve_create_vm "$vmid" "$name" "$cores" "$memory" "$disk" \
|
|
"$ISO_FILENAME" "$seed_file"; then
|
|
error "Failed to create VM '${name}' (VMID ${vmid})"
|
|
exit 1
|
|
fi
|
|
|
|
success "VM '${name}' created (VMID ${vmid})"
|
|
|
|
i=$((i + 1))
|
|
done
|
|
|
|
success "All VMs created"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Remote registration
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Register an incus remote for a deployed VM.
|
|
# Handles stale remotes: if a remote exists but points to a different IP,
|
|
# it is removed and re-added with the correct IP (DHCP may reassign addresses
|
|
# between deploys).
|
|
# Usage: register_remote <name> <ip>
|
|
register_remote() {
|
|
local name="$1"
|
|
local ip="$2"
|
|
|
|
# Only register if incus client is available and a cert was injected
|
|
if ! command -v incus &>/dev/null; then
|
|
return
|
|
fi
|
|
if [[ -z "$CERT_FILE_RESOLVED" ]]; then
|
|
return
|
|
fi
|
|
|
|
# Check if remote already exists
|
|
local existing_url=""
|
|
existing_url=$(incus remote list --format csv 2>/dev/null | \
|
|
awk -F',' -v name="$name" '$1 == name {print $2; exit}') || true
|
|
|
|
if [[ -n "$existing_url" ]]; then
|
|
# Remote exists -- check if the IP matches
|
|
if echo "$existing_url" | grep -q "${ip}"; then
|
|
detail "Remote '${name}' already configured at ${ip}"
|
|
return
|
|
fi
|
|
|
|
# Stale remote -- IP changed (DHCP reassignment between deploys)
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] Would update stale remote '${name}': ${existing_url} -> ${ip}"
|
|
return
|
|
fi
|
|
|
|
warn "Remote '${name}' points to ${existing_url} but VM is at ${ip} -- updating"
|
|
# Switch away first -- can't remove the current/default remote
|
|
local current_remote=""
|
|
current_remote=$(incus remote get-default 2>/dev/null) || true
|
|
if [[ "$current_remote" == "$name" ]]; then
|
|
incus remote switch local 2>/dev/null || true
|
|
fi
|
|
incus remote remove "$name" 2>/dev/null || true
|
|
fi
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] Would add incus remote '${name}' at ${ip}"
|
|
return
|
|
fi
|
|
|
|
# Wait for port 8443 to become reachable (up to 30s)
|
|
local attempt=0
|
|
while [[ $attempt -lt 6 ]]; do
|
|
if curl -sk --connect-timeout 3 "https://${ip}:8443" &>/dev/null; then
|
|
break
|
|
fi
|
|
sleep 5
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
if incus remote add "$name" "$ip" --accept-certificate --quiet 2>/dev/null; then
|
|
success "Remote '${name}' added (${ip})"
|
|
else
|
|
warn "Could not add remote '${name}' at ${ip} -- add manually:"
|
|
warn " incus remote add ${name} ${ip} --accept-certificate"
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 5: Install
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_install() {
|
|
step "Phase 5: Install"
|
|
echo ""
|
|
|
|
local count=${#VM_NAMES[@]}
|
|
local install_failures=0
|
|
local i=0
|
|
while [[ $i -lt $count ]]; do
|
|
local name="${VM_NAMES[$i]}"
|
|
|
|
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
local vmid="${VM_VMIDS[$i]}"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
step "Starting VM '${name}' (VMID ${vmid})"
|
|
info "[dry-run] Would start VM, swap boot order, wait for install, clean up"
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
# Idempotency: check current VM state before acting
|
|
if pve_vm_exists "$vmid"; then
|
|
local cur_status
|
|
cur_status=$(pve_vm_status "$vmid") || cur_status="unknown"
|
|
local has_media=true
|
|
pve_vm_has_install_media "$vmid" || has_media=false
|
|
|
|
if [[ "$cur_status" == "running" ]] && [[ "$has_media" == false ]]; then
|
|
# Already installed and running — detect IP, register remote, skip
|
|
success "VM '${name}' (VMID ${vmid}) is already installed and running"
|
|
local ip
|
|
ip=$(pve_vm_ip "$vmid") || ip=""
|
|
if [[ -n "$ip" ]]; then
|
|
detail "IP: ${ip}"
|
|
register_remote "$name" "$ip"
|
|
fi
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
if [[ "$cur_status" == "stopped" ]] && [[ "$has_media" == false ]]; then
|
|
# Installed but stopped — start from disk
|
|
step "VM '${name}' is installed but stopped — starting from disk"
|
|
if pve_start_vm "$vmid"; then
|
|
sleep 30
|
|
local ip
|
|
ip=$(pve_vm_ip "$vmid") || ip=""
|
|
if [[ -n "$ip" ]]; then
|
|
success "VM '${name}' started at ${ip}"
|
|
register_remote "$name" "$ip"
|
|
else
|
|
success "VM '${name}' started from disk"
|
|
fi
|
|
else
|
|
error "Failed to start VM '${name}' from disk"
|
|
fi
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
if [[ "$cur_status" == "running" ]] && [[ "$has_media" == true ]]; then
|
|
# Mid-install or stuck — fall through to blockstat monitoring
|
|
info "VM '${name}' is running with install media — monitoring installation"
|
|
fi
|
|
|
|
# stopped + has_media → needs install, fall through
|
|
fi
|
|
|
|
step "Starting VM '${name}' (VMID ${vmid})"
|
|
|
|
# 1. Start VM (boots from ISO, reads SEED_DATA, installs)
|
|
if ! pve_start_vm "$vmid"; then
|
|
error "Failed to start VM '${name}' (VMID ${vmid})"
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
success "VM '${name}' started"
|
|
|
|
# 2. Wait for installation to complete by monitoring disk writes
|
|
#
|
|
# Install flow: boot ISO -> read SEED_DATA -> install to disk -> reboot
|
|
# (force_reboot in seed). A guest-level reboot does NOT reset the QEMU
|
|
# VM uptime, so we cannot detect reboot that way. Instead we monitor
|
|
# scsi0 write bytes via blockstat:
|
|
# - During install: wr_bytes increases steadily
|
|
# - After install + reboot: wr_bytes stops (VM is idle at post-reboot
|
|
# screen or "install media detected" error)
|
|
#
|
|
# When wr_bytes has been stable for STABLE_COUNT consecutive polls,
|
|
# installation is complete. We then stop the VM, remove install media,
|
|
# and start cleanly from disk.
|
|
step "Waiting for '${name}' to complete installation"
|
|
info "Monitoring disk write activity (polling every 5s, timeout: 10 minutes)"
|
|
|
|
local timeout=600
|
|
local interval=5
|
|
local elapsed=0
|
|
local install_done=false
|
|
local prev_wr_bytes=0
|
|
local writes_started=false
|
|
local stable_count=0
|
|
local STABLE_THRESHOLD=3 # 3 consecutive polls with no change = 15s idle
|
|
|
|
while [[ $elapsed -lt $timeout ]]; do
|
|
sleep "$interval"
|
|
elapsed=$((elapsed + interval))
|
|
|
|
local status
|
|
status=$(pve_vm_status "$vmid") || status="unknown"
|
|
|
|
case "$status" in
|
|
stopped)
|
|
detail "VM stopped [${elapsed}s]"
|
|
install_done=true
|
|
break
|
|
;;
|
|
running)
|
|
local wr_bytes
|
|
wr_bytes=$(pve_vm_disk_writes "$vmid") || wr_bytes=0
|
|
|
|
if [[ $wr_bytes -gt $prev_wr_bytes ]]; then
|
|
# Disk writes are happening -- install in progress
|
|
writes_started=true
|
|
stable_count=0
|
|
local wr_mb=$(( wr_bytes / 1048576 ))
|
|
# Show progress every 15s
|
|
if [[ $((elapsed % 15)) -eq 0 ]]; then
|
|
info "Installing... (${wr_mb} MiB written) [${elapsed}s]"
|
|
fi
|
|
elif [[ "$writes_started" == true ]]; then
|
|
# Writes have stopped after being active
|
|
stable_count=$((stable_count + 1))
|
|
if [[ $stable_count -ge $STABLE_THRESHOLD ]]; then
|
|
local wr_mb=$(( wr_bytes / 1048576 ))
|
|
info "Disk activity stopped (${wr_mb} MiB written total) [${elapsed}s]"
|
|
install_done=true
|
|
break
|
|
fi
|
|
fi
|
|
|
|
prev_wr_bytes=$wr_bytes
|
|
;;
|
|
*)
|
|
detail "VM status: ${status} [${elapsed}s]"
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [[ "$install_done" != true ]]; then
|
|
warn "VM '${name}' did not complete installation within ${timeout}s"
|
|
warn "Check the VM console on Proxmox for details"
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
# 3. Stop the VM (it's either at "remove install media" prompt or
|
|
# post-reboot "install media detected" error -- either way idle).
|
|
local cur_status
|
|
cur_status=$(pve_vm_status "$vmid") || cur_status="unknown"
|
|
if [[ "$cur_status" == "running" ]]; then
|
|
info "Stopping VM to remove install media..."
|
|
pve_stop_vm "$vmid" || true
|
|
sleep 5
|
|
fi
|
|
|
|
# 4. Remove install media and set clean boot order
|
|
step "Removing install media from '${name}'"
|
|
pve_set_vm "$vmid" "--delete ide2 --delete ide3 --boot order=scsi0" || true
|
|
success "Install media detached, boot order set to disk-only"
|
|
|
|
# 5. Start VM from disk
|
|
step "Starting '${name}' from disk"
|
|
if ! pve_start_vm "$vmid"; then
|
|
error "Failed to start VM '${name}' from disk"
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
# 5. Wait for services to come up and verify health
|
|
info "Waiting for services to start..."
|
|
sleep 30
|
|
|
|
# 6. Detect IP and check port 8443
|
|
local ip
|
|
ip=$(pve_vm_ip "$vmid") || ip=""
|
|
|
|
local healthy=false
|
|
if [[ -n "$ip" ]]; then
|
|
# Poll port 8443 for up to 60s (the 30s sleep above + 60s = 90s total)
|
|
local poll=0
|
|
while [[ $poll -lt 12 ]]; do
|
|
if curl -sk --connect-timeout 3 "https://${ip}:8443" &>/dev/null; then
|
|
healthy=true
|
|
break
|
|
fi
|
|
sleep 5
|
|
poll=$((poll + 1))
|
|
done
|
|
fi
|
|
|
|
echo ""
|
|
if [[ "$healthy" == true ]]; then
|
|
success "VM '${name}' installed and running at ${ip}"
|
|
register_remote "$name" "$ip"
|
|
else
|
|
# Retry once: IncusOS has an intermittent "invalid crontab expression"
|
|
# bug (~50% first-boot hit rate) where registerJobs() fails during
|
|
# startup(). Most likely a race condition between the REST API and
|
|
# startup sequence, or a gocron library edge case. A stop+start
|
|
# cycle resolves it because the transient condition clears on retry.
|
|
if [[ -n "$ip" ]]; then
|
|
warn "VM '${name}': port 8443 not reachable at ${ip} — retrying (stop+start)"
|
|
else
|
|
warn "VM '${name}': could not detect IP — retrying (stop+start)"
|
|
fi
|
|
info "Stopping VM for retry..."
|
|
pve_stop_vm "$vmid" || true
|
|
sleep 10
|
|
info "Starting VM (retry)..."
|
|
pve_start_vm "$vmid" || true
|
|
sleep 30
|
|
|
|
# Re-detect IP (may have changed after restart)
|
|
ip=$(pve_vm_ip "$vmid") || ip=""
|
|
|
|
local retry_healthy=false
|
|
if [[ -n "$ip" ]]; then
|
|
local retry_poll=0
|
|
while [[ $retry_poll -lt 12 ]]; do
|
|
if curl -sk --connect-timeout 3 "https://${ip}:8443" &>/dev/null; then
|
|
retry_healthy=true
|
|
break
|
|
fi
|
|
sleep 5
|
|
retry_poll=$((retry_poll + 1))
|
|
done
|
|
fi
|
|
|
|
if [[ "$retry_healthy" == true ]]; then
|
|
success "VM '${name}' recovered after retry — running at ${ip}"
|
|
register_remote "$name" "$ip"
|
|
elif [[ -n "$ip" ]]; then
|
|
warn "VM '${name}' installed but port 8443 not reachable at ${ip} (even after retry)"
|
|
warn "Likely IncusOS boot failure (check Proxmox console for errors)"
|
|
warn "Redeploy this VM with: ${SCRIPT_NAME} --cleanup --yes ${CONFIG_FILE} && ${SCRIPT_NAME} --yes ${CONFIG_FILE}"
|
|
install_failures=$((install_failures + 1))
|
|
else
|
|
warn "VM '${name}' installed but could not detect IP (even after retry)"
|
|
warn "Check Proxmox console for status"
|
|
install_failures=$((install_failures + 1))
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
i=$((i + 1))
|
|
done
|
|
|
|
if [[ $install_failures -gt 0 ]]; then
|
|
warn "Installation phase complete: ${install_failures} VM(s) failed health check"
|
|
warn "Failed VMs are likely stuck in an IncusOS boot loop"
|
|
warn "Re-run deployment to redeploy, or use --cleanup to start fresh"
|
|
else
|
|
success "Installation phase complete"
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Lab lifecycle (--lab-up, --lab-down)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_lab_down() {
|
|
step "Stopping lab VMs"
|
|
echo ""
|
|
|
|
local count=${#VM_NAMES[@]}
|
|
local stopped=0
|
|
local skipped=0
|
|
local i=0
|
|
while [[ $i -lt $count ]]; do
|
|
local name="${VM_NAMES[$i]}"
|
|
local vmid="${VM_VMIDS[$i]}"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "Would stop ${name} (VMID ${vmid})"
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
if ! pve_vm_exists "$vmid"; then
|
|
warn "${name} (VMID ${vmid}): not found -- skipping"
|
|
skipped=$((skipped + 1))
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
local status
|
|
status=$(pve_vm_status "$vmid") || status="unknown"
|
|
|
|
if [[ "$status" == "running" ]]; then
|
|
info "Stopping ${name} (VMID ${vmid})..."
|
|
if pve_stop_vm "$vmid"; then
|
|
success "${name} stopped"
|
|
stopped=$((stopped + 1))
|
|
else
|
|
error "Failed to stop ${name}"
|
|
fi
|
|
else
|
|
detail "${name}: already ${status}"
|
|
skipped=$((skipped + 1))
|
|
fi
|
|
|
|
i=$((i + 1))
|
|
done
|
|
|
|
echo ""
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "Dry run -- no VMs were stopped"
|
|
else
|
|
success "Lab down: ${stopped} stopped, ${skipped} skipped"
|
|
fi
|
|
}
|
|
|
|
phase_lab_up() {
|
|
step "Starting lab VMs"
|
|
echo ""
|
|
|
|
local count=${#VM_NAMES[@]}
|
|
local started=0
|
|
local skipped=0
|
|
local missing=0
|
|
|
|
# First pass: check how many VMs exist
|
|
if [[ "$DRY_RUN" != true ]]; then
|
|
local i=0
|
|
while [[ $i -lt $count ]]; do
|
|
if ! pve_vm_exists "${VM_VMIDS[$i]}"; then
|
|
missing=$((missing + 1))
|
|
fi
|
|
i=$((i + 1))
|
|
done
|
|
|
|
# If ALL VMs are missing, offer to deploy
|
|
if [[ $missing -eq $count ]]; then
|
|
echo ""
|
|
warn "No VMs found -- lab has not been deployed yet"
|
|
if confirm_yn "Deploy lab now?"; then
|
|
echo ""
|
|
phase_prepare
|
|
echo ""
|
|
phase_upload
|
|
echo ""
|
|
phase_create
|
|
echo ""
|
|
phase_install
|
|
echo ""
|
|
phase_post_deploy
|
|
return
|
|
else
|
|
info "Aborted"
|
|
return
|
|
fi
|
|
fi
|
|
|
|
# If SOME VMs are missing, warn but don't offer auto-deploy
|
|
if [[ $missing -gt 0 ]]; then
|
|
warn "${missing} of ${count} VMs not found -- partial state detected"
|
|
warn "Use --cleanup then redeploy, or deploy missing VMs manually"
|
|
fi
|
|
fi
|
|
|
|
# Second pass: start existing VMs
|
|
local i=0
|
|
while [[ $i -lt $count ]]; do
|
|
local name="${VM_NAMES[$i]}"
|
|
local vmid="${VM_VMIDS[$i]}"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "Would start ${name} (VMID ${vmid})"
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
if ! pve_vm_exists "$vmid"; then
|
|
warn "${name} (VMID ${vmid}): not found -- skipping"
|
|
skipped=$((skipped + 1))
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
local status
|
|
status=$(pve_vm_status "$vmid") || status="unknown"
|
|
|
|
if [[ "$status" == "stopped" ]]; then
|
|
# Safety check: refuse to start VMs with install media still attached
|
|
if pve_vm_has_install_media "$vmid"; then
|
|
warn "${name}: install media still attached -- skipping (run full deploy instead)"
|
|
skipped=$((skipped + 1))
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
info "Starting ${name} (VMID ${vmid})..."
|
|
if pve_start_vm "$vmid"; then
|
|
success "${name} started"
|
|
started=$((started + 1))
|
|
else
|
|
error "Failed to start ${name}"
|
|
fi
|
|
else
|
|
detail "${name}: already ${status}"
|
|
skipped=$((skipped + 1))
|
|
fi
|
|
|
|
i=$((i + 1))
|
|
done
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
echo ""
|
|
info "Dry run -- no VMs were started"
|
|
return
|
|
fi
|
|
|
|
echo ""
|
|
success "Lab up: ${started} started, ${skipped} skipped"
|
|
|
|
if [[ $started -gt 0 ]]; then
|
|
info "Waiting 30s for services to start..."
|
|
sleep 30
|
|
echo ""
|
|
phase_status
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Resource and lab inventory (--resources, --labs)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
do_resources() {
|
|
echo -e "${BOLD}${SCRIPT_NAME} resources${RESET}"
|
|
echo ""
|
|
|
|
echo -e " ${BOLD}Proxmox Host:${RESET} ${PVE_HOST} (node: ${PVE_NODE})"
|
|
echo ""
|
|
|
|
# Host resources (API only -- SSH would need parsing pvesh output)
|
|
if [[ "$PVE_METHOD" != "api" ]]; then
|
|
warn "Resource details require API method (current: ${PVE_METHOD})"
|
|
warn "Use --method api or set method: api in proxmox.yaml"
|
|
return
|
|
fi
|
|
|
|
# Host memory and CPU
|
|
local node_status
|
|
node_status=$(pve_api GET "/nodes/${PVE_NODE}/status" 2>/dev/null) || node_status=""
|
|
if [[ -n "$node_status" ]]; then
|
|
python3 -c "
|
|
import json, sys
|
|
data = json.loads('''${node_status}''').get('data', {})
|
|
mem = data.get('memory', {})
|
|
cpu_count = data.get('cpuinfo', {}).get('cpus', '?')
|
|
cpu_model = data.get('cpuinfo', {}).get('model', 'unknown')
|
|
mem_used = mem.get('used', 0) / (1024**3)
|
|
mem_total = mem.get('total', 0) / (1024**3)
|
|
mem_free = mem_total - mem_used
|
|
mem_pct = (mem_used / mem_total * 100) if mem_total > 0 else 0
|
|
print(f' RAM: {mem_used:.1f} GiB / {mem_total:.1f} GiB used ({mem_pct:.0f}%), {mem_free:.1f} GiB free')
|
|
print(f' CPUs: {cpu_count} cores ({cpu_model})')
|
|
uptime_s = data.get('uptime', 0)
|
|
days = uptime_s // 86400
|
|
hours = (uptime_s % 86400) // 3600
|
|
print(f' Uptime: {days}d {hours}h')
|
|
" 2>/dev/null || warn "Failed to parse node status"
|
|
fi
|
|
|
|
# Storage
|
|
echo ""
|
|
echo -e " ${BOLD}Storage:${RESET}"
|
|
local storage_list
|
|
storage_list=$(pve_api GET "/nodes/${PVE_NODE}/storage" 2>/dev/null) || storage_list=""
|
|
if [[ -n "$storage_list" ]]; then
|
|
python3 -c "
|
|
import json, sys
|
|
data = json.loads('''${storage_list}''').get('data', [])
|
|
thin_types = {'zfspool', 'lvmthin'}
|
|
for s in sorted(data, key=lambda x: x.get('storage', '')):
|
|
name = s.get('storage', '?')
|
|
total = s.get('total', 0) / (1024**3)
|
|
used = s.get('used', 0) / (1024**3)
|
|
avail = s.get('avail', 0) / (1024**3)
|
|
pct = (used / total * 100) if total > 0 else 0
|
|
stype = s.get('type', '?')
|
|
content = s.get('content', '?')
|
|
thin = ' [thin]' if stype in thin_types else ''
|
|
if total > 0:
|
|
print(f' {name:<16} {stype:<12} {used:.1f}G / {total:.1f}G ({pct:.0f}%) -- {content}{thin}')
|
|
else:
|
|
print(f' {name:<16} {stype:<12} (no capacity info) -- {content}{thin}')
|
|
" 2>/dev/null || warn "Failed to parse storage list"
|
|
fi
|
|
|
|
# Per-pool VM summary (if pool is configured)
|
|
if [[ -n "$PVE_POOL" ]]; then
|
|
echo ""
|
|
echo -e " ${BOLD}Pool: ${PVE_POOL}${RESET}"
|
|
local pool_data
|
|
pool_data=$(pve_api GET "/pools/${PVE_POOL}" 2>/dev/null) || pool_data=""
|
|
if [[ -n "$pool_data" ]]; then
|
|
python3 -c "
|
|
import json, sys
|
|
data = json.loads('''${pool_data}''').get('data', {})
|
|
members = data.get('members', [])
|
|
vms = [m for m in members if m.get('type') == 'qemu']
|
|
running = sum(1 for v in vms if v.get('status') == 'running')
|
|
stopped = sum(1 for v in vms if v.get('status') == 'stopped')
|
|
total_mem = sum(v.get('maxmem', 0) for v in vms) / (1024**3)
|
|
total_disk = sum(v.get('maxdisk', 0) for v in vms) / (1024**3)
|
|
print(f' VMs: {len(vms)} total ({running} running, {stopped} stopped)')
|
|
print(f' Allocated: {total_mem:.1f} GiB RAM, {total_disk:.0f} GiB disk')
|
|
" 2>/dev/null || warn "Failed to parse pool data"
|
|
fi
|
|
|
|
# Actual disk usage via storage content API
|
|
if [[ -n "$pool_data" ]]; then
|
|
local storage_content
|
|
storage_content=$(pve_api GET "/nodes/${PVE_NODE}/storage/${PVE_STORAGE}/content" 2>/dev/null) || storage_content=""
|
|
local storage_status_json
|
|
storage_status_json=$(echo "$storage_list" | python3 -c "
|
|
import json, sys
|
|
data = json.load(sys.stdin).get('data', [])
|
|
for s in data:
|
|
if s.get('storage') == '${PVE_STORAGE}':
|
|
print(json.dumps(s))
|
|
break
|
|
" 2>/dev/null) || storage_status_json=""
|
|
|
|
if [[ -n "$storage_content" ]] && [[ -n "$pool_data" ]]; then
|
|
python3 -c "
|
|
import json, sys
|
|
content = json.loads('''${storage_content}''').get('data', [])
|
|
pool = json.loads('''${pool_data}''').get('data', {})
|
|
vm_vmids = {str(m.get('vmid', '')) for m in pool.get('members', []) if m.get('type') == 'qemu'}
|
|
|
|
# Sum actual (used) bytes for volumes belonging to pool VMs
|
|
actual_bytes = 0
|
|
allocated_bytes = 0
|
|
for vol in content:
|
|
vmid = str(vol.get('vmid', ''))
|
|
if vmid in vm_vmids:
|
|
actual_bytes += vol.get('used', 0)
|
|
allocated_bytes += vol.get('size', 0)
|
|
|
|
if allocated_bytes > 0:
|
|
actual_gib = actual_bytes / (1024**3)
|
|
allocated_gib = allocated_bytes / (1024**3)
|
|
ratio = (actual_bytes / allocated_bytes * 100) if allocated_bytes > 0 else 0
|
|
print(f' Actual: {actual_gib:.1f} GiB disk used ({ratio:.0f}% of allocated, thin provisioned)')
|
|
" 2>/dev/null || true
|
|
fi
|
|
|
|
# Storage type note
|
|
if [[ -n "$storage_status_json" ]]; then
|
|
python3 -c "
|
|
import json
|
|
s = json.loads('''${storage_status_json}''')
|
|
stype = s.get('type', '')
|
|
notes = {'zfspool': 'ZFS (thin provisioned, lz4 compression)', 'lvmthin': 'LVM-thin (thin provisioned)'}
|
|
if stype in notes:
|
|
print(f' Storage: {notes[stype]}')
|
|
" 2>/dev/null || true
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
echo ""
|
|
}
|
|
|
|
do_list_labs() {
|
|
echo -e "${BOLD}${SCRIPT_NAME} labs${RESET}"
|
|
echo ""
|
|
|
|
if [[ "$PVE_METHOD" != "api" ]]; then
|
|
warn "Lab listing requires API method (current: ${PVE_METHOD})"
|
|
warn "Use --method api or set method: api in proxmox.yaml"
|
|
return
|
|
fi
|
|
|
|
# Get all managed VMs and group by config file
|
|
local all_vms
|
|
if [[ -n "$PVE_POOL" ]]; then
|
|
all_vms=$(pve_api GET "/pools/${PVE_POOL}" 2>/dev/null) || all_vms=""
|
|
else
|
|
all_vms=$(pve_api GET "/nodes/${PVE_NODE}/qemu" 2>/dev/null) || all_vms=""
|
|
fi
|
|
|
|
if [[ -z "$all_vms" ]]; then
|
|
info "No VMs found"
|
|
return
|
|
fi
|
|
|
|
# Collect managed VMs and group by config= tag in description
|
|
local vmids=()
|
|
if [[ -n "$PVE_POOL" ]]; then
|
|
# Pool response has members[] with type=qemu
|
|
mapfile -t vmids < <(python3 -c "
|
|
import json
|
|
data = json.loads('''${all_vms}''').get('data', {})
|
|
for m in data.get('members', []):
|
|
if m.get('type') == 'qemu':
|
|
print(m.get('vmid', ''))
|
|
" 2>/dev/null)
|
|
else
|
|
mapfile -t vmids < <(python3 -c "
|
|
import json
|
|
data = json.loads('''${all_vms}''').get('data', [])
|
|
for v in data:
|
|
print(v.get('vmid', ''))
|
|
" 2>/dev/null)
|
|
fi
|
|
|
|
# For each VM, get config and check for managed marker
|
|
declare -A lab_configs=() # config_name -> "vmid1:name1:status1 vmid2:name2:status2 ..."
|
|
declare -A lab_ram=() # config_name -> total RAM in MiB
|
|
declare -A lab_disk=() # config_name -> total disk in GiB
|
|
local unmanaged=0
|
|
|
|
for vmid in "${vmids[@]}"; do
|
|
[[ -z "$vmid" ]] && continue
|
|
|
|
local config_output
|
|
config_output=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null) || continue
|
|
|
|
local name="" mem_mib="" disk_gib="" desc="" status
|
|
{
|
|
read -r name
|
|
read -r mem_mib
|
|
read -r disk_gib
|
|
read -r desc
|
|
} < <(python3 -c "
|
|
import json
|
|
data = json.loads('''${config_output}''').get('data', {})
|
|
desc = data.get('description', '')
|
|
name = data.get('name', 'unknown')
|
|
mem = int(data.get('memory', 0))
|
|
# Rough disk from scsi0 size
|
|
scsi0 = data.get('scsi0', '')
|
|
disk_g = 0
|
|
for part in scsi0.split(','):
|
|
if part.strip().startswith('size='):
|
|
sz = part.strip().split('=')[1]
|
|
if sz.endswith('G'):
|
|
disk_g = int(sz[:-1])
|
|
print(name)
|
|
print(mem)
|
|
print(disk_g)
|
|
print(desc)
|
|
" 2>/dev/null || true)
|
|
|
|
# Check for managed marker
|
|
if [[ "$desc" != *"$MANAGED_MARKER"* ]]; then
|
|
unmanaged=$((unmanaged + 1))
|
|
continue
|
|
fi
|
|
|
|
# Extract config= tag
|
|
local config_tag="unknown"
|
|
if [[ "$desc" =~ config=([^\ ]+) ]]; then
|
|
config_tag="${BASH_REMATCH[1]}"
|
|
fi
|
|
|
|
# Get VM status
|
|
status=$(pve_vm_status "$vmid") || status="unknown"
|
|
|
|
# Accumulate
|
|
lab_configs["$config_tag"]="${lab_configs[$config_tag]:-} ${vmid}:${name}:${status}"
|
|
lab_ram["$config_tag"]=$(( ${lab_ram[$config_tag]:-0} + ${mem_mib:-0} ))
|
|
lab_disk["$config_tag"]=$(( ${lab_disk[$config_tag]:-0} + ${disk_gib:-0} ))
|
|
done
|
|
|
|
if [[ ${#lab_configs[@]} -eq 0 ]]; then
|
|
info "No managed labs found"
|
|
[[ $unmanaged -gt 0 ]] && detail "${unmanaged} unmanaged VMs in pool"
|
|
return 0
|
|
fi
|
|
|
|
local sorted_configs
|
|
sorted_configs=$(printf '%s\n' "${!lab_configs[@]}" | sort)
|
|
for config in $sorted_configs; do
|
|
local vms="${lab_configs[$config]}"
|
|
local ram_mib="${lab_ram[$config]}"
|
|
local disk_gib="${lab_disk[$config]}"
|
|
local vm_count=0
|
|
local running=0
|
|
local stopped=0
|
|
|
|
echo -e " ${BOLD}${config}${RESET}"
|
|
|
|
for entry in $vms; do
|
|
[[ -z "$entry" ]] && continue
|
|
local v_vmid v_name v_status
|
|
IFS=':' read -r v_vmid v_name v_status <<< "$entry"
|
|
vm_count=$((vm_count + 1))
|
|
[[ "$v_status" == "running" ]] && running=$((running + 1))
|
|
[[ "$v_status" == "stopped" ]] && stopped=$((stopped + 1))
|
|
|
|
local status_color="${DIM}"
|
|
[[ "$v_status" == "running" ]] && status_color="${GREEN}"
|
|
[[ "$v_status" == "stopped" ]] && status_color="${YELLOW}"
|
|
echo -e " ${v_name} (${v_vmid}): ${status_color}${v_status}${RESET}"
|
|
done
|
|
|
|
local lab_status="${GREEN}all running${RESET}"
|
|
if [[ $running -eq 0 ]] && [[ $stopped -gt 0 ]]; then
|
|
lab_status="${YELLOW}all stopped${RESET}"
|
|
elif [[ $running -gt 0 ]] && [[ $stopped -gt 0 ]]; then
|
|
lab_status="${YELLOW}mixed${RESET}"
|
|
fi
|
|
|
|
echo -e " Status: ${lab_status} | RAM: $((ram_mib / 1024))G | Disk: ${disk_gib}G"
|
|
echo ""
|
|
done
|
|
|
|
[[ $unmanaged -gt 0 ]] && detail "${unmanaged} unmanaged VMs in pool"
|
|
return 0
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Status check
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_status() {
|
|
step "Deployment Status"
|
|
echo ""
|
|
|
|
local count=${#VM_NAMES[@]}
|
|
local i=0
|
|
while [[ $i -lt $count ]]; do
|
|
local name="${VM_NAMES[$i]}"
|
|
|
|
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
local vmid="${VM_VMIDS[$i]}"
|
|
local app="${VM_APPS[$i]}"
|
|
|
|
echo -e " ${BOLD}${name}${RESET} (VMID ${vmid}, app: ${app})"
|
|
|
|
# Proxmox state
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
echo -e " Proxmox: ${DIM}[dry-run]${RESET}"
|
|
i=$((i + 1))
|
|
echo ""
|
|
continue
|
|
fi
|
|
|
|
if ! pve_vm_exists "$vmid"; then
|
|
echo -e " Proxmox: ${RED}not found${RESET}"
|
|
i=$((i + 1))
|
|
echo ""
|
|
continue
|
|
fi
|
|
|
|
local managed=false
|
|
pve_vm_is_managed "$vmid" && managed=true
|
|
|
|
local status
|
|
status=$(pve_vm_status "$vmid") || status="unknown"
|
|
|
|
local has_media=false
|
|
pve_vm_has_install_media "$vmid" && has_media=true
|
|
|
|
local install_state="installed"
|
|
if [[ "$has_media" == true ]]; then
|
|
install_state="not installed (media attached)"
|
|
fi
|
|
|
|
local status_color="$GREEN"
|
|
if [[ "$status" != "running" ]]; then
|
|
status_color="$YELLOW"
|
|
fi
|
|
|
|
echo -e " Proxmox: exists managed: ${managed} status: ${status_color}${status}${RESET} ${install_state}"
|
|
|
|
# Network and service checks (only for running, installed VMs)
|
|
if [[ "$status" == "running" ]] && [[ "$has_media" == false ]]; then
|
|
local ip
|
|
ip=$(pve_vm_ip "$vmid") || ip=""
|
|
|
|
if [[ -n "$ip" ]]; then
|
|
local port_status="${RED}closed${RESET}"
|
|
if curl -sk --connect-timeout 5 "https://${ip}:8443" &>/dev/null; then
|
|
port_status="${GREEN}open${RESET}"
|
|
fi
|
|
echo -e " Network: ${ip} port 8443: ${port_status}"
|
|
|
|
# Incus remote check + auto-register
|
|
if command -v incus &>/dev/null; then
|
|
local remote_name=""
|
|
remote_name=$(incus remote list --format csv 2>/dev/null | \
|
|
awk -F',' -v addr="$ip" '$2 ~ addr {print $1; exit}') || true
|
|
if [[ -n "$remote_name" ]]; then
|
|
echo -e " Remote: ${GREEN}incus remote '${remote_name}' configured${RESET}"
|
|
else
|
|
register_remote "$name" "$ip"
|
|
# Re-check after registration attempt
|
|
remote_name=$(incus remote list --format csv 2>/dev/null | \
|
|
awk -F',' -v addr="$ip" '$2 ~ addr {print $1; exit}') || true
|
|
if [[ -n "$remote_name" ]]; then
|
|
echo -e " Remote: ${GREEN}incus remote '${remote_name}' configured${RESET}"
|
|
else
|
|
echo -e " Remote: ${YELLOW}no incus remote for ${ip}${RESET}"
|
|
fi
|
|
fi
|
|
fi
|
|
else
|
|
echo -e " Network: ${YELLOW}IP not detected${RESET}"
|
|
fi
|
|
fi
|
|
|
|
i=$((i + 1))
|
|
echo ""
|
|
done
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Post-deployment checks
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_post_deploy() {
|
|
step "Post-Deployment Checks"
|
|
echo ""
|
|
|
|
local count=${#VM_NAMES[@]}
|
|
local all_ok=true
|
|
local i=0
|
|
while [[ $i -lt $count ]]; do
|
|
local name="${VM_NAMES[$i]}"
|
|
|
|
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
local vmid="${VM_VMIDS[$i]}"
|
|
local app="${VM_APPS[$i]}"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] Would check '${name}' (VMID ${vmid})"
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
# Only check running, installed VMs
|
|
if ! pve_vm_exists "$vmid"; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
local status
|
|
status=$(pve_vm_status "$vmid") || status="unknown"
|
|
if [[ "$status" != "running" ]]; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
local has_media=false
|
|
pve_vm_has_install_media "$vmid" && has_media=true
|
|
if [[ "$has_media" == true ]]; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
local ip
|
|
ip=$(pve_vm_ip "$vmid") || ip=""
|
|
if [[ -z "$ip" ]]; then
|
|
warn "VM '${name}': could not detect IP address"
|
|
all_ok=false
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
# Port 8443 check
|
|
if curl -sk --connect-timeout 5 "https://${ip}:8443" &>/dev/null; then
|
|
success "VM '${name}' (${ip}): port 8443 reachable"
|
|
else
|
|
warn "VM '${name}' (${ip}): port 8443 not reachable (may still be starting)"
|
|
all_ok=false
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
# App-specific checks
|
|
if [[ "$app" == "incus" ]] && command -v incus &>/dev/null; then
|
|
local remote_name=""
|
|
remote_name=$(incus remote list --format csv 2>/dev/null | \
|
|
awk -F',' -v addr="$ip" '$2 ~ addr {print $1; exit}') || true
|
|
if [[ -n "$remote_name" ]]; then
|
|
if incus info "${remote_name}:" &>/dev/null; then
|
|
success "VM '${name}': incus remote '${remote_name}' connected"
|
|
else
|
|
warn "VM '${name}': incus remote '${remote_name}' exists but connection failed"
|
|
all_ok=false
|
|
fi
|
|
else
|
|
# Auto-register the remote
|
|
register_remote "$name" "$ip"
|
|
# Verify
|
|
remote_name=$(incus remote list --format csv 2>/dev/null | \
|
|
awk -F',' -v addr="$ip" '$2 ~ addr {print $1; exit}') || true
|
|
if [[ -n "$remote_name" ]]; then
|
|
success "VM '${name}': incus remote '${remote_name}' connected"
|
|
else
|
|
info "VM '${name}': no incus remote configured"
|
|
info " Add with: incus remote add ${name} ${ip} --accept-certificate"
|
|
fi
|
|
fi
|
|
elif [[ "$app" == "operations-center" ]]; then
|
|
success "VM '${name}': Operations Center at https://${ip}:8443"
|
|
info " Import PKCS#12 cert in browser for web UI access:"
|
|
info " openssl pkcs12 -export -inkey ~/.config/incus/client.key -in ~/.config/incus/client.crt -out client.pfx"
|
|
fi
|
|
|
|
i=$((i + 1))
|
|
done
|
|
|
|
if [[ "$all_ok" == true ]] && [[ "$DRY_RUN" != true ]]; then
|
|
echo ""
|
|
success "All post-deployment checks passed"
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cleanup mode
|
|
# ---------------------------------------------------------------------------
|
|
|
|
do_cleanup() {
|
|
step "Cleanup: destroying VMs defined in config"
|
|
if [[ "$FORCE_CLEANUP" == true ]]; then
|
|
warn "Force cleanup enabled -- will destroy VMs even without management marker"
|
|
fi
|
|
echo ""
|
|
|
|
local count=${#VM_NAMES[@]}
|
|
local destroyed=0
|
|
local not_found=0
|
|
local skipped_unmanaged=0
|
|
local i=0
|
|
|
|
while [[ $i -lt $count ]]; do
|
|
local name="${VM_NAMES[$i]}"
|
|
|
|
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
local vmid="${VM_VMIDS[$i]}"
|
|
|
|
# 1. Check if VM exists
|
|
if ! pve_vm_exists "$vmid"; then
|
|
detail "VM '${name}' (VMID ${vmid}): does not exist"
|
|
not_found=$((not_found + 1))
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
# 2. Check if VM is managed by us
|
|
if [[ "$DRY_RUN" != true ]]; then
|
|
if ! pve_vm_is_managed "$vmid"; then
|
|
if [[ "$FORCE_CLEANUP" == true ]]; then
|
|
warn "VM '${name}' (VMID ${vmid}): NOT managed by ${SCRIPT_NAME} -- destroying anyway (--force-cleanup)"
|
|
else
|
|
warn "VM '${name}' (VMID ${vmid}): NOT managed by ${SCRIPT_NAME} -- refusing to destroy"
|
|
warn " This VM was not created by this tool. To destroy it manually:"
|
|
warn " ssh ${PVE_SSH_USER}@${PVE_HOST} qm stop ${vmid} && ssh ${PVE_SSH_USER}@${PVE_HOST} qm destroy ${vmid} --purge"
|
|
warn " Or use --force-cleanup to override this safety check"
|
|
skipped_unmanaged=$((skipped_unmanaged + 1))
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# 3. Destroy the VM
|
|
step "Destroying VM '${name}' (VMID ${vmid})"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] Would stop and destroy VM ${vmid}"
|
|
destroyed=$((destroyed + 1))
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
# Stop if running
|
|
local status
|
|
status=$(pve_vm_status "$vmid") || status="unknown"
|
|
if [[ "$status" == "running" ]]; then
|
|
info "Stopping VM..."
|
|
pve_stop_vm "$vmid" || true
|
|
sleep 5
|
|
fi
|
|
|
|
# Destroy
|
|
if pve_destroy_vm "$vmid"; then
|
|
success "VM '${name}' (VMID ${vmid}) destroyed"
|
|
destroyed=$((destroyed + 1))
|
|
else
|
|
error "Failed to destroy VM '${name}' (VMID ${vmid})"
|
|
fi
|
|
|
|
i=$((i + 1))
|
|
done
|
|
|
|
# Summary
|
|
echo ""
|
|
local summary="Cleanup complete: ${destroyed} destroyed"
|
|
if [[ $not_found -gt 0 ]]; then
|
|
summary+=", ${not_found} not found"
|
|
fi
|
|
if [[ $skipped_unmanaged -gt 0 ]]; then
|
|
summary+=", ${skipped_unmanaged} skipped (unmanaged)"
|
|
fi
|
|
success "$summary"
|
|
}
|
|
|
|
# Detect which IncusOS ISO was used by checking ide2 config on existing VMs.
|
|
# Sets DEPLOYMENT_ISO to the filename (e.g. "IncusOS_25.03.iso") or "" if unknown.
|
|
DEPLOYMENT_ISO=""
|
|
|
|
detect_deployment_iso() {
|
|
DEPLOYMENT_ISO=""
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
return
|
|
fi
|
|
|
|
local count=${#VM_NAMES[@]}
|
|
local i=0
|
|
while [[ $i -lt $count ]]; do
|
|
local vmid="${VM_VMIDS[$i]}"
|
|
if pve_vm_exists "$vmid"; then
|
|
local config=""
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
config=$(pve_run "qm config ${vmid} 2>/dev/null" 2>/dev/null) || config=""
|
|
;;
|
|
api)
|
|
config=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null) || config=""
|
|
;;
|
|
esac
|
|
|
|
if [[ -n "$config" ]]; then
|
|
local iso_name
|
|
# Extract IncusOS ISO name from ide2 line or JSON
|
|
iso_name=$(echo "$config" | python3 -c "
|
|
import sys, json, re
|
|
text = sys.stdin.read()
|
|
# Try JSON (API response)
|
|
try:
|
|
data = json.loads(text)
|
|
ide2 = data.get('data', data).get('ide2', '')
|
|
except (json.JSONDecodeError, AttributeError):
|
|
ide2 = ''
|
|
for line in text.splitlines():
|
|
if line.startswith('ide2:'):
|
|
ide2 = line.split(':', 1)[1].strip()
|
|
break
|
|
m = re.search(r'(IncusOS_[^,]+\.iso)', ide2)
|
|
if m:
|
|
print(m.group(1))
|
|
" 2>/dev/null) || iso_name=""
|
|
|
|
if [[ -n "$iso_name" ]]; then
|
|
DEPLOYMENT_ISO="$iso_name"
|
|
detail "Detected deployment ISO: ${DEPLOYMENT_ISO} (from VM ${vmid})"
|
|
return
|
|
fi
|
|
fi
|
|
fi
|
|
i=$((i + 1))
|
|
done
|
|
}
|
|
|
|
# Deep cleanup: remove seeds and the specific IncusOS ISO used by this deployment
|
|
cleanup_storage() {
|
|
step "Deep cleanup: removing seeds and deployment ISO from Proxmox storage"
|
|
|
|
local count=${#VM_NAMES[@]}
|
|
local i=0
|
|
|
|
# Remove per-VM seed ISOs
|
|
while [[ $i -lt $count ]]; do
|
|
local name="${VM_NAMES[$i]}"
|
|
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
local seed_file="seed-${name}.iso"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] Would delete ${PVE_ISO_STORAGE}:iso/${seed_file}"
|
|
else
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
pve_run "pvesm free ${PVE_ISO_STORAGE}:iso/${seed_file} 2>/dev/null" &>/dev/null || true
|
|
;;
|
|
api)
|
|
pve_api DELETE "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content/${PVE_ISO_STORAGE}:iso/${seed_file}" &>/dev/null || true
|
|
;;
|
|
esac
|
|
info "Deleted seed: ${seed_file}"
|
|
fi
|
|
|
|
i=$((i + 1))
|
|
done
|
|
|
|
# Remove only the specific IncusOS ISO used by this deployment
|
|
if [[ -n "$DEPLOYMENT_ISO" ]]; then
|
|
local volid="${PVE_ISO_STORAGE}:iso/${DEPLOYMENT_ISO}"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] Would delete ${volid}"
|
|
else
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
pve_run "pvesm free ${volid} 2>/dev/null" &>/dev/null || true
|
|
;;
|
|
api)
|
|
local encoded_volid
|
|
encoded_volid=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${volid}', safe=''))" 2>/dev/null)
|
|
pve_api DELETE "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content/${encoded_volid}" &>/dev/null || true
|
|
;;
|
|
esac
|
|
info "Deleted ISO: ${DEPLOYMENT_ISO}"
|
|
fi
|
|
else
|
|
detail "No deployment ISO detected (VMs may already be destroyed)"
|
|
fi
|
|
|
|
success "Storage cleanup complete"
|
|
}
|
|
|
|
# Aggressive cleanup: remove ALL IncusOS ISOs and seed ISOs from storage
|
|
cleanup_storage_all() {
|
|
step "Deep cleanup: removing ALL IncusOS ISOs and seeds from Proxmox storage"
|
|
|
|
local iso_list=""
|
|
if [[ "$DRY_RUN" != true ]]; then
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
iso_list=$(pve_run "pvesm list ${PVE_ISO_STORAGE} --content iso 2>/dev/null" 2>/dev/null | grep -E "(IncusOS_|seed-)" | awk '{print $1}') || iso_list=""
|
|
;;
|
|
api)
|
|
iso_list=$(pve_api GET "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content?content=iso" 2>/dev/null | \
|
|
python3 -c "
|
|
import json, sys
|
|
data = json.load(sys.stdin)
|
|
for item in data.get('data', []):
|
|
volid = item.get('volid', '')
|
|
if 'IncusOS_' in volid or 'seed-' in volid:
|
|
print(volid)
|
|
" 2>/dev/null) || iso_list=""
|
|
;;
|
|
esac
|
|
fi
|
|
|
|
if [[ -n "$iso_list" ]]; then
|
|
while IFS= read -r volid; do
|
|
[[ -z "$volid" ]] && continue
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] Would delete ${volid}"
|
|
else
|
|
case "$PVE_METHOD" in
|
|
ssh)
|
|
pve_run "pvesm free ${volid} 2>/dev/null" &>/dev/null || true
|
|
;;
|
|
api)
|
|
local encoded_volid
|
|
encoded_volid=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${volid}', safe=''))" 2>/dev/null)
|
|
pve_api DELETE "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content/${encoded_volid}" &>/dev/null || true
|
|
;;
|
|
esac
|
|
info "Deleted: ${volid}"
|
|
fi
|
|
done <<< "$iso_list"
|
|
else
|
|
info "No IncusOS ISOs or seeds found in storage"
|
|
fi
|
|
|
|
success "Storage cleanup complete"
|
|
}
|
|
|
|
# Remove incus remotes. Uses VM_NAMES (config-based cleanup) or
|
|
# DESTROYED_VM_NAMES (pool-wide cleanup) depending on what's populated.
|
|
cleanup_remotes() {
|
|
step "Deep cleanup: removing incus remotes"
|
|
|
|
if ! command -v incus &>/dev/null; then
|
|
detail "incus client not installed, skipping remote cleanup"
|
|
return
|
|
fi
|
|
|
|
# Determine which name list to use
|
|
local names_to_check=()
|
|
if [[ ${#DESTROYED_VM_NAMES[@]} -gt 0 ]]; then
|
|
# Pool-wide cleanup: use names of VMs actually destroyed
|
|
names_to_check=("${DESTROYED_VM_NAMES[@]}")
|
|
elif [[ ${#VM_NAMES[@]} -gt 0 ]]; then
|
|
# Config-based cleanup
|
|
local i=0
|
|
while [[ $i -lt ${#VM_NAMES[@]} ]]; do
|
|
if [[ -n "$VM_FILTER" ]] && [[ "${VM_NAMES[$i]}" != "$VM_FILTER" ]]; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
names_to_check+=("${VM_NAMES[$i]}")
|
|
i=$((i + 1))
|
|
done
|
|
fi
|
|
|
|
if [[ ${#names_to_check[@]} -eq 0 ]]; then
|
|
detail "No VM names to check for remote cleanup"
|
|
success "Remote cleanup complete"
|
|
return
|
|
fi
|
|
|
|
for name in "${names_to_check[@]}"; do
|
|
# Check if a remote with this name exists
|
|
if incus remote list --format csv 2>/dev/null | grep -q "^${name},"; then
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] Would remove incus remote '${name}'"
|
|
else
|
|
incus remote remove "$name" 2>/dev/null || true
|
|
info "Removed incus remote: ${name}"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
success "Remote cleanup complete"
|
|
}
|
|
|
|
cleanup_local_cache() {
|
|
step "Deep cleanup: clearing local ISO cache"
|
|
|
|
local cache_base="${XDG_CACHE_HOME:-${HOME}/.cache}/${SCRIPT_NAME}/iso-cache"
|
|
if [[ -d "$cache_base" ]]; then
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] Would clear ${cache_base}"
|
|
else
|
|
rm -rf "$cache_base"
|
|
info "Cleared cache: ${cache_base}"
|
|
fi
|
|
else
|
|
detail "No local cache to clear"
|
|
fi
|
|
|
|
success "Local cache cleanup complete"
|
|
}
|
|
|
|
# Names of VMs destroyed during cleanup-all (used by cleanup_remotes)
|
|
declare -a DESTROYED_VM_NAMES=()
|
|
|
|
# Pool-wide cleanup: destroy ALL managed VMs in the resource pool
|
|
do_cleanup_all() {
|
|
step "Pool-wide cleanup: destroying all managed VMs"
|
|
echo ""
|
|
|
|
if [[ -z "$PVE_POOL" ]]; then
|
|
error "Pool-wide cleanup requires a resource pool (proxmox.pool)"
|
|
error "Set pool in proxmox.yaml or pass a config with proxmox.pool"
|
|
error "Without a pool, there is no safe way to scope which VMs to destroy"
|
|
exit 1
|
|
fi
|
|
|
|
# Get all VMs in the pool
|
|
local vm_list
|
|
if ! vm_list=$(pve_list_vms); then
|
|
error "Failed to list VMs in pool '${PVE_POOL}'"
|
|
error "Check Proxmox connectivity and API credentials"
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -z "$vm_list" ]]; then
|
|
info "No VMs found in pool '${PVE_POOL}'"
|
|
return
|
|
fi
|
|
|
|
# Filter for managed VMs
|
|
local managed_vms=()
|
|
local managed_names=()
|
|
while IFS=' ' read -r eid ename; do
|
|
[[ -z "$eid" ]] && continue
|
|
if pve_vm_is_managed "$eid"; then
|
|
managed_vms+=("$eid")
|
|
managed_names+=("${ename:-unknown}")
|
|
fi
|
|
done <<< "$vm_list"
|
|
|
|
if [[ ${#managed_vms[@]} -eq 0 ]]; then
|
|
info "No managed VMs found in pool '${PVE_POOL}'"
|
|
return
|
|
fi
|
|
|
|
# Show what will be destroyed
|
|
echo -e " ${BOLD}Managed VMs in pool '${PVE_POOL}':${RESET}"
|
|
local idx=0
|
|
while [[ $idx -lt ${#managed_vms[@]} ]]; do
|
|
echo -e " VMID ${managed_vms[$idx]}: ${managed_names[$idx]}"
|
|
idx=$((idx + 1))
|
|
done
|
|
echo ""
|
|
|
|
# Confirm
|
|
confirm "Destroy all ${#managed_vms[@]} managed VM(s) in pool '${PVE_POOL}'?"
|
|
|
|
# Destroy each
|
|
local destroyed=0
|
|
idx=0
|
|
while [[ $idx -lt ${#managed_vms[@]} ]]; do
|
|
local vmid="${managed_vms[$idx]}"
|
|
local name="${managed_names[$idx]}"
|
|
|
|
step "Destroying VM '${name}' (VMID ${vmid})"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "[dry-run] Would stop and destroy VM ${vmid}"
|
|
destroyed=$((destroyed + 1))
|
|
DESTROYED_VM_NAMES+=("$name")
|
|
idx=$((idx + 1))
|
|
continue
|
|
fi
|
|
|
|
local status
|
|
status=$(pve_vm_status "$vmid") || status="unknown"
|
|
if [[ "$status" == "running" ]]; then
|
|
info "Stopping VM..."
|
|
pve_stop_vm "$vmid" || true
|
|
sleep 5
|
|
fi
|
|
|
|
if pve_destroy_vm "$vmid"; then
|
|
success "VM '${name}' (VMID ${vmid}) destroyed"
|
|
destroyed=$((destroyed + 1))
|
|
DESTROYED_VM_NAMES+=("$name")
|
|
else
|
|
error "Failed to destroy VM '${name}' (VMID ${vmid})"
|
|
fi
|
|
|
|
idx=$((idx + 1))
|
|
done
|
|
|
|
echo ""
|
|
success "Pool cleanup complete: ${destroyed} VM(s) destroyed"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Reconcile existing deployments
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Check if VMs from config already exist and offer interactive options.
|
|
# Only called for PHASE=all, non-cleanup, non-dry-run runs.
|
|
# Returns 0 to continue with fresh deploy, or exits.
|
|
reconcile_existing() {
|
|
local count=${#VM_NAMES[@]}
|
|
local existing=0
|
|
local running=0
|
|
local installed=0
|
|
local i=0
|
|
|
|
while [[ $i -lt $count ]]; do
|
|
local name="${VM_NAMES[$i]}"
|
|
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
|
|
i=$((i + 1))
|
|
continue
|
|
fi
|
|
|
|
local vmid="${VM_VMIDS[$i]}"
|
|
if pve_vm_exists "$vmid"; then
|
|
existing=$((existing + 1))
|
|
|
|
local status
|
|
status=$(pve_vm_status "$vmid") || status="unknown"
|
|
if [[ "$status" == "running" ]]; then
|
|
running=$((running + 1))
|
|
fi
|
|
|
|
local has_media=false
|
|
pve_vm_has_install_media "$vmid" && has_media=true
|
|
if [[ "$has_media" == false ]]; then
|
|
installed=$((installed + 1))
|
|
fi
|
|
fi
|
|
|
|
i=$((i + 1))
|
|
done
|
|
|
|
# No existing VMs — fresh deploy, proceed normally
|
|
if [[ $existing -eq 0 ]]; then
|
|
return 0
|
|
fi
|
|
|
|
# Count of VMs we're deploying (respecting filter)
|
|
local target=0
|
|
for name in "${VM_NAMES[@]}"; do
|
|
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
|
|
continue
|
|
fi
|
|
target=$((target + 1))
|
|
done
|
|
|
|
echo ""
|
|
echo -e "${BOLD}Existing deployment detected${RESET}"
|
|
echo -e " ${existing}/${target} VMs exist (${running} running, ${installed} installed)"
|
|
echo ""
|
|
|
|
if [[ "$existing" -lt "$target" ]]; then
|
|
local incomplete=$((target - existing))
|
|
echo -e " ${YELLOW}Partial deployment:${RESET} ${incomplete} VM(s) still need to be created/installed"
|
|
fi
|
|
|
|
# With --yes, default to safe option (post-deploy checks)
|
|
if [[ "$YES" == true ]]; then
|
|
info "Existing VMs detected with --yes; running post-deployment checks"
|
|
echo ""
|
|
phase_status
|
|
phase_post_deploy
|
|
exit 0
|
|
fi
|
|
|
|
echo -e " Options:"
|
|
echo -e " ${BOLD}1)${RESET} Run status checks and post-deployment verification"
|
|
echo -e " ${BOLD}2)${RESET} Continue deployment (skip existing, install incomplete VMs)"
|
|
echo -e " ${BOLD}3)${RESET} Destroy and redeploy cleanly"
|
|
echo -e " ${BOLD}4)${RESET} Abort"
|
|
echo ""
|
|
echo -n -e " ${BOLD}Choose [1/2/3/4]: ${RESET}"
|
|
local choice
|
|
read -r choice
|
|
|
|
case "$choice" in
|
|
1)
|
|
phase_status
|
|
phase_post_deploy
|
|
exit 0
|
|
;;
|
|
2)
|
|
info "Continuing deployment (existing VMs will be skipped)"
|
|
return 0
|
|
;;
|
|
3)
|
|
warn "Destroying existing VMs before redeployment"
|
|
do_cleanup
|
|
echo ""
|
|
return 0
|
|
;;
|
|
4|*)
|
|
info "Aborted"
|
|
exit 0
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main
|
|
# ---------------------------------------------------------------------------
|
|
|
|
main() {
|
|
setup_colors
|
|
parse_args "$@"
|
|
validate_args
|
|
|
|
# Handle doctor mode early (before config parsing)
|
|
if [[ "$DOCTOR" == true ]]; then
|
|
run_doctor
|
|
return
|
|
fi
|
|
|
|
if [[ "$QUIET" != true ]]; then
|
|
echo -e "${BOLD}${SCRIPT_NAME}${RESET} v${VERSION}"
|
|
echo ""
|
|
fi
|
|
|
|
setup_workdir
|
|
|
|
# Handle --labs and --resources without a config file (only needs proxmox.yaml)
|
|
if { [[ "$LIST_LABS" == true ]] || [[ "$RESOURCES" == true ]]; } && [[ -z "$CONFIG_FILE" ]]; then
|
|
if ! find_proxmox_config; then
|
|
error "No proxmox.yaml found and no config file specified"
|
|
error "Create proxmox.yaml or use --proxmox FILE"
|
|
exit 1
|
|
fi
|
|
load_proxmox_config "$PROXMOX_CONFIG_RESOLVED"
|
|
|
|
[[ -z "$PVE_NODE" ]] && PVE_NODE="pve"
|
|
[[ -z "$PVE_STORAGE" ]] && PVE_STORAGE="local-lvm"
|
|
[[ -z "$PVE_ISO_STORAGE" ]] && PVE_ISO_STORAGE="local"
|
|
[[ -z "$PVE_BRIDGE" ]] && PVE_BRIDGE="vmbr0"
|
|
[[ -z "$PVE_SSH_USER" ]] && PVE_SSH_USER="root"
|
|
[[ -z "$PVE_METHOD" ]] && PVE_METHOD="ssh"
|
|
[[ -n "$METHOD" ]] && PVE_METHOD="$METHOD"
|
|
[[ -n "$HOST_OVERRIDE" ]] && PVE_HOST="$HOST_OVERRIDE"
|
|
|
|
if [[ -z "$PVE_HOST" ]]; then
|
|
error "proxmox.yaml must contain 'host:'"
|
|
exit 1
|
|
fi
|
|
|
|
require_api_auth
|
|
[[ "$RESOURCES" == true ]] && do_resources
|
|
[[ "$LIST_LABS" == true ]] && do_list_labs
|
|
return 0
|
|
fi
|
|
|
|
# Handle cleanup-all without a config file (only needs proxmox.yaml)
|
|
if [[ "$CLEANUP_ALL" == true ]] && [[ -z "$CONFIG_FILE" ]]; then
|
|
if ! find_proxmox_config; then
|
|
error "No proxmox.yaml found and no config file specified"
|
|
error "Create proxmox.yaml or use --proxmox FILE"
|
|
exit 1
|
|
fi
|
|
load_proxmox_config "$PROXMOX_CONFIG_RESOLVED"
|
|
|
|
# Apply defaults for anything still unset
|
|
[[ -z "$PVE_NODE" ]] && PVE_NODE="pve"
|
|
[[ -z "$PVE_STORAGE" ]] && PVE_STORAGE="local-lvm"
|
|
[[ -z "$PVE_ISO_STORAGE" ]] && PVE_ISO_STORAGE="local"
|
|
[[ -z "$PVE_BRIDGE" ]] && PVE_BRIDGE="vmbr0"
|
|
[[ -z "$PVE_SSH_USER" ]] && PVE_SSH_USER="root"
|
|
[[ -z "$PVE_METHOD" ]] && PVE_METHOD="ssh"
|
|
[[ -n "$METHOD" ]] && PVE_METHOD="$METHOD"
|
|
[[ -n "$HOST_OVERRIDE" ]] && PVE_HOST="$HOST_OVERRIDE"
|
|
|
|
if [[ -z "$PVE_HOST" ]]; then
|
|
error "proxmox.yaml must contain 'host:'"
|
|
exit 1
|
|
fi
|
|
|
|
require_api_auth
|
|
do_cleanup_all
|
|
if [[ "$DEEP_CLEANUP" == true ]]; then
|
|
echo ""
|
|
cleanup_storage_all
|
|
cleanup_remotes
|
|
cleanup_local_cache
|
|
fi
|
|
return
|
|
fi
|
|
|
|
if [[ "$DRY_RUN" != true ]]; then
|
|
check_dependencies
|
|
fi
|
|
echo ""
|
|
|
|
# Always run validate first
|
|
phase_validate
|
|
|
|
# Handle cleanup-all mode (with config file for additional context)
|
|
if [[ "$CLEANUP_ALL" == true ]]; then
|
|
do_cleanup_all
|
|
if [[ "$DEEP_CLEANUP" == true ]]; then
|
|
echo ""
|
|
cleanup_storage_all
|
|
cleanup_remotes
|
|
cleanup_local_cache
|
|
fi
|
|
return
|
|
fi
|
|
|
|
# Handle cleanup mode
|
|
if [[ "$CLEANUP" == true ]]; then
|
|
if [[ "$DEEP_CLEANUP" == true ]]; then
|
|
# Detect which ISO was used before destroying VMs
|
|
detect_deployment_iso
|
|
fi
|
|
confirm "Destroy VMs listed above?"
|
|
do_cleanup
|
|
if [[ "$DEEP_CLEANUP" == true ]]; then
|
|
echo ""
|
|
cleanup_storage
|
|
cleanup_remotes
|
|
cleanup_local_cache
|
|
fi
|
|
return
|
|
fi
|
|
|
|
# Handle lab lifecycle commands
|
|
if [[ "$LAB_DOWN" == true ]]; then
|
|
phase_lab_down
|
|
return
|
|
fi
|
|
|
|
if [[ "$LAB_UP" == true ]]; then
|
|
phase_lab_up
|
|
return
|
|
fi
|
|
|
|
# Handle --resources and --labs with config file
|
|
if [[ "$RESOURCES" == true ]] || [[ "$LIST_LABS" == true ]]; then
|
|
[[ "$RESOURCES" == true ]] && do_resources
|
|
[[ "$LIST_LABS" == true ]] && do_list_labs
|
|
return 0
|
|
fi
|
|
|
|
# Reconcile existing deployments (interactive menu on re-runs)
|
|
if [[ "$PHASE" == "all" ]] && [[ "$DRY_RUN" != true ]]; then
|
|
reconcile_existing
|
|
fi
|
|
|
|
# Confirm before proceeding
|
|
if [[ "$PHASE" == "all" ]]; then
|
|
confirm "Proceed with deployment?"
|
|
fi
|
|
|
|
# Run requested phases
|
|
case "$PHASE" in
|
|
validate)
|
|
# Already done above
|
|
;;
|
|
prepare)
|
|
phase_prepare
|
|
;;
|
|
upload)
|
|
phase_prepare
|
|
phase_upload
|
|
;;
|
|
create)
|
|
phase_prepare
|
|
phase_upload
|
|
phase_create
|
|
;;
|
|
install)
|
|
phase_prepare
|
|
phase_upload
|
|
phase_create
|
|
phase_install
|
|
;;
|
|
status)
|
|
phase_status
|
|
phase_post_deploy
|
|
;;
|
|
all)
|
|
phase_prepare
|
|
echo ""
|
|
phase_upload
|
|
echo ""
|
|
phase_create
|
|
echo ""
|
|
phase_install
|
|
echo ""
|
|
phase_post_deploy
|
|
;;
|
|
esac
|
|
|
|
echo ""
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "Dry run complete -- no changes were made"
|
|
else
|
|
success "Done"
|
|
fi
|
|
}
|
|
|
|
main "$@"
|