maarten
/
incus-contrib
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
incus-contrib/incusos/incusos-proxmox

2366 lines
75 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
# incusos-proxmox - Declarative IncusOS deployment on Proxmox VE
#
# Reads a YAML configuration file defining VMs and deploys them on a Proxmox
# host with IncusOS-correct settings (UEFI, TPM 2.0, VirtIO-SCSI, etc.).
# Generates per-VM seed images, uploads ISOs, creates VMs, and boots them
# through automated installation.
#
# Usage: incusos-proxmox [OPTIONS] CONFIG_FILE
# Run 'incusos-proxmox --help' for full usage information.
set -euo pipefail
readonly VERSION="0.1.0"
readonly SCRIPT_NAME="incusos-proxmox"
readonly CDN_INDEX="https://images.linuxcontainers.org/os/index.json"
readonly CDN_BASE="https://images.linuxcontainers.org/os"
readonly MIN_DISK_GIB=50
readonly MIN_MEMORY_MIB=4096
readonly MANAGED_MARKER="[incusos-lab:managed]"
# ---------------------------------------------------------------------------
# Color and formatting
# ---------------------------------------------------------------------------
setup_colors() {
if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[0;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
BOLD='\033[1m'
DIM='\033[2m'
RESET='\033[0m'
else
RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' RESET=''
fi
}
info() { [[ "$QUIET" == true ]] && return; echo -e "${BLUE}[info]${RESET} $*"; }
success() { [[ "$QUIET" == true ]] && return; echo -e "${GREEN}[ok]${RESET} $*"; }
warn() { echo -e "${YELLOW}[warn]${RESET} $*" >&2; }
error() { echo -e "${RED}[error]${RESET} $*" >&2; }
step() { [[ "$QUIET" == true ]] && return; echo -e "${CYAN}[step]${RESET} ${BOLD}$*${RESET}"; }
detail() { [[ "$QUIET" == true ]] && return; echo -e "${DIM} $*${RESET}"; }
# ---------------------------------------------------------------------------
# Help
# ---------------------------------------------------------------------------
usage() {
cat <<EOF
${BOLD}${SCRIPT_NAME}${RESET} v${VERSION} - Declarative IncusOS deployment on Proxmox VE
${BOLD}USAGE${RESET}
${SCRIPT_NAME} [OPTIONS] CONFIG_FILE
${BOLD}DESCRIPTION${RESET}
Reads a YAML configuration file defining VMs and deploys them on a Proxmox
host with IncusOS-correct settings. Handles seed generation, ISO upload, VM
creation, and automated installation.
IncusOS requires specific Proxmox VM settings:
- UEFI firmware without pre-enrolled Secure Boot keys
- TPM 2.0 virtual device
- VirtIO-SCSI controller (not VirtIO-blk)
- CPU type 'host' (x86_64_v3 instruction set requirement)
- Minimum 50 GiB disk, minimum 4 GiB RAM
This script ensures all settings are correct, preventing install failures.
${BOLD}OPTIONS${RESET}
${BOLD}-p, --phase${RESET} PHASE Run a specific phase:
${CYAN}validate${RESET} -- parse config, check connectivity
${CYAN}prepare${RESET} -- download ISO, generate seeds
${CYAN}upload${RESET} -- upload ISO and seeds to Proxmox
${CYAN}create${RESET} -- create VMs with correct settings
${CYAN}install${RESET} -- start VMs and monitor installation
${CYAN}all${RESET} -- run all phases (default)
${BOLD}-m, --method${RESET} METHOD Connection method: ${CYAN}ssh${RESET} (default), ${CYAN}api${RESET}
${BOLD}-H, --host${RESET} HOST Override Proxmox host from config
${BOLD}-i, --iso${RESET} FILE Use local ISO file (skip CDN download)
${BOLD}-c, --cert${RESET} FILE Client certificate PEM file
${BOLD}--no-cert${RESET} Skip certificate injection
${BOLD}--vm${RESET} NAME Operate on a single VM only
${BOLD}-y, --yes${RESET} Skip confirmation prompts
${BOLD}--cleanup${RESET} Destroy VMs defined in config (managed only)
${BOLD}--force-cleanup${RESET} Destroy VMs even without management marker
${BOLD}--dry-run${RESET} Preview actions without executing
${BOLD}-q, --quiet${RESET} Suppress informational output
${BOLD}-h, --help${RESET} Show this help message
${BOLD}-V, --version${RESET} Show version
${BOLD}CONFIG FILE${RESET}
YAML file defining the Proxmox connection and VM specifications.
See examples/proxmox-lab.yaml for a full example.
${BOLD}proxmox:${RESET}
host: 192.168.1.10 # Proxmox host (required)
node: pve # Node name (default: pve)
storage: local-lvm # VM disk storage (default: local-lvm)
iso_storage: local # ISO storage (default: local)
bridge: vmbr0 # Network bridge (default: vmbr0)
method: ssh # ssh or api (default: ssh)
ssh_user: root # SSH user (default: root)
${BOLD}defaults:${RESET}
cores: 4 # Default CPU cores
memory: 8192 # Default RAM in MiB (min: 4096)
disk: 64 # Default disk in GiB (min: 50)
channel: stable # Update channel
start_vmid: 200 # Auto-assign VMIDs from here
${BOLD}vms:${RESET}
- name: my-vm # VM name (required)
app: incus # incus or operations-center (required)
apply_defaults: true # Apply IncusOS defaults
cores: 4 # Override default cores
memory: 8192 # Override default memory
disk: 64 # Override default disk
vmid: 200 # Explicit VMID (optional)
${BOLD}EXAMPLES${RESET}
# Preview a full deployment
${SCRIPT_NAME} --dry-run examples/proxmox-lab.yaml
# Validate config and check Proxmox connectivity
${SCRIPT_NAME} --phase validate examples/proxmox-lab.yaml
# Deploy everything (with confirmation prompts)
${SCRIPT_NAME} examples/proxmox-lab.yaml
# Deploy without prompts
${SCRIPT_NAME} --yes examples/proxmox-lab.yaml
# Deploy a single VM from the config
${SCRIPT_NAME} --vm ops-center examples/proxmox-lab.yaml
# Use a local ISO instead of downloading
${SCRIPT_NAME} --iso IncusOS_20250201.iso examples/proxmox-lab.yaml
# Destroy all VMs defined in config
${SCRIPT_NAME} --cleanup examples/proxmox-lab.yaml
# Use API method with token
export PROXMOX_TOKEN_SECRET="xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
${SCRIPT_NAME} --method api examples/proxmox-lab.yaml
${BOLD}PROXMOX SETUP${RESET}
${BOLD}SSH method (default):${RESET}
Requires key-based SSH authentication to the Proxmox host.
Test with: ssh root@<host> pvesh get /version
${BOLD}API method:${RESET}
Requires an API token. Set up minimal privileges:
pveum role add IncusOSDeploy --privs "VM.Allocate VM.Config.Disk \\
VM.Config.CPU VM.Config.Memory VM.Config.Network VM.Config.CDROM \\
VM.Config.Options VM.Config.HWType VM.PowerMgmt VM.Audit \\
Datastore.AllocateSpace Datastore.AllocateTemplate \\
Datastore.Audit SDN.Use"
pveum user add automation@pve
pveum aclmod / -user automation@pve -role IncusOSDeploy
pveum user token add automation@pve deploy --privsep 0
Set the token secret:
export PROXMOX_TOKEN_SECRET="<secret-from-above>"
And in your config file:
proxmox:
method: api
api_token_id: automation@pve!deploy
${BOLD}PREREQUISITES${RESET}
- python3 with PyYAML (python3-yaml / py3-yaml)
- jq (for JSON field access; falls back to python3)
- ssh + scp (for SSH method)
- curl (for API method and CDN downloads)
- incusos-seed sibling script (auto-detected)
- Bash 4+ (associative arrays)
EOF
exit 0
}
# ---------------------------------------------------------------------------
# Argument parsing
# ---------------------------------------------------------------------------
# Defaults
PHASE="all"
METHOD=""
HOST_OVERRIDE=""
LOCAL_ISO=""
CERT_FILE=""
NO_CERT=false
VM_FILTER=""
YES=false
CLEANUP=false
FORCE_CLEANUP=false
DRY_RUN=false
QUIET=false
CONFIG_FILE=""
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
-p|--phase)
PHASE="${2:?'--phase requires a value'}"
shift 2
;;
-m|--method)
METHOD="${2:?'--method requires a value'}"
shift 2
;;
-H|--host)
HOST_OVERRIDE="${2:?'--host requires a value'}"
shift 2
;;
-i|--iso)
LOCAL_ISO="${2:?'--iso requires a file path'}"
shift 2
;;
-c|--cert)
CERT_FILE="${2:?'--cert requires a file path'}"
shift 2
;;
--no-cert)
NO_CERT=true
shift
;;
--vm)
VM_FILTER="${2:?'--vm requires a name'}"
shift 2
;;
-y|--yes)
YES=true
shift
;;
--cleanup)
CLEANUP=true
shift
;;
--force-cleanup)
CLEANUP=true
FORCE_CLEANUP=true
shift
;;
--dry-run)
DRY_RUN=true
shift
;;
-q|--quiet)
QUIET=true
shift
;;
-h|--help)
usage
;;
-V|--version)
echo "${SCRIPT_NAME} v${VERSION}"
exit 0
;;
-*)
error "Unknown option: $1"
echo "Run '${SCRIPT_NAME} --help' for usage information." >&2
exit 1
;;
*)
if [[ -z "$CONFIG_FILE" ]]; then
CONFIG_FILE="$1"
else
error "Unexpected argument: $1 (config file already set to ${CONFIG_FILE})"
echo "Run '${SCRIPT_NAME} --help' for usage information." >&2
exit 1
fi
shift
;;
esac
done
}
# ---------------------------------------------------------------------------
# Validation
# ---------------------------------------------------------------------------
validate_args() {
if [[ -z "$CONFIG_FILE" ]]; then
error "No configuration file specified"
echo "Usage: ${SCRIPT_NAME} [OPTIONS] CONFIG_FILE" >&2
echo "Run '${SCRIPT_NAME} --help' for usage information." >&2
exit 1
fi
if [[ ! -f "$CONFIG_FILE" ]]; then
error "Configuration file not found: ${CONFIG_FILE}"
exit 1
fi
case "$PHASE" in
validate|prepare|upload|create|install|all) ;;
*)
error "Invalid phase: ${PHASE}"
error "Supported: validate, prepare, upload, create, install, all"
exit 1
;;
esac
if [[ -n "$METHOD" ]]; then
case "$METHOD" in
ssh|api) ;;
*)
error "Invalid method: ${METHOD}"
error "Supported: ssh, api"
exit 1
;;
esac
fi
if [[ -n "$CERT_FILE" ]] && [[ "$NO_CERT" == true ]]; then
error "Cannot use both --cert and --no-cert"
exit 1
fi
if [[ -n "$CERT_FILE" ]] && [[ ! -f "$CERT_FILE" ]]; then
error "Certificate file not found: ${CERT_FILE}"
exit 1
fi
if [[ -n "$LOCAL_ISO" ]] && [[ ! -f "$LOCAL_ISO" ]]; then
error "ISO file not found: ${LOCAL_ISO}"
exit 1
fi
}
# ---------------------------------------------------------------------------
# Dependency checks
# ---------------------------------------------------------------------------
SEED_TOOL=""
check_dependencies() {
step "Checking dependencies"
local missing=false
# python3
if ! command -v python3 &>/dev/null; then
error "python3 is required but not found"
error "Install Python 3 for your platform (https://www.python.org/)"
missing=true
else
detail "python3: $(command -v python3)"
fi
# python3-yaml (optional -- basic YAML fallback built-in)
if command -v python3 &>/dev/null; then
if python3 -c "import yaml" 2>/dev/null; then
detail "PyYAML: available"
else
detail "PyYAML: not found (using built-in basic parser)"
fi
fi
# jq (optional, python3 fallback)
if command -v jq &>/dev/null; then
detail "jq: $(command -v jq)"
else
detail "jq: not found (using python3 fallback)"
fi
# curl
if ! command -v curl &>/dev/null; then
error "curl is required but not found"
missing=true
else
detail "curl: $(command -v curl)"
fi
# ssh/scp (for SSH method)
if [[ "${PVE_METHOD:-ssh}" == "ssh" ]]; then
if ! command -v ssh &>/dev/null; then
error "ssh is required for SSH method but not found"
missing=true
fi
if ! command -v scp &>/dev/null; then
error "scp is required for SSH method but not found"
missing=true
fi
fi
# incusos-seed sibling script
local script_dir
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SEED_TOOL="${script_dir}/incusos-seed"
if [[ ! -x "$SEED_TOOL" ]]; then
error "incusos-seed not found at: ${SEED_TOOL}"
error "The incusos-seed script must be in the same directory as ${SCRIPT_NAME}"
missing=true
else
detail "incusos-seed: ${SEED_TOOL}"
fi
# bash version
if [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then
error "Bash 4+ is required (found ${BASH_VERSION})"
missing=true
else
detail "bash: ${BASH_VERSION}"
fi
if [[ "$missing" == true ]]; then
error "Missing dependencies -- cannot continue"
exit 1
fi
success "All dependencies satisfied"
}
# ---------------------------------------------------------------------------
# Temporary directory management
# ---------------------------------------------------------------------------
WORKDIR=""
setup_workdir() {
WORKDIR=$(mktemp -d "${TMPDIR:-/tmp}/${SCRIPT_NAME}-XXXXXX")
trap 'rm -rf "$WORKDIR"' EXIT
}
# ---------------------------------------------------------------------------
# YAML parsing via python3
# ---------------------------------------------------------------------------
# Convert YAML config to JSON, stored in a file for repeated access
CONFIG_JSON=""
parse_config() {
step "Parsing configuration: ${CONFIG_FILE}"
CONFIG_JSON="${WORKDIR}/config.json"
# Try PyYAML first, fall back to a basic pure-Python parser
if ! python3 -c "
import json, sys, re
def _parse_value(v):
v = v.strip()
# Strip inline comments (but not inside quotes)
if not v.startswith(('\"', \"'\")):
m = re.match(r'^(.*?)\s+#.*$', v)
if m:
v = m.group(1).rstrip()
if not v:
return None
if (v.startswith('\"') and v.endswith('\"')) or (v.startswith(\"'\") and v.endswith(\"'\")):
return v[1:-1]
if v == 'true':
return True
if v == 'false':
return False
if v == 'null' or v == '~':
return None
try:
return int(v)
except ValueError:
pass
try:
return float(v)
except ValueError:
pass
return v
def parse_yaml_basic(path):
\"\"\"Minimal YAML parser for simple config files.
Handles: mappings, lists of mappings, lists of scalars, inline comments.
Does NOT handle: anchors, multi-line strings, flow style, merge keys.\"\"\"
with open(path) as f:
lines = f.readlines()
# Pre-process: strip comments and blank lines, track indent + content
processed = []
for raw in lines:
s = raw.rstrip()
if not s or s.lstrip().startswith('#'):
continue
indent = len(s) - len(s.lstrip())
content = s.lstrip()
processed.append((indent, content))
def parse_block(idx, base_indent):
\"\"\"Parse a block at the given indentation level, return (result, next_idx).\"\"\"
if idx >= len(processed):
return None, idx
indent, content = processed[idx]
# Detect if this block is a list or mapping
if content.startswith('- '):
return parse_list(idx, indent)
else:
return parse_mapping(idx, indent)
def parse_mapping(idx, base_indent):
result = {}
while idx < len(processed):
indent, content = processed[idx]
if indent < base_indent:
break
if indent > base_indent:
break
if content.startswith('- '):
break
# Must be key: value
m = re.match(r'^([^:]+?):\s*(.*?)$', content)
if not m:
idx += 1
continue
key = m.group(1).strip()
val_str = m.group(2).strip()
# Strip inline comment from value
if val_str and not val_str.startswith(('\"', \"'\")):
cm = re.match(r'^(.*?)\s+#.*$', val_str)
if cm:
val_str = cm.group(1).rstrip()
if val_str and not val_str.startswith('#'):
result[key] = _parse_value(val_str)
idx += 1
else:
# Value is a nested block
idx += 1
if idx < len(processed) and processed[idx][0] > base_indent:
child_indent = processed[idx][0]
child, idx = parse_block(idx, child_indent)
result[key] = child if child is not None else {}
else:
result[key] = {}
return result, idx
def parse_list(idx, base_indent):
result = []
while idx < len(processed):
indent, content = processed[idx]
if indent < base_indent:
break
if indent > base_indent:
break
if not content.startswith('- '):
break
item_str = content[2:].strip()
# Check if it's a mapping item: '- key: value'
m = re.match(r'^([^:]+?):\s*(.*?)$', item_str)
if m:
# It's a mapping -- parse this item and continuation lines
obj = {}
key = m.group(1).strip()
val_str = m.group(2).strip()
if val_str and not val_str.startswith('#'):
if not val_str.startswith(('\"', \"'\")):
cm = re.match(r'^(.*?)\s+#.*$', val_str)
if cm:
val_str = cm.group(1).rstrip()
obj[key] = _parse_value(val_str)
else:
idx += 1
if idx < len(processed) and processed[idx][0] > indent:
child_indent = processed[idx][0]
child, idx = parse_block(idx, child_indent)
obj[key] = child if child is not None else {}
else:
obj[key] = {}
# Parse remaining keys at item indent + 2
item_indent = indent + 2
while idx < len(processed) and processed[idx][0] == item_indent:
ic = processed[idx][1]
if ic.startswith('- '):
break
im = re.match(r'^([^:]+?):\s*(.*?)$', ic)
if im:
ik = im.group(1).strip()
iv = im.group(2).strip()
if iv and not iv.startswith('#'):
if not iv.startswith(('\"', \"'\")):
icm = re.match(r'^(.*?)\s+#.*$', iv)
if icm:
iv = icm.group(1).rstrip()
obj[ik] = _parse_value(iv)
idx += 1
else:
idx += 1
if idx < len(processed) and processed[idx][0] > item_indent:
child_indent = processed[idx][0]
child, idx = parse_block(idx, child_indent)
obj[ik] = child if child is not None else {}
else:
obj[ik] = {}
else:
idx += 1
result.append(obj)
continue
idx += 1
# Parse remaining keys of this list item
item_indent = indent + 2
while idx < len(processed) and processed[idx][0] >= item_indent:
if processed[idx][0] == item_indent:
ic = processed[idx][1]
if ic.startswith('- '):
break
im = re.match(r'^([^:]+?):\s*(.*?)$', ic)
if im:
ik = im.group(1).strip()
iv = im.group(2).strip()
if iv and not iv.startswith('#'):
if not iv.startswith(('\"', \"'\")):
icm = re.match(r'^(.*?)\s+#.*$', iv)
if icm:
iv = icm.group(1).rstrip()
obj[ik] = _parse_value(iv)
idx += 1
else:
idx += 1
if idx < len(processed) and processed[idx][0] > item_indent:
child_indent = processed[idx][0]
child, idx = parse_block(idx, child_indent)
obj[ik] = child if child is not None else {}
else:
obj[ik] = {}
else:
idx += 1
else:
# Deeper nesting handled by recursive calls above
break
result.append(obj)
else:
# Scalar list item
result.append(_parse_value(item_str))
idx += 1
return result, idx
result, _ = parse_block(0, 0)
return result if result is not None else {}
try:
import yaml
with open(sys.argv[1]) as f:
data = yaml.safe_load(f)
except ImportError:
data = parse_yaml_basic(sys.argv[1])
json.dump(data, sys.stdout, indent=2)
" "$CONFIG_FILE" > "$CONFIG_JSON" 2>/dev/null; then
error "Failed to parse YAML configuration: ${CONFIG_FILE}"
error "Check that the file is valid YAML"
error "For complex YAML features, install PyYAML: pip3 install pyyaml"
exit 1
fi
success "Configuration parsed"
}
# JSON field accessor -- uses jq if available, falls back to python3
json_get() {
local file="$1"
local query="$2"
local default="${3:-}"
local result
if command -v jq &>/dev/null; then
result=$(jq -r "$query // empty" "$file" 2>/dev/null) || result=""
else
result=$(python3 -c "
import json, sys
data = json.load(open(sys.argv[1]))
path = sys.argv[2]
# Simple jq-like path traversal for common patterns
if path.startswith('.'):
path = path[1:]
parts = path.replace('[]','').split('.')
obj = data
for p in parts:
if p == '':
continue
if isinstance(obj, dict):
obj = obj.get(p)
elif isinstance(obj, list) and p.isdigit():
obj = obj[int(p)]
else:
obj = None
if obj is None:
break
if obj is None:
print('')
elif isinstance(obj, bool):
print('true' if obj else 'false')
elif isinstance(obj, (dict, list)):
json.dump(obj, sys.stdout)
else:
print(obj)
" "$file" "$query" 2>/dev/null) || result=""
fi
if [[ -z "$result" ]] && [[ -n "$default" ]]; then
echo "$default"
else
echo "$result"
fi
}
# Get the number of VMs in config
vm_count() {
if command -v jq &>/dev/null; then
jq -r '.vms | length' "$CONFIG_JSON" 2>/dev/null || echo "0"
else
python3 -c "
import json
data = json.load(open('$CONFIG_JSON'))
print(len(data.get('vms', [])))
" 2>/dev/null || echo "0"
fi
}
# Get a field from a specific VM by index
vm_get() {
local index="$1"
local field="$2"
local default="${3:-}"
local result
if command -v jq &>/dev/null; then
result=$(jq -r ".vms[${index}].${field} // empty" "$CONFIG_JSON" 2>/dev/null) || result=""
else
result=$(python3 -c "
import json
data = json.load(open('$CONFIG_JSON'))
vm = data.get('vms', [])[${index}]
val = vm.get('${field}')
if val is None:
print('')
elif isinstance(val, bool):
print('true' if val else 'false')
else:
print(val)
" 2>/dev/null) || result=""
fi
if [[ -z "$result" ]] && [[ -n "$default" ]]; then
echo "$default"
else
echo "$result"
fi
}
# ---------------------------------------------------------------------------
# Configuration loading
# ---------------------------------------------------------------------------
# Proxmox connection settings
PVE_HOST=""
PVE_NODE=""
PVE_STORAGE=""
PVE_ISO_STORAGE=""
PVE_BRIDGE=""
PVE_METHOD=""
PVE_SSH_USER=""
PVE_API_TOKEN_ID=""
# Default VM settings
DEF_CORES=""
DEF_MEMORY=""
DEF_DISK=""
DEF_CHANNEL=""
DEF_START_VMID=""
# VM arrays (indexed by position)
declare -a VM_NAMES=()
declare -a VM_APPS=()
declare -a VM_APPLY_DEFAULTS=()
declare -a VM_CORES=()
declare -a VM_MEMORY=()
declare -a VM_DISK=()
declare -a VM_VMIDS=()
declare -a VM_CLUSTERS=()
declare -a VM_CLUSTER_ROLES=()
load_config() {
step "Loading configuration values"
# Proxmox settings
PVE_HOST=$(json_get "$CONFIG_JSON" ".proxmox.host" "")
PVE_NODE=$(json_get "$CONFIG_JSON" ".proxmox.node" "pve")
PVE_STORAGE=$(json_get "$CONFIG_JSON" ".proxmox.storage" "local-lvm")
PVE_ISO_STORAGE=$(json_get "$CONFIG_JSON" ".proxmox.iso_storage" "local")
PVE_BRIDGE=$(json_get "$CONFIG_JSON" ".proxmox.bridge" "vmbr0")
PVE_SSH_USER=$(json_get "$CONFIG_JSON" ".proxmox.ssh_user" "root")
PVE_API_TOKEN_ID=$(json_get "$CONFIG_JSON" ".proxmox.api_token_id" "")
# Method: CLI override > config > default
if [[ -z "$PVE_METHOD" ]]; then
PVE_METHOD=$(json_get "$CONFIG_JSON" ".proxmox.method" "ssh")
fi
# CLI override for method
if [[ -n "$METHOD" ]]; then
PVE_METHOD="$METHOD"
fi
# Host override
if [[ -n "$HOST_OVERRIDE" ]]; then
PVE_HOST="$HOST_OVERRIDE"
fi
# Default VM settings
DEF_CORES=$(json_get "$CONFIG_JSON" ".defaults.cores" "4")
DEF_MEMORY=$(json_get "$CONFIG_JSON" ".defaults.memory" "8192")
DEF_DISK=$(json_get "$CONFIG_JSON" ".defaults.disk" "64")
DEF_CHANNEL=$(json_get "$CONFIG_JSON" ".defaults.channel" "stable")
DEF_START_VMID=$(json_get "$CONFIG_JSON" ".defaults.start_vmid" "200")
# Load VMs
local count
count=$(vm_count)
local i=0
while [[ $i -lt $count ]]; do
VM_NAMES+=("$(vm_get "$i" "name" "")")
VM_APPS+=("$(vm_get "$i" "app" "incus")")
VM_APPLY_DEFAULTS+=("$(vm_get "$i" "apply_defaults" "false")")
VM_CORES+=("$(vm_get "$i" "cores" "$DEF_CORES")")
VM_MEMORY+=("$(vm_get "$i" "memory" "$DEF_MEMORY")")
VM_DISK+=("$(vm_get "$i" "disk" "$DEF_DISK")")
VM_VMIDS+=("$(vm_get "$i" "vmid" "")")
VM_CLUSTERS+=("$(vm_get "$i" "cluster" "")")
VM_CLUSTER_ROLES+=("$(vm_get "$i" "cluster_role" "")")
i=$((i + 1))
done
success "Loaded ${count} VM definition(s)"
}
# ---------------------------------------------------------------------------
# Configuration validation
# ---------------------------------------------------------------------------
validate_config() {
step "Validating configuration"
local errors=0
# Required: Proxmox host
if [[ -z "$PVE_HOST" ]]; then
error "proxmox.host is required"
errors=$((errors + 1))
fi
# Method validation
case "$PVE_METHOD" in
ssh|api) ;;
*)
error "Invalid proxmox.method: ${PVE_METHOD} (must be ssh or api)"
errors=$((errors + 1))
;;
esac
# API method requires token
if [[ "$PVE_METHOD" == "api" ]]; then
if [[ -z "$PVE_API_TOKEN_ID" ]]; then
error "proxmox.api_token_id is required for API method"
error "Example: automation@pve!deploy"
errors=$((errors + 1))
fi
if [[ -z "${PROXMOX_TOKEN_SECRET:-}" ]]; then
error "PROXMOX_TOKEN_SECRET environment variable is required for API method"
error "Set it with: export PROXMOX_TOKEN_SECRET=\"<your-token-secret>\""
errors=$((errors + 1))
fi
fi
# Must have at least one VM
local count=${#VM_NAMES[@]}
if [[ $count -eq 0 ]]; then
error "No VMs defined in configuration"
errors=$((errors + 1))
fi
# Validate each VM
local seen_names=""
local cluster_inits=""
local i=0
while [[ $i -lt $count ]]; do
local name="${VM_NAMES[$i]}"
local app="${VM_APPS[$i]}"
local memory="${VM_MEMORY[$i]}"
local disk="${VM_DISK[$i]}"
local cluster="${VM_CLUSTERS[$i]}"
local role="${VM_CLUSTER_ROLES[$i]}"
# Name required
if [[ -z "$name" ]]; then
error "VM #$((i + 1)): name is required"
errors=$((errors + 1))
i=$((i + 1))
continue
fi
# Unique names
if echo "$seen_names" | grep -qw "$name"; then
error "VM '${name}': duplicate VM name"
errors=$((errors + 1))
fi
seen_names="${seen_names} ${name}"
# Valid app
case "$app" in
incus|operations-center) ;;
*)
error "VM '${name}': invalid app '${app}' (must be incus or operations-center)"
errors=$((errors + 1))
;;
esac
# Minimum memory
if [[ "$memory" -lt "$MIN_MEMORY_MIB" ]] 2>/dev/null; then
error "VM '${name}': memory ${memory} MiB is below minimum ${MIN_MEMORY_MIB} MiB"
errors=$((errors + 1))
fi
# Minimum disk
if [[ "$disk" -lt "$MIN_DISK_GIB" ]] 2>/dev/null; then
error "VM '${name}': disk ${disk} GiB is below minimum ${MIN_DISK_GIB} GiB"
errors=$((errors + 1))
fi
# Cluster role validation
if [[ -n "$role" ]] && [[ -z "$cluster" ]]; then
error "VM '${name}': cluster_role '${role}' set but no cluster name specified"
errors=$((errors + 1))
fi
if [[ -n "$role" ]]; then
case "$role" in
init|join) ;;
*)
error "VM '${name}': invalid cluster_role '${role}' (must be init or join)"
errors=$((errors + 1))
;;
esac
fi
# Track cluster init nodes
if [[ -n "$cluster" ]] && [[ "$role" == "init" ]]; then
if echo "$cluster_inits" | grep -qw "$cluster"; then
error "Cluster '${cluster}': multiple init nodes defined (only one allowed)"
errors=$((errors + 1))
fi
cluster_inits="${cluster_inits} ${cluster}"
fi
i=$((i + 1))
done
# VM filter validation
if [[ -n "$VM_FILTER" ]]; then
local found=false
for name in "${VM_NAMES[@]}"; do
if [[ "$name" == "$VM_FILTER" ]]; then
found=true
break
fi
done
if [[ "$found" != true ]]; then
error "VM '${VM_FILTER}' not found in configuration"
error "Available VMs: ${VM_NAMES[*]}"
errors=$((errors + 1))
fi
fi
if [[ $errors -gt 0 ]]; then
error "${errors} validation error(s) found"
exit 1
fi
success "Configuration valid"
}
# ---------------------------------------------------------------------------
# Proxmox transport layer
# ---------------------------------------------------------------------------
# Execute a command on the Proxmox host
pve_run() {
local cmd="$1"
if [[ "$DRY_RUN" == true ]]; then
detail "[dry-run] ${PVE_METHOD}: ${cmd}"
return 0
fi
case "$PVE_METHOD" in
ssh)
ssh -o BatchMode=yes -o ConnectTimeout=10 \
"${PVE_SSH_USER}@${PVE_HOST}" "$cmd"
;;
api)
error "API execution of arbitrary commands is not supported"
error "Use specific API functions instead"
return 1
;;
esac
}
# Execute a Proxmox API call (API method)
pve_api() {
local method="$1"
local endpoint="$2"
local data="${3:-}"
local url="https://${PVE_HOST}:8006/api2/json${endpoint}"
local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET:-}"
if [[ "$DRY_RUN" == true ]]; then
detail "[dry-run] API ${method} ${endpoint}"
if [[ -n "$data" ]]; then
detail "[dry-run] data: ${data}"
fi
return 0
fi
local curl_args=(-fsSk -H "$auth")
case "$method" in
GET)
curl "${curl_args[@]}" "$url"
;;
POST)
if [[ -n "$data" ]]; then
curl "${curl_args[@]}" -X POST -d "$data" "$url"
else
curl "${curl_args[@]}" -X POST "$url"
fi
;;
PUT)
if [[ -n "$data" ]]; then
curl "${curl_args[@]}" -X PUT -d "$data" "$url"
else
curl "${curl_args[@]}" -X PUT "$url"
fi
;;
DELETE)
curl "${curl_args[@]}" -X DELETE "$url"
;;
esac
}
# Upload a file to Proxmox ISO storage
pve_upload() {
local local_path="$1"
local remote_filename="$2"
if [[ "$DRY_RUN" == true ]]; then
detail "[dry-run] upload: ${local_path} -> ${PVE_ISO_STORAGE}:iso/${remote_filename}"
return 0
fi
case "$PVE_METHOD" in
ssh)
local iso_dir
# Query the actual path of the ISO storage
iso_dir=$(pve_run "pvesm path ${PVE_ISO_STORAGE}:iso/test 2>/dev/null" | sed 's|/test$||') || true
if [[ -z "$iso_dir" ]]; then
# Common default paths
iso_dir="/var/lib/vz/template/iso"
fi
scp -o BatchMode=yes -o ConnectTimeout=10 \
"$local_path" "${PVE_SSH_USER}@${PVE_HOST}:${iso_dir}/${remote_filename}"
;;
api)
curl -fsSk \
-H "Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET:-}" \
-X POST \
-F "content=iso" \
-F "filename=${remote_filename}" \
-F "file=@${local_path}" \
"https://${PVE_HOST}:8006/api2/json/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/upload"
;;
esac
}
# Check if an ISO exists on Proxmox storage
pve_iso_exists() {
local filename="$1"
if [[ "$DRY_RUN" == true ]]; then
return 1
fi
case "$PVE_METHOD" in
ssh)
pve_run "pvesm list ${PVE_ISO_STORAGE} --content iso 2>/dev/null" | grep -q "$filename"
;;
api)
pve_api GET "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content?content=iso" | \
grep -q "$filename"
;;
esac
}
# Get list of existing VMIDs
pve_list_vmids() {
if [[ "$DRY_RUN" == true ]]; then
echo ""
return 0
fi
case "$PVE_METHOD" in
ssh)
pve_run "qm list 2>/dev/null" | awk 'NR>1 {print $1}' || true
;;
api)
pve_api GET "/nodes/${PVE_NODE}/qemu" 2>/dev/null | \
python3 -c "
import json, sys
data = json.load(sys.stdin)
for vm in data.get('data', []):
print(vm.get('vmid', ''))
" 2>/dev/null || true
;;
esac
}
# Create a VM via qm create
pve_create_vm() {
local vmid="$1"
local name="$2"
local cores="$3"
local memory="$4"
local disk_gib="$5"
local iso_filename="$6"
local seed_filename="$7"
local description
description=$(build_vm_description)
local cmd="qm create ${vmid}"
cmd+=" --name ${name}"
cmd+=" --description '${description}'"
cmd+=" --ostype l26"
cmd+=" --bios ovmf"
cmd+=" --machine q35"
cmd+=" --efidisk0 ${PVE_STORAGE}:0,efitype=4m,pre-enrolled-keys=0"
cmd+=" --tpmstate0 ${PVE_STORAGE}:1,version=v2.0"
cmd+=" --cpu host"
cmd+=" --cores ${cores}"
cmd+=" --memory ${memory}"
cmd+=" --balloon 0"
cmd+=" --scsihw virtio-scsi-pci"
cmd+=" --scsi0 ${PVE_STORAGE}:${disk_gib},iothread=1"
cmd+=" --ide2 ${PVE_ISO_STORAGE}:iso/${iso_filename},media=cdrom"
cmd+=" --ide3 ${PVE_ISO_STORAGE}:iso/${seed_filename},media=cdrom"
cmd+=" --net0 virtio,bridge=${PVE_BRIDGE}"
cmd+=" --boot order=ide2\\;scsi0"
cmd+=" --agent 1"
if [[ "$DRY_RUN" == true ]]; then
detail "[dry-run] ${cmd}"
return 0
fi
case "$PVE_METHOD" in
ssh)
pve_run "$cmd"
;;
api)
# URL-encode the description for the API
local url_encoded_desc
url_encoded_desc=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${description}'))" 2>/dev/null)
# Build API parameters
local params=""
params+="vmid=${vmid}"
params+="&name=${name}"
params+="&description=${url_encoded_desc}"
params+="&ostype=l26"
params+="&bios=ovmf"
params+="&machine=q35"
params+="&efidisk0=${PVE_STORAGE}:0,efitype=4m,pre-enrolled-keys=0"
params+="&tpmstate0=${PVE_STORAGE}:1,version=v2.0"
params+="&cpu=host"
params+="&cores=${cores}"
params+="&memory=${memory}"
params+="&balloon=0"
params+="&scsihw=virtio-scsi-pci"
params+="&scsi0=${PVE_STORAGE}:${disk_gib},iothread=1"
params+="&ide2=${PVE_ISO_STORAGE}:iso/${iso_filename},media=cdrom"
params+="&ide3=${PVE_ISO_STORAGE}:iso/${seed_filename},media=cdrom"
params+="&net0=virtio,bridge=${PVE_BRIDGE}"
params+="&boot=order%3Dide2%3Bscsi0"
params+="&agent=1"
pve_api POST "/nodes/${PVE_NODE}/qemu" "$params"
;;
esac
}
# Start a VM
pve_start_vm() {
local vmid="$1"
case "$PVE_METHOD" in
ssh) pve_run "qm start ${vmid}" ;;
api) pve_api POST "/nodes/${PVE_NODE}/qemu/${vmid}/status/start" ;;
esac
}
# Stop a VM
pve_stop_vm() {
local vmid="$1"
case "$PVE_METHOD" in
ssh) pve_run "qm stop ${vmid}" ;;
api) pve_api POST "/nodes/${PVE_NODE}/qemu/${vmid}/status/stop" ;;
esac
}
# Destroy a VM
pve_destroy_vm() {
local vmid="$1"
case "$PVE_METHOD" in
ssh) pve_run "qm destroy ${vmid} --purge" ;;
api) pve_api DELETE "/nodes/${PVE_NODE}/qemu/${vmid}?purge=1" ;;
esac
}
# Set VM configuration
pve_set_vm() {
local vmid="$1"
shift
local opts="$*"
case "$PVE_METHOD" in
ssh) pve_run "qm set ${vmid} ${opts}" ;;
api)
# Convert CLI-style opts to API params
local params
params=$(echo "$opts" | sed 's/--//g; s/ /\&/g; s/\&\([a-z]\)/\&\1/g')
pve_api PUT "/nodes/${PVE_NODE}/qemu/${vmid}/config" "$params"
;;
esac
}
# Get VM status
pve_vm_status() {
local vmid="$1"
case "$PVE_METHOD" in
ssh)
pve_run "qm status ${vmid}" | awk '{print $2}'
;;
api)
pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" | \
python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status','unknown'))" 2>/dev/null || echo "unknown"
;;
esac
}
# Check if a VMID exists
pve_vm_exists() {
local vmid="$1"
if [[ "$DRY_RUN" == true ]]; then
return 1
fi
case "$PVE_METHOD" in
ssh)
pve_run "qm status ${vmid}" &>/dev/null
;;
api)
pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" &>/dev/null
;;
esac
}
# Get VM's IP address (best-effort via QEMU guest agent)
pve_vm_ip() {
local vmid="$1"
case "$PVE_METHOD" in
ssh)
pve_run "qm guest cmd ${vmid} network-get-interfaces 2>/dev/null" | \
python3 -c "
import json, sys
try:
data = json.load(sys.stdin)
for iface in data:
if iface.get('name') == 'lo':
continue
for addr in iface.get('ip-addresses', []):
if addr.get('ip-address-type') == 'ipv4':
print(addr['ip-address'])
sys.exit(0)
except:
pass
" 2>/dev/null || true
;;
api)
pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/agent/network-get-interfaces" 2>/dev/null | \
python3 -c "
import json, sys
try:
data = json.load(sys.stdin).get('data', {}).get('result', [])
for iface in data:
if iface.get('name') == 'lo':
continue
for addr in iface.get('ip-addresses', []):
if addr.get('ip-address-type') == 'ipv4':
print(addr['ip-address'])
sys.exit(0)
except:
pass
" 2>/dev/null || true
;;
esac
}
# ---------------------------------------------------------------------------
# VM management marker
# ---------------------------------------------------------------------------
# Build the description string for a managed VM
build_vm_description() {
local config_basename
config_basename=$(basename "$CONFIG_FILE")
local timestamp
timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
echo "${MANAGED_MARKER} config=${config_basename} created=${timestamp} by=${SCRIPT_NAME}/${VERSION}"
}
# Check if a VM has our management marker in its description
pve_vm_is_managed() {
local vmid="$1"
if [[ "$DRY_RUN" == true ]]; then
return 1
fi
local config_output=""
case "$PVE_METHOD" in
ssh)
config_output=$(pve_run "qm config ${vmid}" 2>/dev/null) || return 1
;;
api)
config_output=$(pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null) || return 1
;;
esac
echo "$config_output" | grep -qF "$MANAGED_MARKER"
}
# List all VMs as "vmid name" pairs (one per line)
pve_list_vms() {
if [[ "$DRY_RUN" == true ]]; then
echo ""
return 0
fi
case "$PVE_METHOD" in
ssh)
pve_run "qm list 2>/dev/null" | awk 'NR>1 {print $1, $2}' || true
;;
api)
pve_api GET "/nodes/${PVE_NODE}/qemu" 2>/dev/null | \
python3 -c "
import json, sys
data = json.load(sys.stdin)
for vm in data.get('data', []):
print('{} {}'.format(vm.get('vmid', ''), vm.get('name', '')))
" 2>/dev/null || true
;;
esac
}
# ---------------------------------------------------------------------------
# Pre-flight checks
# ---------------------------------------------------------------------------
preflight_checks() {
step "Running pre-flight checks"
local errors=0
local warnings=0
# 1. Check that storage pools exist on Proxmox
if [[ "$DRY_RUN" != true ]]; then
case "$PVE_METHOD" in
ssh)
if ! pve_run "pvesm status 2>/dev/null" | awk 'NR>1 {print \$1}' | grep -qw "$PVE_STORAGE"; then
error "Storage pool '${PVE_STORAGE}' not found on Proxmox"
error "Available pools: $(pve_run "pvesm status 2>/dev/null" | awk 'NR>1 {print \$1}' | tr '\n' ' ')"
error "Fix: update 'proxmox.storage' in your config file"
errors=$((errors + 1))
fi
if ! pve_run "pvesm status 2>/dev/null" | awk 'NR>1 {print \$1}' | grep -qw "$PVE_ISO_STORAGE"; then
error "ISO storage '${PVE_ISO_STORAGE}' not found on Proxmox"
error "Available pools: $(pve_run "pvesm status 2>/dev/null" | awk 'NR>1 {print \$1}' | tr '\n' ' ')"
error "Fix: update 'proxmox.iso_storage' in your config file"
errors=$((errors + 1))
fi
;;
api)
local storage_list
storage_list=$(pve_api GET "/nodes/${PVE_NODE}/storage" 2>/dev/null | \
python3 -c "
import json, sys
data = json.load(sys.stdin)
for s in data.get('data', []):
print(s.get('storage', ''))
" 2>/dev/null) || storage_list=""
if ! echo "$storage_list" | grep -qw "$PVE_STORAGE"; then
error "Storage pool '${PVE_STORAGE}' not found on Proxmox"
error "Available pools: $(echo "$storage_list" | tr '\n' ' ')"
error "Fix: update 'proxmox.storage' in your config file"
errors=$((errors + 1))
fi
if ! echo "$storage_list" | grep -qw "$PVE_ISO_STORAGE"; then
error "ISO storage '${PVE_ISO_STORAGE}' not found on Proxmox"
error "Available pools: $(echo "$storage_list" | tr '\n' ' ')"
error "Fix: update 'proxmox.iso_storage' in your config file"
errors=$((errors + 1))
fi
;;
esac
# 2. Network bridge check (SSH only, warn-level)
if [[ "$PVE_METHOD" == "ssh" ]]; then
if ! pve_run "ip link show ${PVE_BRIDGE} 2>/dev/null" &>/dev/null; then
warn "Network bridge '${PVE_BRIDGE}' not found on Proxmox"
warn "VMs may fail to get network connectivity"
warn "Fix: update 'proxmox.bridge' in your config file"
warnings=$((warnings + 1))
fi
fi
# 3. ISO availability: check if already on Proxmox or CDN reachable
if [[ -z "$LOCAL_ISO" ]]; then
local any_iso_present=false
local iso_list=""
case "$PVE_METHOD" in
ssh)
iso_list=$(pve_run "pvesm list ${PVE_ISO_STORAGE} --content iso 2>/dev/null" 2>/dev/null) || iso_list=""
;;
api)
iso_list=$(pve_api GET "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content?content=iso" 2>/dev/null) || iso_list=""
;;
esac
if echo "$iso_list" | grep -q "IncusOS_"; then
any_iso_present=true
fi
if [[ "$any_iso_present" != true ]]; then
# No ISO on Proxmox -- check if CDN is reachable
if ! curl -fsSL --connect-timeout 5 -o /dev/null "$CDN_INDEX" 2>/dev/null; then
error "No IncusOS ISO found on Proxmox and CDN is unreachable: ${CDN_INDEX}"
error "Fix: provide a local ISO with --iso FILE, or check internet connectivity"
errors=$((errors + 1))
fi
fi
fi
fi
# 4. FAT tools for SEED_DATA generation
if ! command -v mcopy &>/dev/null; then
error "mcopy (mtools) is required for SEED_DATA generation but not found"
error "Install mtools for your platform (e.g. mtools package)"
errors=$((errors + 1))
fi
if ! command -v mkfs.fat &>/dev/null; then
error "mkfs.fat (dosfstools) is required for SEED_DATA generation but not found"
error "Install dosfstools for your platform (e.g. dosfstools package)"
errors=$((errors + 1))
fi
if [[ $errors -gt 0 ]]; then
error "${errors} pre-flight check(s) failed"
exit 1
fi
if [[ $warnings -gt 0 ]]; then
success "Pre-flight checks passed with ${warnings} warning(s)"
else
success "Pre-flight checks passed"
fi
}
# ---------------------------------------------------------------------------
# Connectivity check
# ---------------------------------------------------------------------------
check_connectivity() {
step "Checking Proxmox connectivity (${PVE_METHOD})"
if [[ "$DRY_RUN" == true ]]; then
detail "[dry-run] Would check connectivity to ${PVE_HOST}"
success "Connectivity check skipped (dry-run)"
return 0
fi
case "$PVE_METHOD" in
ssh)
if ! ssh -o BatchMode=yes -o ConnectTimeout=10 \
"${PVE_SSH_USER}@${PVE_HOST}" "pvesh get /version --output-format json" &>/dev/null; then
error "Cannot connect to Proxmox host via SSH"
error "Ensure key-based SSH auth is set up:"
error " ssh-copy-id ${PVE_SSH_USER}@${PVE_HOST}"
error "Test with: ssh ${PVE_SSH_USER}@${PVE_HOST} pvesh get /version"
exit 1
fi
local pve_version
pve_version=$(ssh -o BatchMode=yes "${PVE_SSH_USER}@${PVE_HOST}" \
"pvesh get /version --output-format json" 2>/dev/null | \
python3 -c "import json,sys; print(json.load(sys.stdin).get('version','unknown'))" 2>/dev/null) || pve_version="unknown"
success "Connected to ${PVE_HOST} (Proxmox VE ${pve_version})"
;;
api)
local response
if ! response=$(pve_api GET "/version" 2>/dev/null); then
error "Cannot connect to Proxmox API at https://${PVE_HOST}:8006"
error "Check that:"
error " 1. The host is reachable"
error " 2. The API token ID is correct: ${PVE_API_TOKEN_ID}"
error " 3. PROXMOX_TOKEN_SECRET is set"
exit 1
fi
local pve_version
pve_version=$(echo "$response" | \
python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('version','unknown'))" 2>/dev/null) || pve_version="unknown"
success "Connected to ${PVE_HOST} (Proxmox VE ${pve_version})"
;;
esac
}
# ---------------------------------------------------------------------------
# VMID allocation
# ---------------------------------------------------------------------------
allocate_vmids() {
step "Allocating VMIDs"
# Get existing VMs as "vmid name" pairs
local vm_list
vm_list=$(pve_list_vms)
# Build lookup: existing IDs and name-to-VMID map
local existing_ids=""
declare -A existing_name_to_vmid
if [[ -n "$vm_list" ]]; then
while IFS=' ' read -r eid ename; do
[[ -z "$eid" ]] && continue
existing_ids="${existing_ids} ${eid}"
if [[ -n "$ename" ]]; then
existing_name_to_vmid["$ename"]="$eid"
fi
done <<< "$vm_list"
fi
local next_id="$DEF_START_VMID"
local count=${#VM_NAMES[@]}
local errors=0
local i=0
while [[ $i -lt $count ]]; do
local name="${VM_NAMES[$i]}"
# Skip if filtered
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
i=$((i + 1))
continue
fi
# Check for name collision with existing Proxmox VMs
if [[ -n "${existing_name_to_vmid[$name]+x}" ]]; then
local collision_vmid="${existing_name_to_vmid[$name]}"
if pve_vm_is_managed "$collision_vmid"; then
warn "VM '${name}' already exists at VMID ${collision_vmid} (managed by ${SCRIPT_NAME})"
warn "Use --cleanup to redeploy, or remove it manually"
# Assign the existing VMID so cleanup/skip works correctly
VM_VMIDS[$i]="$collision_vmid"
i=$((i + 1))
continue
else
error "VM name '${name}' already exists at VMID ${collision_vmid} but is NOT managed by ${SCRIPT_NAME}"
error "This VM was not created by this tool and will not be modified"
error "Fix: rename the VM in your config file, or manually remove VMID ${collision_vmid}:"
error " ssh ${PVE_SSH_USER}@${PVE_HOST} qm destroy ${collision_vmid} --purge"
errors=$((errors + 1))
i=$((i + 1))
continue
fi
fi
if [[ -n "${VM_VMIDS[$i]}" ]]; then
# Explicit VMID
detail "VM '${name}': VMID ${VM_VMIDS[$i]} (explicit)"
else
# Auto-assign, skipping existing
while echo "$existing_ids" | grep -qw "$next_id" 2>/dev/null; do
next_id=$((next_id + 1))
done
VM_VMIDS[$i]="$next_id"
detail "VM '${name}': VMID ${next_id} (auto-assigned)"
next_id=$((next_id + 1))
fi
i=$((i + 1))
done
if [[ $errors -gt 0 ]]; then
error "${errors} name collision(s) with unmanaged VMs -- cannot continue"
exit 1
fi
success "VMIDs allocated"
}
# ---------------------------------------------------------------------------
# Certificate detection (reused pattern from sibling scripts)
# ---------------------------------------------------------------------------
CERT_FILE_RESOLVED=""
detect_client_cert() {
if [[ "$NO_CERT" == true ]]; then
info "Skipping certificate injection (--no-cert)"
return
fi
if [[ -n "$CERT_FILE" ]]; then
if [[ ! -f "$CERT_FILE" ]]; then
error "Certificate file not found: ${CERT_FILE}"
exit 1
fi
if ! grep -q "BEGIN CERTIFICATE" "$CERT_FILE"; then
error "File does not appear to be a PEM certificate: ${CERT_FILE}"
exit 1
fi
CERT_FILE_RESOLVED="$CERT_FILE"
success "Certificate: ${CERT_FILE_RESOLVED}"
return
fi
step "Auto-detecting client certificate"
local cert_paths=(
"${HOME}/.config/incus/client.crt"
"${HOME}/snap/incus/common/config/client.crt"
"${HOME}/.config/lxc/client.crt"
)
for path in "${cert_paths[@]}"; do
if [[ -f "$path" ]] && grep -q "BEGIN CERTIFICATE" "$path"; then
CERT_FILE_RESOLVED="$path"
success "Certificate: ${CERT_FILE_RESOLVED}"
return
fi
done
# Try CLI fallback
if command -v incus &>/dev/null; then
local tmpfile="${WORKDIR}/client.crt"
if incus remote get-client-certificate > "$tmpfile" 2>/dev/null; then
CERT_FILE_RESOLVED="$tmpfile"
success "Certificate obtained via Incus CLI"
return
fi
fi
# Check if any VM is ops-center (which requires a cert)
local needs_cert=false
for app in "${VM_APPS[@]}"; do
if [[ "$app" == "operations-center" ]]; then
needs_cert=true
break
fi
done
if [[ "$needs_cert" == true ]]; then
error "No client certificate found"
error "Operations Center requires at least one trusted certificate for access"
error ""
error "Options:"
error " --cert FILE Provide a certificate file directly"
error " Generate one: incus remote list (creates ~/.config/incus/client.crt)"
error " --no-cert Skip certificate injection (you will be locked out!)"
exit 1
fi
warn "No client certificate found -- VMs will require manual trust setup"
warn "Use --cert FILE to provide a certificate, or --no-cert to suppress this warning"
}
# ---------------------------------------------------------------------------
# CDN operations (reused from incusos-iso)
# ---------------------------------------------------------------------------
LATEST_VERSION=""
fetch_latest_version() {
local channel="${DEF_CHANNEL:-stable}"
step "Fetching latest IncusOS version (channel: ${channel})"
local index_file="${WORKDIR}/cdn-index.json"
if ! curl -fsSL "$CDN_INDEX" -o "$index_file" 2>/dev/null; then
error "Failed to fetch CDN index from ${CDN_INDEX}"
error "Check your internet connection and try again"
exit 1
fi
if command -v jq &>/dev/null; then
LATEST_VERSION=$(jq -r \
--arg ch "$channel" \
'[.updates[] | select(.channels[] == $ch)] | sort_by(.version) | last | .version // empty' \
"$index_file")
else
LATEST_VERSION=$(python3 -c "
import json
data = json.load(open('${index_file}'))
versions = [u['version'] for u in data.get('updates', []) if '${channel}' in u.get('channels', [])]
if versions:
print(sorted(versions)[-1])
" 2>/dev/null)
fi
if [[ -z "$LATEST_VERSION" ]]; then
error "No version found for channel '${channel}'"
exit 1
fi
success "Latest IncusOS version: ${LATEST_VERSION}"
}
download_iso() {
local version="$1"
local output_path="$2"
local url="${CDN_BASE}/${version}/x86_64/IncusOS_${version}.iso.gz"
local gz_path="${output_path}.gz"
step "Downloading IncusOS ISO"
detail "URL: ${url}"
if ! curl -fL --progress-bar "$url" -o "$gz_path"; then
error "Failed to download ISO from ${url}"
rm -f "$gz_path"
exit 1
fi
step "Decompressing ISO"
if ! gzip -d "$gz_path"; then
error "Failed to decompress ISO"
rm -f "$gz_path"
exit 1
fi
success "ISO ready: ${output_path}"
}
# ---------------------------------------------------------------------------
# Deployment plan summary
# ---------------------------------------------------------------------------
print_plan() {
local count=${#VM_NAMES[@]}
echo ""
echo -e "${BOLD}Deployment Plan${RESET}"
echo -e " Proxmox host: ${CYAN}${PVE_HOST}${RESET} (${PVE_METHOD})"
echo -e " Node: ${PVE_NODE}"
echo -e " Storage: ${PVE_STORAGE} (disk), ${PVE_ISO_STORAGE} (iso)"
echo -e " Bridge: ${PVE_BRIDGE}"
if [[ -n "$CERT_FILE_RESOLVED" ]]; then
echo -e " Certificate: ${GREEN}${CERT_FILE_RESOLVED}${RESET}"
elif [[ "$NO_CERT" == true ]]; then
echo -e " Certificate: ${DIM}none (--no-cert)${RESET}"
else
echo -e " Certificate: ${YELLOW}not found${RESET}"
fi
echo ""
echo -e " ${BOLD}VMs (${count}):${RESET}"
local i=0
while [[ $i -lt $count ]]; do
local name="${VM_NAMES[$i]}"
local skip=""
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
skip=" ${DIM}(skipped)${RESET}"
fi
echo -e " ${CYAN}${name}${RESET}${skip}"
echo -e " VMID: ${VM_VMIDS[$i]} App: ${VM_APPS[$i]} Defaults: ${VM_APPLY_DEFAULTS[$i]}"
echo -e " CPU: ${VM_CORES[$i]} cores RAM: ${VM_MEMORY[$i]} MiB Disk: ${VM_DISK[$i]} GiB"
if [[ -n "${VM_CLUSTERS[$i]}" ]]; then
echo -e " Cluster: ${VM_CLUSTERS[$i]} (${VM_CLUSTER_ROLES[$i]})"
fi
i=$((i + 1))
done
echo ""
}
# ---------------------------------------------------------------------------
# Confirmation prompt
# ---------------------------------------------------------------------------
confirm() {
local message="$1"
if [[ "$YES" == true ]] || [[ "$DRY_RUN" == true ]]; then
return 0
fi
echo -n -e "${BOLD}${message} [y/N] ${RESET}"
local reply
read -r reply
if [[ ! "$reply" =~ ^[Yy]$ ]]; then
info "Aborted"
exit 0
fi
}
# ---------------------------------------------------------------------------
# Phase 1: Validate
# ---------------------------------------------------------------------------
phase_validate() {
step "Phase 1: Validate"
echo ""
parse_config
load_config
validate_config
check_connectivity
allocate_vmids
preflight_checks
detect_client_cert
print_plan
}
# ---------------------------------------------------------------------------
# Phase 2: Prepare (download ISO + generate seeds)
# ---------------------------------------------------------------------------
ISO_FILENAME=""
phase_prepare() {
step "Phase 2: Prepare"
echo ""
# Determine ISO
if [[ -n "$LOCAL_ISO" ]]; then
ISO_FILENAME=$(basename "$LOCAL_ISO")
info "Using local ISO: ${LOCAL_ISO}"
if [[ "$DRY_RUN" != true ]]; then
cp "$LOCAL_ISO" "${WORKDIR}/${ISO_FILENAME}"
fi
elif [[ "$DRY_RUN" == true ]]; then
ISO_FILENAME="IncusOS_latest.iso"
detail "[dry-run] Would fetch latest version from CDN and download ISO"
else
# Check if ISO already exists on Proxmox
fetch_latest_version
ISO_FILENAME="IncusOS_${LATEST_VERSION}.iso"
if pve_iso_exists "$ISO_FILENAME"; then
success "ISO already on Proxmox: ${ISO_FILENAME} (reusing from previous deployment)"
else
download_iso "$LATEST_VERSION" "${WORKDIR}/${ISO_FILENAME}"
fi
fi
# Generate per-VM seed images
step "Generating seed images"
local count=${#VM_NAMES[@]}
local i=0
while [[ $i -lt $count ]]; do
local name="${VM_NAMES[$i]}"
# Skip if filtered
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
i=$((i + 1))
continue
fi
local app="${VM_APPS[$i]}"
local apply_defaults="${VM_APPLY_DEFAULTS[$i]}"
local seed_output="${WORKDIR}/seed-${name}.img"
detail "Generating seed for '${name}' (${app}, defaults=${apply_defaults})"
# Build incusos-seed command
local seed_cmd=("$SEED_TOOL")
seed_cmd+=(--format fat)
seed_cmd+=(--app "$app")
seed_cmd+=(--output "$seed_output")
seed_cmd+=(--disk-target "scsi-*")
seed_cmd+=(--force-install)
seed_cmd+=(--force-reboot)
seed_cmd+=(--hostname "$name")
seed_cmd+=(--quiet)
if [[ "$apply_defaults" == "true" ]]; then
seed_cmd+=(--defaults)
fi
if [[ -n "$CERT_FILE_RESOLVED" ]]; then
seed_cmd+=(--cert "$CERT_FILE_RESOLVED")
elif [[ "$NO_CERT" == true ]]; then
seed_cmd+=(--no-cert)
fi
if [[ "$DRY_RUN" == true ]]; then
detail "[dry-run] ${seed_cmd[*]}"
else
if ! "${seed_cmd[@]}"; then
error "Failed to generate seed for VM '${name}'"
exit 1
fi
fi
i=$((i + 1))
done
success "Seed images generated"
}
# ---------------------------------------------------------------------------
# Phase 3: Upload
# ---------------------------------------------------------------------------
phase_upload() {
step "Phase 3: Upload"
echo ""
# Upload ISO (if we downloaded it)
local iso_path="${WORKDIR}/${ISO_FILENAME}"
if [[ -f "$iso_path" ]]; then
if pve_iso_exists "$ISO_FILENAME"; then
success "ISO already on Proxmox: ${ISO_FILENAME}"
else
step "Uploading ISO: ${ISO_FILENAME}"
if ! pve_upload "$iso_path" "$ISO_FILENAME"; then
error "Failed to upload ISO to Proxmox"
exit 1
fi
success "ISO uploaded: ${ISO_FILENAME}"
fi
else
detail "ISO already on Proxmox, skipping upload"
fi
# Upload seed images
local count=${#VM_NAMES[@]}
local i=0
while [[ $i -lt $count ]]; do
local name="${VM_NAMES[$i]}"
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
i=$((i + 1))
continue
fi
local seed_file="seed-${name}.img"
local seed_path="${WORKDIR}/${seed_file}"
if [[ -f "$seed_path" ]] || [[ "$DRY_RUN" == true ]]; then
step "Uploading seed: ${seed_file}"
if ! pve_upload "$seed_path" "$seed_file"; then
error "Failed to upload seed for VM '${name}'"
exit 1
fi
success "Seed uploaded: ${seed_file}"
fi
i=$((i + 1))
done
success "All files uploaded"
}
# ---------------------------------------------------------------------------
# Phase 4: Create VMs
# ---------------------------------------------------------------------------
phase_create() {
step "Phase 4: Create VMs"
echo ""
local count=${#VM_NAMES[@]}
local i=0
while [[ $i -lt $count ]]; do
local name="${VM_NAMES[$i]}"
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
i=$((i + 1))
continue
fi
local vmid="${VM_VMIDS[$i]}"
local cores="${VM_CORES[$i]}"
local memory="${VM_MEMORY[$i]}"
local disk="${VM_DISK[$i]}"
local seed_file="seed-${name}.img"
step "Creating VM '${name}' (VMID ${vmid})"
# Check if VM already exists
if pve_vm_exists "$vmid"; then
if pve_vm_is_managed "$vmid"; then
warn "VM ${vmid} ('${name}') already exists (managed) -- skipping"
warn "Use --cleanup to redeploy"
else
warn "VM ${vmid} ('${name}') already exists (NOT managed by ${SCRIPT_NAME}) -- skipping"
warn "Choose a different VMID or remove it manually:"
warn " ssh ${PVE_SSH_USER}@${PVE_HOST} qm destroy ${vmid} --purge"
fi
i=$((i + 1))
continue
fi
detail "cores=${cores} memory=${memory}MiB disk=${disk}GiB"
detail "bios=ovmf machine=q35 cpu=host scsihw=virtio-scsi-pci"
detail "efidisk: pre-enrolled-keys=0, tpm: v2.0, balloon: 0"
detail "iso=${ISO_FILENAME} seed=${seed_file}"
if ! pve_create_vm "$vmid" "$name" "$cores" "$memory" "$disk" \
"$ISO_FILENAME" "$seed_file"; then
error "Failed to create VM '${name}' (VMID ${vmid})"
exit 1
fi
success "VM '${name}' created (VMID ${vmid})"
i=$((i + 1))
done
success "All VMs created"
}
# ---------------------------------------------------------------------------
# Phase 5: Install
# ---------------------------------------------------------------------------
phase_install() {
step "Phase 5: Install"
echo ""
local count=${#VM_NAMES[@]}
local i=0
while [[ $i -lt $count ]]; do
local name="${VM_NAMES[$i]}"
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
i=$((i + 1))
continue
fi
local vmid="${VM_VMIDS[$i]}"
step "Starting VM '${name}' (VMID ${vmid})"
if [[ "$DRY_RUN" == true ]]; then
detail "[dry-run] Would start VM, swap boot order, wait for install, clean up"
i=$((i + 1))
continue
fi
# 1. Start VM (boots from ISO, reads SEED_DATA, installs)
if ! pve_start_vm "$vmid"; then
error "Failed to start VM '${name}' (VMID ${vmid})"
i=$((i + 1))
continue
fi
success "VM '${name}' started"
# 2. Immediately swap boot order (takes effect on next reboot)
detail "Swapping boot order to disk-first"
if ! pve_set_vm "$vmid" "--boot order=scsi0\\;ide2"; then
warn "Failed to swap boot order for '${name}' -- manual intervention may be needed"
fi
# 3. Wait for installation + reboot cycle
step "Waiting for '${name}' to complete installation"
detail "IncusOS installs from ISO, then reboots to disk automatically"
detail "Polling every 15s (timeout: 10 minutes)"
local timeout=600
local interval=15
local elapsed=0
local booted=false
# Wait for the VM to go through: running -> (install) -> stopped -> running
# The force_reboot in seed causes an automatic reboot after install.
# We detect the reboot by looking for the VM to be "running" after
# having been "stopped" (or we just wait for it to stabilize).
local saw_stop=false
while [[ $elapsed -lt $timeout ]]; do
sleep "$interval"
elapsed=$((elapsed + interval))
local status
status=$(pve_vm_status "$vmid") || status="unknown"
case "$status" in
stopped)
if [[ "$saw_stop" == false ]]; then
detail "VM stopped (installation complete, awaiting reboot) [${elapsed}s]"
saw_stop=true
# The VM should auto-reboot via force_reboot, but if it
# doesn't after 30s, start it manually
sleep 30
elapsed=$((elapsed + 30))
local recheck
recheck=$(pve_vm_status "$vmid") || recheck="unknown"
if [[ "$recheck" == "stopped" ]]; then
detail "VM did not auto-reboot, starting manually"
pve_start_vm "$vmid" || true
fi
fi
;;
running)
if [[ "$saw_stop" == true ]] || [[ $elapsed -ge 120 ]]; then
# VM has been running long enough post-reboot -- likely booted from disk
booted=true
break
fi
detail "VM running (installing...) [${elapsed}s]"
;;
*)
detail "VM status: ${status} [${elapsed}s]"
;;
esac
done
if [[ "$booted" != true ]]; then
warn "VM '${name}' did not complete installation within ${timeout}s"
warn "Check the VM console on Proxmox for details"
i=$((i + 1))
continue
fi
# 4. Give it a moment to fully initialize
detail "Waiting for services to start..."
sleep 15
# 5. Remove CD-ROM drives
step "Cleaning up '${name}'"
pve_set_vm "$vmid" "--delete ide2 --delete ide3" || true
# 6. Try to detect IP
local ip
ip=$(pve_vm_ip "$vmid") || ip=""
echo ""
if [[ -n "$ip" ]]; then
success "VM '${name}' installed and running at ${ip}"
if [[ -n "$CERT_FILE_RESOLVED" ]]; then
detail "Add remote: incus remote add ${name} ${ip}"
fi
else
success "VM '${name}' installed and running"
detail "Check Proxmox console for IP address"
fi
echo ""
i=$((i + 1))
done
success "Installation phase complete"
}
# ---------------------------------------------------------------------------
# Cleanup mode
# ---------------------------------------------------------------------------
do_cleanup() {
step "Cleanup: destroying VMs defined in config"
if [[ "$FORCE_CLEANUP" == true ]]; then
warn "Force cleanup enabled -- will destroy VMs even without management marker"
fi
echo ""
local count=${#VM_NAMES[@]}
local destroyed=0
local not_found=0
local skipped_unmanaged=0
local i=0
while [[ $i -lt $count ]]; do
local name="${VM_NAMES[$i]}"
if [[ -n "$VM_FILTER" ]] && [[ "$name" != "$VM_FILTER" ]]; then
i=$((i + 1))
continue
fi
local vmid="${VM_VMIDS[$i]}"
# 1. Check if VM exists
if ! pve_vm_exists "$vmid"; then
detail "VM '${name}' (VMID ${vmid}): does not exist"
not_found=$((not_found + 1))
i=$((i + 1))
continue
fi
# 2. Check if VM is managed by us
if [[ "$DRY_RUN" != true ]]; then
if ! pve_vm_is_managed "$vmid"; then
if [[ "$FORCE_CLEANUP" == true ]]; then
warn "VM '${name}' (VMID ${vmid}): NOT managed by ${SCRIPT_NAME} -- destroying anyway (--force-cleanup)"
else
warn "VM '${name}' (VMID ${vmid}): NOT managed by ${SCRIPT_NAME} -- refusing to destroy"
warn " This VM was not created by this tool. To destroy it manually:"
warn " ssh ${PVE_SSH_USER}@${PVE_HOST} qm stop ${vmid} && ssh ${PVE_SSH_USER}@${PVE_HOST} qm destroy ${vmid} --purge"
warn " Or use --force-cleanup to override this safety check"
skipped_unmanaged=$((skipped_unmanaged + 1))
i=$((i + 1))
continue
fi
fi
fi
# 3. Destroy the VM
step "Destroying VM '${name}' (VMID ${vmid})"
if [[ "$DRY_RUN" == true ]]; then
detail "[dry-run] Would stop and destroy VM ${vmid}"
destroyed=$((destroyed + 1))
i=$((i + 1))
continue
fi
# Stop if running
local status
status=$(pve_vm_status "$vmid") || status="unknown"
if [[ "$status" == "running" ]]; then
detail "Stopping VM..."
pve_stop_vm "$vmid" || true
sleep 5
fi
# Destroy
if pve_destroy_vm "$vmid"; then
success "VM '${name}' (VMID ${vmid}) destroyed"
destroyed=$((destroyed + 1))
else
error "Failed to destroy VM '${name}' (VMID ${vmid})"
fi
i=$((i + 1))
done
# Summary
echo ""
local summary="Cleanup complete: ${destroyed} destroyed"
if [[ $not_found -gt 0 ]]; then
summary+=", ${not_found} not found"
fi
if [[ $skipped_unmanaged -gt 0 ]]; then
summary+=", ${skipped_unmanaged} skipped (unmanaged)"
fi
success "$summary"
}
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
main() {
setup_colors
parse_args "$@"
validate_args
if [[ "$QUIET" != true ]]; then
echo -e "${BOLD}${SCRIPT_NAME}${RESET} v${VERSION}"
echo ""
fi
setup_workdir
if [[ "$DRY_RUN" != true ]]; then
check_dependencies
fi
echo ""
# Always run validate first
phase_validate
# Handle cleanup mode
if [[ "$CLEANUP" == true ]]; then
confirm "Destroy VMs listed above?"
do_cleanup
return
fi
# Confirm before proceeding
if [[ "$PHASE" == "all" ]]; then
confirm "Proceed with deployment?"
fi
# Run requested phases
case "$PHASE" in
validate)
# Already done above
;;
prepare)
phase_prepare
;;
upload)
phase_prepare
phase_upload
;;
create)
phase_prepare
phase_upload
phase_create
;;
install)
phase_prepare
phase_upload
phase_create
phase_install
;;
all)
phase_prepare
echo ""
phase_upload
echo ""
phase_create
echo ""
phase_install
;;
esac
echo ""
if [[ "$DRY_RUN" == true ]]; then
info "Dry run complete -- no changes were made"
else
success "Done"
fi
}
main "$@"
Reference in New Issue View Git Blame Copy Permalink