1042 lines
33 KiB
Bash
Executable File
1042 lines
33 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# observe-deploy - Single-VM IncusOS deploy with rapid console screenshots
|
|
#
|
|
# Captures the complete IncusOS lifecycle on screen — from ISO boot through
|
|
# installation, reboot, first-boot sequence, and port 8443 readiness. Takes
|
|
# screenshots every 2 seconds via Proxmox SSH root access (screendump).
|
|
#
|
|
# Purpose: diagnose boot failures by building a frame-by-frame timeline of
|
|
# exactly what IncusOS shows at every stage. NOT a batch test — one VM at
|
|
# a time, maximum visibility.
|
|
#
|
|
# Usage:
|
|
# ./observe-deploy # Standard deploy with screenshots
|
|
# ./observe-deploy --no-update-yaml # Omit update.yaml from seed
|
|
# ./observe-deploy --label run-01 # Custom run label
|
|
# ./observe-deploy --vmid 855 # Use a specific VMID
|
|
# ./observe-deploy --dry-run # Preview without deploying
|
|
#
|
|
# Output: timestamped directory with PNG frames + deploy.log
|
|
# observe-runs/YYYY-MM-DD_HH-MM-SS_<label>/
|
|
# frame-001.png
|
|
# frame-002.png
|
|
# ...
|
|
# deploy.log
|
|
|
|
set -euo pipefail
|
|
|
|
readonly VERSION="0.1.0"
|
|
readonly SCRIPT_NAME="observe-deploy"
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
SEED_TOOL="${SCRIPT_DIR}/incusos-seed"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Defaults
|
|
# ---------------------------------------------------------------------------
|
|
|
|
VMID=850
|
|
VM_NAME="observe-deploy"
|
|
LABEL=""
|
|
NO_UPDATE_YAML=false
|
|
DRY_RUN=false
|
|
SCREENSHOT_INTERVAL=2 # seconds between screenshots
|
|
INSTALL_TIMEOUT=600 # seconds to wait for install completion
|
|
BOOT_TIMEOUT=180 # seconds to wait for port 8443 after disk boot
|
|
INSTALL_POLL=5 # blockstat poll interval during install phase
|
|
STABLE_THRESHOLD=3 # consecutive stable polls = install done
|
|
|
|
# Proxmox connection (from proxmox.yaml)
|
|
PVE_HOST=""
|
|
PVE_NODE=""
|
|
PVE_STORAGE=""
|
|
PVE_ISO_STORAGE=""
|
|
PVE_BRIDGE=""
|
|
PVE_POOL=""
|
|
PVE_API_TOKEN_ID=""
|
|
|
|
# IncusOS ISO
|
|
ISO_FILENAME=""
|
|
|
|
# Run output
|
|
RUN_DIR=""
|
|
LOG_FILE=""
|
|
FRAME_NUM=0
|
|
|
|
# State tracking
|
|
PHASE="init" # init → iso_boot → installing → install_done → transitioning → disk_boot → ready/failed
|
|
PREV_WR_BYTES=0
|
|
WRITES_STARTED=false
|
|
STABLE_COUNT=0
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Colors
|
|
# ---------------------------------------------------------------------------
|
|
|
|
setup_colors() {
|
|
if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[0;33m'
|
|
BLUE='\033[0;34m'
|
|
CYAN='\033[0;36m'
|
|
BOLD='\033[1m'
|
|
DIM='\033[2m'
|
|
RESET='\033[0m'
|
|
else
|
|
RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' RESET=''
|
|
fi
|
|
}
|
|
setup_colors
|
|
|
|
step() { echo -e "${CYAN}[step]${RESET} ${BOLD}$*${RESET}"; }
|
|
success() { echo -e "${GREEN}[ok]${RESET} $*"; }
|
|
warn() { echo -e "${YELLOW}[warn]${RESET} $*" >&2; }
|
|
error() { echo -e "${RED}[error]${RESET} $*" >&2; }
|
|
info() { echo -e "${BLUE}[info]${RESET} $*"; }
|
|
detail() { echo -e "${DIM} $*${RESET}"; }
|
|
|
|
# Log to both console and file
|
|
log() {
|
|
local ts
|
|
ts=$(date '+%H:%M:%S.%3N')
|
|
local msg="[${ts}] $*"
|
|
echo -e "${DIM}${msg}${RESET}"
|
|
[[ -n "$LOG_FILE" ]] && echo "$msg" >> "$LOG_FILE"
|
|
}
|
|
|
|
# Log to file only (for high-frequency events)
|
|
log_quiet() {
|
|
local ts
|
|
ts=$(date '+%H:%M:%S.%3N')
|
|
[[ -n "$LOG_FILE" ]] && echo "[${ts}] $*" >> "$LOG_FILE"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Argument parsing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--vmid) VMID="${2:?'--vmid requires a number'}"; shift 2 ;;
|
|
--label) LABEL="${2:?'--label requires a string'}"; shift 2 ;;
|
|
--no-update-yaml) NO_UPDATE_YAML=true; shift ;;
|
|
--interval) SCREENSHOT_INTERVAL="${2:?'--interval requires seconds'}"; shift 2 ;;
|
|
--dry-run) DRY_RUN=true; shift ;;
|
|
-h|--help)
|
|
cat <<EOF
|
|
${BOLD}${SCRIPT_NAME}${RESET} v${VERSION} - Observational IncusOS deploy with screenshots
|
|
|
|
${BOLD}USAGE${RESET}
|
|
${SCRIPT_NAME} [OPTIONS]
|
|
|
|
${BOLD}OPTIONS${RESET}
|
|
--vmid N VMID to use (default: 850, must be 850-869)
|
|
--label NAME Run label for output directory (default: auto)
|
|
--no-update-yaml Omit update.yaml from seed (Group A equivalent)
|
|
--interval N Screenshot interval in seconds (default: 2)
|
|
--dry-run Preview without deploying
|
|
-h, --help Show this help
|
|
|
|
${BOLD}OUTPUT${RESET}
|
|
Screenshots and logs are saved to:
|
|
observe-runs/<timestamp>_<label>/
|
|
frame-001.png ... frame-NNN.png
|
|
deploy.log
|
|
|
|
${BOLD}REQUIREMENTS${RESET}
|
|
- proxmox.yaml with Proxmox connection settings
|
|
- env file with PROXMOX_TOKEN_SECRET and PROXMOX_ROOT_PASSWORD
|
|
- IncusOS ISO already uploaded to Proxmox (use incusos-proxmox --phase upload)
|
|
- python3 with PIL/Pillow for PPM→PNG conversion
|
|
- sshpass for root SSH to Proxmox host
|
|
|
|
EOF
|
|
exit 0
|
|
;;
|
|
*) error "Unknown option: $1"; exit 1 ;;
|
|
esac
|
|
done
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Load environment
|
|
# ---------------------------------------------------------------------------
|
|
|
|
load_env() {
|
|
local dir="$SCRIPT_DIR"
|
|
while [[ "$dir" != "/" ]]; do
|
|
if [[ -f "${dir}/env" ]]; then
|
|
# shellcheck disable=SC1091
|
|
source "${dir}/env"
|
|
return
|
|
fi
|
|
dir="$(dirname "$dir")"
|
|
done
|
|
}
|
|
load_env
|
|
|
|
if [[ -z "${PROXMOX_TOKEN_SECRET:-}" ]]; then
|
|
error "PROXMOX_TOKEN_SECRET not set. Create an env file or export it."
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -z "${PROXMOX_ROOT_PASSWORD:-}" ]]; then
|
|
error "PROXMOX_ROOT_PASSWORD not set. Add it to the env file."
|
|
error "Required for SSH root screenshots (screendump)."
|
|
exit 1
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Load proxmox.yaml
|
|
# ---------------------------------------------------------------------------
|
|
|
|
yaml_get() {
|
|
local file="$1" key="$2" default="${3:-}"
|
|
local val
|
|
val=$(awk -F': ' -v k="$key" '$1 == k {print $2; exit}' "$file" 2>/dev/null) || val=""
|
|
val="${val#\"}" ; val="${val%\"}"
|
|
val="${val#\'}" ; val="${val%\'}"
|
|
echo "${val:-$default}"
|
|
}
|
|
|
|
load_proxmox_yaml() {
|
|
local yaml_file=""
|
|
if [[ -f "${SCRIPT_DIR}/proxmox.yaml" ]]; then
|
|
yaml_file="${SCRIPT_DIR}/proxmox.yaml"
|
|
elif [[ -f "proxmox.yaml" ]]; then
|
|
yaml_file="proxmox.yaml"
|
|
else
|
|
error "proxmox.yaml not found"
|
|
exit 1
|
|
fi
|
|
|
|
PVE_HOST=$(yaml_get "$yaml_file" "host")
|
|
PVE_NODE=$(yaml_get "$yaml_file" "node" "pve")
|
|
PVE_STORAGE=$(yaml_get "$yaml_file" "storage" "local-lvm")
|
|
PVE_ISO_STORAGE=$(yaml_get "$yaml_file" "iso_storage" "local")
|
|
PVE_BRIDGE=$(yaml_get "$yaml_file" "bridge" "vmbr0")
|
|
PVE_POOL=$(yaml_get "$yaml_file" "pool")
|
|
PVE_API_TOKEN_ID=$(yaml_get "$yaml_file" "api_token_id")
|
|
|
|
if [[ -z "$PVE_HOST" ]]; then
|
|
error "proxmox.yaml: 'host' is required"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Proxmox API helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
pve_api() {
|
|
local method="$1"
|
|
local endpoint="$2"
|
|
local data="${3:-}"
|
|
local content_type="${4:-form}"
|
|
|
|
local url="https://${PVE_HOST}:8006/api2/json${endpoint}"
|
|
local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}"
|
|
|
|
local curl_args=(-fsSk -H "$auth")
|
|
if [[ "$content_type" == "json" ]]; then
|
|
curl_args+=(-H "Content-Type: application/json")
|
|
fi
|
|
|
|
case "$method" in
|
|
GET) curl "${curl_args[@]}" "$url" ;;
|
|
POST) curl "${curl_args[@]}" -X POST ${data:+-d "$data"} "$url" ;;
|
|
PUT) curl "${curl_args[@]}" -X PUT ${data:+-d "$data"} "$url" ;;
|
|
DELETE) curl "${curl_args[@]}" -X DELETE "$url" ;;
|
|
esac
|
|
}
|
|
|
|
pve_vm_exists() {
|
|
local vmid="$1"
|
|
pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" &>/dev/null
|
|
}
|
|
|
|
pve_vm_status() {
|
|
local vmid="$1"
|
|
pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" 2>/dev/null | \
|
|
python3 -c "import json,sys; print(json.load(sys.stdin)['data']['status'])" 2>/dev/null
|
|
}
|
|
|
|
pve_start_vm() {
|
|
local vmid="$1"
|
|
pve_api POST "/nodes/${PVE_NODE}/qemu/${vmid}/status/start" &>/dev/null
|
|
}
|
|
|
|
pve_stop_vm() {
|
|
local vmid="$1"
|
|
pve_api POST "/nodes/${PVE_NODE}/qemu/${vmid}/status/stop" &>/dev/null
|
|
}
|
|
|
|
pve_destroy_vm() {
|
|
local vmid="$1"
|
|
local destroy_response
|
|
destroy_response=$(pve_api DELETE "/nodes/${PVE_NODE}/qemu/${vmid}?purge=1&destroy-unreferenced-disks=1" 2>/dev/null) || return 1
|
|
# Wait for async destroy task to complete (avoids VMID reuse issues)
|
|
local upid
|
|
upid=$(echo "$destroy_response" | python3 -c "import json,sys; print(json.load(sys.stdin).get('data',''))" 2>/dev/null) || upid=""
|
|
if [[ -n "$upid" ]]; then
|
|
local encoded_upid
|
|
encoded_upid=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${upid}', safe=''))" 2>/dev/null)
|
|
local wait_elapsed=0
|
|
while [[ $wait_elapsed -lt 30 ]]; do
|
|
sleep 2
|
|
wait_elapsed=$((wait_elapsed + 2))
|
|
local task_status
|
|
task_status=$(pve_api GET "/nodes/${PVE_NODE}/tasks/${encoded_upid}/status" 2>/dev/null | \
|
|
python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status',''))" 2>/dev/null) || task_status=""
|
|
if [[ "$task_status" == "stopped" ]]; then
|
|
break
|
|
fi
|
|
done
|
|
fi
|
|
}
|
|
|
|
pve_vm_disk_writes() {
|
|
local vmid="$1"
|
|
pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" 2>/dev/null | \
|
|
python3 -c "
|
|
import json,sys
|
|
d = json.load(sys.stdin).get('data',{})
|
|
bs = d.get('blockstat',{}).get('scsi0',{})
|
|
print(bs.get('wr_bytes',0))
|
|
" 2>/dev/null || echo "0"
|
|
}
|
|
|
|
pve_vm_mac() {
|
|
local vmid="$1"
|
|
pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null | \
|
|
python3 -c "
|
|
import json, sys, re
|
|
data = json.load(sys.stdin).get('data', {})
|
|
net0 = data.get('net0', '')
|
|
m = re.search(r'([0-9A-Fa-f:]{17})', net0)
|
|
print(m.group(1).lower() if m else '')
|
|
" 2>/dev/null
|
|
}
|
|
|
|
pve_vm_ip() {
|
|
local vmid="$1"
|
|
local mac
|
|
mac=$(pve_vm_mac "$vmid") || return 1
|
|
[[ -z "$mac" ]] && return 1
|
|
|
|
ip neigh flush nud stale 2>/dev/null || true
|
|
|
|
local bridge_subnet
|
|
bridge_subnet=$(ip -4 addr show | awk '/inet / && !/127\.0\.0/ {print $2; exit}')
|
|
|
|
if [[ -n "$bridge_subnet" ]]; then
|
|
local broadcast
|
|
broadcast=$(python3 -c "import ipaddress; print(ipaddress.ip_interface('${bridge_subnet}').network.broadcast_address)" 2>/dev/null) || true
|
|
if [[ -n "$broadcast" ]]; then
|
|
ping -c 2 -W 1 -b "$broadcast" &>/dev/null || true
|
|
sleep 1
|
|
fi
|
|
fi
|
|
|
|
local found_ip
|
|
found_ip=$(ip neigh show | awk -v mac="$mac" 'tolower($0) ~ tolower(mac) && (/REACHABLE/ || /DELAY/ || /STALE/) {print $1; exit}') || true
|
|
|
|
if [[ -n "$found_ip" ]] && ping -c 1 -W 1 "$found_ip" &>/dev/null; then
|
|
echo "$found_ip"
|
|
return 0
|
|
fi
|
|
|
|
# Parallel ping sweep fallback
|
|
if [[ -n "$bridge_subnet" ]]; then
|
|
local hosts
|
|
hosts=$(python3 -c "
|
|
import ipaddress
|
|
net = ipaddress.ip_interface('${bridge_subnet}').network
|
|
for h in net.hosts():
|
|
print(h)
|
|
" 2>/dev/null)
|
|
if [[ -n "$hosts" ]]; then
|
|
while IFS= read -r host_ip; do
|
|
ping -c 1 -W 1 "$host_ip" &>/dev/null &
|
|
done <<< "$hosts"
|
|
wait
|
|
sleep 1
|
|
|
|
found_ip=$(ip neigh show | awk -v mac="$mac" 'tolower($0) ~ tolower(mac) && (/REACHABLE/ || /DELAY/) {print $1; exit}') || true
|
|
|
|
if [[ -n "$found_ip" ]] && ping -c 1 -W 1 "$found_ip" &>/dev/null; then
|
|
echo "$found_ip"
|
|
return 0
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
return 1
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Screenshot capture
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# SSH options for root access to Proxmox (screenshots only)
|
|
SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -o LogLevel=ERROR)
|
|
|
|
capture_screenshot() {
|
|
# Captures a single frame from the VM console.
|
|
# Returns 0 on success, 1 on failure (VM not running, SSH error, etc.)
|
|
local vmid="$1"
|
|
local output_png="$2"
|
|
|
|
local remote_ppm="/tmp/vm-${vmid}-screen.ppm"
|
|
local local_ppm="/tmp/vm-${vmid}-screen.ppm"
|
|
|
|
# Take screenshot via QEMU monitor (screendump)
|
|
if ! sshpass -p "$PROXMOX_ROOT_PASSWORD" ssh "${SSH_OPTS[@]}" \
|
|
"root@${PVE_HOST}" \
|
|
"echo 'screendump ${remote_ppm}' | qm monitor ${vmid}" &>/dev/null; then
|
|
return 1
|
|
fi
|
|
|
|
# Retrieve PPM file
|
|
if ! sshpass -p "$PROXMOX_ROOT_PASSWORD" scp "${SSH_OPTS[@]}" \
|
|
"root@${PVE_HOST}:${remote_ppm}" "$local_ppm" &>/dev/null; then
|
|
return 1
|
|
fi
|
|
|
|
# Cleanup remote file
|
|
sshpass -p "$PROXMOX_ROOT_PASSWORD" ssh "${SSH_OPTS[@]}" \
|
|
"root@${PVE_HOST}" "rm -f ${remote_ppm}" &>/dev/null || true
|
|
|
|
# Convert PPM → PNG
|
|
if ! python3 -c "from PIL import Image; Image.open('${local_ppm}').save('${output_png}')" 2>/dev/null; then
|
|
# If PIL not available, keep PPM
|
|
mv "$local_ppm" "${output_png%.png}.ppm" 2>/dev/null || true
|
|
return 0
|
|
fi
|
|
|
|
rm -f "$local_ppm"
|
|
return 0
|
|
}
|
|
|
|
take_frame() {
|
|
# Capture a numbered frame and log it
|
|
FRAME_NUM=$((FRAME_NUM + 1))
|
|
local frame_name
|
|
frame_name=$(printf "frame-%03d.png" "$FRAME_NUM")
|
|
local frame_path="${RUN_DIR}/${frame_name}"
|
|
|
|
if capture_screenshot "$VMID" "$frame_path"; then
|
|
log_quiet "Frame ${FRAME_NUM}: ${frame_name} (phase=${PHASE})"
|
|
return 0
|
|
else
|
|
log_quiet "Frame ${FRAME_NUM}: FAILED (phase=${PHASE})"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Find IncusOS ISO on Proxmox
|
|
# ---------------------------------------------------------------------------
|
|
|
|
find_incusos_iso() {
|
|
step "Finding IncusOS ISO on Proxmox"
|
|
local iso_list
|
|
iso_list=$(pve_api GET "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content?content=iso" 2>/dev/null) || iso_list=""
|
|
|
|
ISO_FILENAME=$(echo "$iso_list" | python3 -c "
|
|
import json, sys
|
|
data = json.load(sys.stdin).get('data', [])
|
|
isos = [d['volid'].split('/')[-1] for d in data if 'IncusOS_' in d.get('volid', '')]
|
|
if isos:
|
|
isos.sort()
|
|
print(isos[-1])
|
|
" 2>/dev/null) || ISO_FILENAME=""
|
|
|
|
if [[ -z "$ISO_FILENAME" ]]; then
|
|
error "No IncusOS ISO found on Proxmox storage '${PVE_ISO_STORAGE}'"
|
|
error "Deploy a lab first to upload the ISO, or use incusos-proxmox --phase upload"
|
|
exit 1
|
|
fi
|
|
|
|
success "Using ISO: ${ISO_FILENAME}"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Seed generation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
SEED_ISO=""
|
|
SEED_TMPDIR=""
|
|
|
|
generate_seed() {
|
|
step "Generating seed ISO"
|
|
SEED_TMPDIR=$(mktemp -d)
|
|
local seed_name="seed-${VM_NAME}.iso"
|
|
SEED_ISO="$seed_name"
|
|
local local_path="${SEED_TMPDIR}/${seed_name}"
|
|
|
|
if [[ "$NO_UPDATE_YAML" == true ]]; then
|
|
info "Omitting update.yaml from seed (Group A equivalent)"
|
|
|
|
# Generate full seed as tar, extract, remove update.yaml, create ISO
|
|
"$SEED_TOOL" --format tar --app incus --output "${SEED_TMPDIR}/seed.tar" \
|
|
--force-install --force-reboot --hostname "$VM_NAME" \
|
|
--defaults --quiet &>/dev/null
|
|
|
|
local seeddir="${SEED_TMPDIR}/seedfiles"
|
|
mkdir -p "$seeddir"
|
|
tar xf "${SEED_TMPDIR}/seed.tar" -C "$seeddir"
|
|
rm -f "${seeddir}/update.yaml"
|
|
|
|
# Create ISO from modified seed directory
|
|
if command -v genisoimage &>/dev/null; then
|
|
genisoimage -V SEED_DATA -J -r -o "$local_path" "$seeddir" 2>/dev/null
|
|
elif command -v mkisofs &>/dev/null; then
|
|
mkisofs -V SEED_DATA -J -r -o "$local_path" "$seeddir" 2>/dev/null
|
|
else
|
|
error "genisoimage or mkisofs required for ISO creation"
|
|
exit 1
|
|
fi
|
|
|
|
# Log seed contents
|
|
log "Seed contents (no update.yaml):"
|
|
for f in "$seeddir"/*.yaml; do
|
|
[[ -f "$f" ]] && log " $(basename "$f")"
|
|
done
|
|
else
|
|
info "Standard seed (with update.yaml, check_frequency=6h)"
|
|
|
|
"$SEED_TOOL" --format iso --app incus --output "$local_path" \
|
|
--force-install --force-reboot --hostname "$VM_NAME" \
|
|
--defaults --quiet &>/dev/null
|
|
|
|
log "Seed: standard incusos-seed output"
|
|
fi
|
|
|
|
success "Seed ISO: ${seed_name}"
|
|
|
|
# Upload to Proxmox
|
|
step "Uploading seed ISO"
|
|
local url="https://${PVE_HOST}:8006/api2/json/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/upload"
|
|
local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}"
|
|
|
|
curl -fsSk -H "$auth" \
|
|
-F "content=iso" \
|
|
-F "filename=@${local_path};filename=${seed_name}" \
|
|
"$url" &>/dev/null
|
|
|
|
success "Seed uploaded"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# VM lifecycle
|
|
# ---------------------------------------------------------------------------
|
|
|
|
create_vm() {
|
|
step "Creating VM (VMID ${VMID}, name ${VM_NAME})"
|
|
|
|
local json_payload
|
|
json_payload=$(python3 -c "
|
|
import json
|
|
payload = {
|
|
'vmid': ${VMID},
|
|
'name': '${VM_NAME}',
|
|
'description': '[incusos-lab:managed] observe-deploy',
|
|
'ostype': 'l26',
|
|
'bios': 'ovmf',
|
|
'machine': 'q35',
|
|
'efidisk0': '${PVE_STORAGE}:0,efitype=4m,pre-enrolled-keys=0',
|
|
'tpmstate0': '${PVE_STORAGE}:1,version=v2.0',
|
|
'cpu': 'host',
|
|
'cores': 4,
|
|
'memory': 4096,
|
|
'balloon': 0,
|
|
'scsihw': 'virtio-scsi-pci',
|
|
'scsi0': '${PVE_STORAGE}:50,iothread=1',
|
|
'ide2': '${PVE_ISO_STORAGE}:iso/${ISO_FILENAME},media=cdrom',
|
|
'ide3': '${PVE_ISO_STORAGE}:iso/${SEED_ISO},media=cdrom',
|
|
'net0': 'virtio,bridge=${PVE_BRIDGE}',
|
|
'boot': 'order=ide2;scsi0',
|
|
'agent': 1,
|
|
}
|
|
pool = '${PVE_POOL}'
|
|
if pool:
|
|
payload['pool'] = pool
|
|
print(json.dumps(payload))
|
|
")
|
|
|
|
local url="https://${PVE_HOST}:8006/api2/json/nodes/${PVE_NODE}/qemu"
|
|
local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}"
|
|
|
|
local create_response
|
|
create_response=$(curl -fsSk -H "$auth" -H "Content-Type: application/json" \
|
|
-X POST -d "$json_payload" "$url" 2>/dev/null) || {
|
|
error "Failed to create VM"
|
|
exit 1
|
|
}
|
|
|
|
# Wait for async task
|
|
local upid
|
|
upid=$(echo "$create_response" | python3 -c "import json,sys; print(json.load(sys.stdin).get('data',''))" 2>/dev/null) || upid=""
|
|
if [[ -n "$upid" ]]; then
|
|
local encoded_upid
|
|
encoded_upid=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${upid}', safe=''))" 2>/dev/null)
|
|
local wait_elapsed=0
|
|
while [[ $wait_elapsed -lt 60 ]]; do
|
|
sleep 2
|
|
wait_elapsed=$((wait_elapsed + 2))
|
|
local task_status
|
|
task_status=$(pve_api GET "/nodes/${PVE_NODE}/tasks/${encoded_upid}/status" 2>/dev/null | \
|
|
python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status',''))" 2>/dev/null) || task_status=""
|
|
if [[ "$task_status" == "stopped" ]]; then
|
|
break
|
|
fi
|
|
done
|
|
fi
|
|
|
|
success "VM created"
|
|
}
|
|
|
|
cleanup_vm() {
|
|
step "Cleaning up VM ${VMID}"
|
|
|
|
local status
|
|
status=$(pve_vm_status "$VMID" 2>/dev/null) || status="unknown"
|
|
if [[ "$status" == "running" ]]; then
|
|
pve_stop_vm "$VMID" || true
|
|
sleep 3
|
|
fi
|
|
|
|
pve_destroy_vm "$VMID" || true
|
|
|
|
# Remove seed ISO from Proxmox storage
|
|
pve_api DELETE "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content/${PVE_ISO_STORAGE}:iso/${SEED_ISO}" &>/dev/null || true
|
|
|
|
# Remove incus remote
|
|
if command -v incus &>/dev/null; then
|
|
local current_remote
|
|
current_remote=$(incus remote get-default 2>/dev/null) || true
|
|
if [[ "$current_remote" == "$VM_NAME" ]]; then
|
|
incus remote switch local 2>/dev/null || true
|
|
fi
|
|
incus remote remove "$VM_NAME" 2>/dev/null || true
|
|
fi
|
|
|
|
# Cleanup temp files
|
|
rm -rf "${SEED_TMPDIR:-}" 2>/dev/null || true
|
|
rm -f "/tmp/vm-${VMID}-screen.ppm" 2>/dev/null || true
|
|
|
|
success "Cleanup complete"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Install monitoring (inline blockstat check)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
check_install_progress() {
|
|
# Called periodically during the ISO boot phase to detect when
|
|
# installation completes. Updates PHASE when install is done.
|
|
# Returns immediately — does not block.
|
|
|
|
local wr_bytes
|
|
wr_bytes=$(pve_vm_disk_writes "$VMID" 2>/dev/null) || wr_bytes=0
|
|
|
|
if [[ $wr_bytes -gt $PREV_WR_BYTES ]]; then
|
|
if [[ "$WRITES_STARTED" != true ]]; then
|
|
WRITES_STARTED=true
|
|
PHASE="installing"
|
|
local wr_mb=$(( wr_bytes / 1048576 ))
|
|
log "Install started — disk writes detected (${wr_mb} MiB)"
|
|
fi
|
|
STABLE_COUNT=0
|
|
elif [[ "$WRITES_STARTED" == true ]]; then
|
|
STABLE_COUNT=$((STABLE_COUNT + 1))
|
|
if [[ $STABLE_COUNT -ge $STABLE_THRESHOLD ]]; then
|
|
local wr_mb=$(( wr_bytes / 1048576 ))
|
|
log "Install complete — disk activity stopped (${wr_mb} MiB total, ${STABLE_COUNT} stable polls)"
|
|
PHASE="install_done"
|
|
fi
|
|
fi
|
|
|
|
PREV_WR_BYTES=$wr_bytes
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main deploy loop
|
|
# ---------------------------------------------------------------------------
|
|
|
|
run_deploy() {
|
|
local deploy_start
|
|
deploy_start=$(date +%s)
|
|
|
|
# --- Phase 1: Boot from ISO ---
|
|
step "Starting VM (ISO boot + screenshot capture)"
|
|
pve_start_vm "$VMID"
|
|
PHASE="iso_boot"
|
|
log "VM started — phase: iso_boot"
|
|
|
|
local elapsed=0
|
|
local last_blockstat_check=0
|
|
|
|
# Screenshot + monitor loop during ISO boot and installation
|
|
while [[ "$PHASE" == "iso_boot" ]] || [[ "$PHASE" == "installing" ]]; do
|
|
take_frame || true
|
|
|
|
# Check blockstat every INSTALL_POLL seconds (not every screenshot)
|
|
local now
|
|
now=$(date +%s)
|
|
if [[ $((now - last_blockstat_check)) -ge $INSTALL_POLL ]]; then
|
|
check_install_progress
|
|
last_blockstat_check=$now
|
|
fi
|
|
|
|
sleep "$SCREENSHOT_INTERVAL"
|
|
elapsed=$((elapsed + SCREENSHOT_INTERVAL))
|
|
|
|
# Check if VM stopped on its own (force_reboot caused guest reboot,
|
|
# but QEMU uptime doesn't reset — VM stays "running". If it's
|
|
# actually stopped, something went wrong.)
|
|
local status
|
|
status=$(pve_vm_status "$VMID" 2>/dev/null) || status="unknown"
|
|
if [[ "$status" == "stopped" ]]; then
|
|
log "VM stopped unexpectedly during install"
|
|
PHASE="install_done"
|
|
fi
|
|
|
|
if [[ $elapsed -ge $INSTALL_TIMEOUT ]]; then
|
|
log "Install timeout after ${INSTALL_TIMEOUT}s"
|
|
warn "Installation did not complete within ${INSTALL_TIMEOUT}s"
|
|
|
|
# Take a final frame for diagnosis
|
|
take_frame || true
|
|
PHASE="failed"
|
|
break
|
|
fi
|
|
|
|
# Progress logging every 30s
|
|
if [[ $((elapsed % 30)) -lt $SCREENSHOT_INTERVAL ]]; then
|
|
local wr_mb=$(( PREV_WR_BYTES / 1048576 ))
|
|
log "Progress: ${elapsed}s elapsed, ${wr_mb} MiB written, phase=${PHASE}, frame=${FRAME_NUM}"
|
|
fi
|
|
done
|
|
|
|
if [[ "$PHASE" == "failed" ]]; then
|
|
return 1
|
|
fi
|
|
|
|
# --- Phase 2: Transition to disk boot ---
|
|
PHASE="transitioning"
|
|
log "Transitioning: stop VM, remove install media, start from disk"
|
|
|
|
# Take a few more frames during transition
|
|
take_frame || true
|
|
|
|
# Stop the VM
|
|
step "Stopping VM for media removal"
|
|
pve_stop_vm "$VMID" || true
|
|
|
|
# Wait for VM to actually stop
|
|
local stop_wait=0
|
|
while [[ $stop_wait -lt 30 ]]; do
|
|
local status
|
|
status=$(pve_vm_status "$VMID" 2>/dev/null) || status="unknown"
|
|
if [[ "$status" == "stopped" ]]; then
|
|
break
|
|
fi
|
|
sleep 2
|
|
stop_wait=$((stop_wait + 2))
|
|
done
|
|
log "VM stopped after ${stop_wait}s"
|
|
|
|
# Remove install media and set boot order
|
|
step "Removing install media (ide2, ide3) and setting boot to scsi0"
|
|
pve_api PUT "/nodes/${PVE_NODE}/qemu/${VMID}/config" \
|
|
'{"delete":"ide2,ide3","boot":"order=scsi0"}' json &>/dev/null || {
|
|
error "Failed to remove install media"
|
|
PHASE="failed"
|
|
return 1
|
|
}
|
|
log "Install media removed, boot order set to scsi0"
|
|
|
|
# --- Phase 3: Boot from disk ---
|
|
step "Starting VM from disk (first boot)"
|
|
pve_start_vm "$VMID"
|
|
PHASE="disk_boot"
|
|
log "VM started from disk — phase: disk_boot"
|
|
|
|
local boot_elapsed=0
|
|
local ip_detected=""
|
|
local ip_detect_started=false
|
|
|
|
# Screenshot loop during first boot
|
|
while [[ "$PHASE" == "disk_boot" ]]; do
|
|
take_frame || true
|
|
|
|
sleep "$SCREENSHOT_INTERVAL"
|
|
boot_elapsed=$((boot_elapsed + SCREENSHOT_INTERVAL))
|
|
|
|
# After 30s, start trying to detect IP
|
|
if [[ $boot_elapsed -ge 30 ]] && [[ "$ip_detect_started" != true ]]; then
|
|
ip_detect_started=true
|
|
log "Starting IP detection (${boot_elapsed}s into disk boot)"
|
|
fi
|
|
|
|
# Try IP detection every 10s after the initial 30s wait
|
|
if [[ "$ip_detect_started" == true ]] && [[ $((boot_elapsed % 10)) -lt $SCREENSHOT_INTERVAL ]]; then
|
|
local ip
|
|
ip=$(pve_vm_ip "$VMID" 2>/dev/null) || ip=""
|
|
if [[ -n "$ip" ]]; then
|
|
if [[ -z "$ip_detected" ]]; then
|
|
ip_detected="$ip"
|
|
log "IP detected: ${ip} (${boot_elapsed}s into disk boot)"
|
|
fi
|
|
|
|
# Check port 8443
|
|
if curl -sk --connect-timeout 3 "https://${ip}:8443" &>/dev/null; then
|
|
log "Port 8443 responding at ${ip} (${boot_elapsed}s into disk boot)"
|
|
PHASE="ready"
|
|
break
|
|
else
|
|
log_quiet "Port 8443 not yet responding at ${ip}"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
if [[ $boot_elapsed -ge $BOOT_TIMEOUT ]]; then
|
|
log "Boot timeout after ${BOOT_TIMEOUT}s"
|
|
warn "Port 8443 not reachable after ${BOOT_TIMEOUT}s"
|
|
|
|
# Take final diagnostic frames
|
|
take_frame || true
|
|
PHASE="failed"
|
|
break
|
|
fi
|
|
|
|
# Progress logging every 15s
|
|
if [[ $((boot_elapsed % 15)) -lt $SCREENSHOT_INTERVAL ]]; then
|
|
log "Disk boot: ${boot_elapsed}s elapsed, ip=${ip_detected:-none}, phase=${PHASE}, frame=${FRAME_NUM}"
|
|
fi
|
|
done
|
|
|
|
# Take a few final frames after reaching ready state
|
|
if [[ "$PHASE" == "ready" ]]; then
|
|
for i in 1 2 3; do
|
|
sleep "$SCREENSHOT_INTERVAL"
|
|
take_frame || true
|
|
done
|
|
fi
|
|
|
|
local deploy_end
|
|
deploy_end=$(date +%s)
|
|
local total_seconds=$((deploy_end - deploy_start))
|
|
|
|
log "Deploy finished: phase=${PHASE}, total_time=${total_seconds}s, frames=${FRAME_NUM}"
|
|
|
|
if [[ "$PHASE" == "ready" ]]; then
|
|
return 0
|
|
else
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Preflight checks
|
|
# ---------------------------------------------------------------------------
|
|
|
|
preflight() {
|
|
step "Preflight checks"
|
|
|
|
# Check required tools
|
|
local missing=()
|
|
command -v sshpass &>/dev/null || missing+=("sshpass")
|
|
command -v python3 &>/dev/null || missing+=("python3")
|
|
command -v curl &>/dev/null || missing+=("curl")
|
|
|
|
if [[ ${#missing[@]} -gt 0 ]]; then
|
|
error "Missing required tools: ${missing[*]}"
|
|
exit 1
|
|
fi
|
|
|
|
# Check PIL availability
|
|
if ! python3 -c "from PIL import Image" &>/dev/null; then
|
|
warn "python3 PIL/Pillow not available — screenshots will be saved as PPM (larger files)"
|
|
fi
|
|
|
|
# Check SSH connectivity
|
|
if ! sshpass -p "$PROXMOX_ROOT_PASSWORD" ssh "${SSH_OPTS[@]}" \
|
|
"root@${PVE_HOST}" "echo ok" &>/dev/null; then
|
|
error "Cannot SSH to root@${PVE_HOST}"
|
|
error "Check PROXMOX_ROOT_PASSWORD in env file and SSH access"
|
|
exit 1
|
|
fi
|
|
success "SSH to root@${PVE_HOST}: OK"
|
|
|
|
# Check VMID range (must be 850-869 per CLAUDE.md)
|
|
if [[ $VMID -lt 850 ]] || [[ $VMID -gt 869 ]]; then
|
|
error "VMID ${VMID} outside allowed test range (850-869)"
|
|
exit 1
|
|
fi
|
|
|
|
# Check VMID not in use
|
|
if pve_vm_exists "$VMID"; then
|
|
error "VMID ${VMID} already exists on Proxmox"
|
|
error "Clean it up first or use --vmid to pick a different one"
|
|
exit 1
|
|
fi
|
|
success "VMID ${VMID} available"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Setup output directory
|
|
# ---------------------------------------------------------------------------
|
|
|
|
setup_output() {
|
|
local timestamp
|
|
timestamp=$(date '+%Y-%m-%d_%H-%M-%S')
|
|
|
|
if [[ -z "$LABEL" ]]; then
|
|
if [[ "$NO_UPDATE_YAML" == true ]]; then
|
|
LABEL="no-update-yaml"
|
|
else
|
|
LABEL="standard"
|
|
fi
|
|
fi
|
|
|
|
RUN_DIR="${SCRIPT_DIR}/observe-runs/${timestamp}_${LABEL}"
|
|
mkdir -p "$RUN_DIR"
|
|
|
|
LOG_FILE="${RUN_DIR}/deploy.log"
|
|
touch "$LOG_FILE"
|
|
|
|
success "Output directory: ${RUN_DIR}"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main
|
|
# ---------------------------------------------------------------------------
|
|
|
|
echo -e "${BOLD}${SCRIPT_NAME}${RESET} v${VERSION}"
|
|
echo -e "Observational single-VM deploy with rapid console screenshots"
|
|
echo ""
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
echo -e "${YELLOW}[dry-run] Preview mode — no VMs will be created${RESET}"
|
|
echo ""
|
|
echo "Configuration:"
|
|
echo " VMID: ${VMID}"
|
|
echo " VM name: ${VM_NAME}"
|
|
echo " Update.yaml: $(if [[ "$NO_UPDATE_YAML" == true ]]; then echo "OMITTED"; else echo "included (6h)"; fi)"
|
|
echo " Screenshot interval: ${SCREENSHOT_INTERVAL}s"
|
|
echo ""
|
|
|
|
step "Would load proxmox.yaml"
|
|
load_proxmox_yaml
|
|
detail "Host: ${PVE_HOST}, Node: ${PVE_NODE}, Storage: ${PVE_STORAGE}"
|
|
detail "Pool: ${PVE_POOL}, Token: ${PVE_API_TOKEN_ID}"
|
|
|
|
echo ""
|
|
step "Would check:"
|
|
detail "- SSH to root@${PVE_HOST} for screenshots"
|
|
detail "- VMID ${VMID} availability"
|
|
detail "- IncusOS ISO on Proxmox"
|
|
detail "- PIL/Pillow for PPM→PNG conversion"
|
|
|
|
echo ""
|
|
step "Would execute:"
|
|
detail "1. Generate seed ISO ($(if [[ "$NO_UPDATE_YAML" == true ]]; then echo "no update.yaml"; else echo "standard"; fi))"
|
|
detail "2. Upload seed to Proxmox"
|
|
detail "3. Create VM ${VMID} (${VM_NAME})"
|
|
detail "4. Start VM, begin screenshot loop (every ${SCREENSHOT_INTERVAL}s)"
|
|
detail "5. Monitor blockstat for install completion"
|
|
detail "6. Stop VM, remove media, start from disk"
|
|
detail "7. Continue screenshots until port 8443 responds (or ${BOOT_TIMEOUT}s timeout)"
|
|
detail "8. Save frames + deploy.log"
|
|
detail "9. Cleanup VM"
|
|
|
|
echo ""
|
|
info "Dry run complete"
|
|
exit 0
|
|
fi
|
|
|
|
# Load configuration
|
|
step "Loading configuration"
|
|
load_proxmox_yaml
|
|
detail "Host: ${PVE_HOST}, Node: ${PVE_NODE}, Storage: ${PVE_STORAGE}"
|
|
|
|
# Run preflight checks
|
|
preflight
|
|
|
|
# Find IncusOS ISO
|
|
find_incusos_iso
|
|
|
|
# Setup output directory
|
|
setup_output
|
|
|
|
# Log configuration
|
|
log "=== observe-deploy v${VERSION} ==="
|
|
log "VMID: ${VMID}"
|
|
log "VM name: ${VM_NAME}"
|
|
log "ISO: ${ISO_FILENAME}"
|
|
log "Update.yaml: $(if [[ "$NO_UPDATE_YAML" == true ]]; then echo "OMITTED"; else echo "included (6h)"; fi)"
|
|
log "Screenshot interval: ${SCREENSHOT_INTERVAL}s"
|
|
log "Install timeout: ${INSTALL_TIMEOUT}s"
|
|
log "Boot timeout: ${BOOT_TIMEOUT}s"
|
|
|
|
# Generate and upload seed
|
|
generate_seed
|
|
|
|
# Create VM
|
|
create_vm
|
|
|
|
echo ""
|
|
echo -e "${BOLD}Starting observational deploy${RESET}"
|
|
echo " Frames saved to: ${RUN_DIR}/"
|
|
echo " Log: ${RUN_DIR}/deploy.log"
|
|
echo " Press Ctrl+C to abort (VM will be cleaned up)"
|
|
echo ""
|
|
|
|
# Trap for clean exit
|
|
trap_cleanup() {
|
|
echo ""
|
|
warn "Interrupted — cleaning up"
|
|
cleanup_vm
|
|
exit 130
|
|
}
|
|
trap trap_cleanup INT TERM
|
|
|
|
# Run the deploy with screenshots
|
|
deploy_result=0
|
|
run_deploy || deploy_result=$?
|
|
|
|
echo ""
|
|
echo -e "${BOLD}========================================${RESET}"
|
|
echo -e "${BOLD} Deploy Results${RESET}"
|
|
echo -e "${BOLD}========================================${RESET}"
|
|
echo ""
|
|
echo " VMID: ${VMID}"
|
|
echo " Seed: $(if [[ "$NO_UPDATE_YAML" == true ]]; then echo "no update.yaml"; else echo "standard (check_frequency=6h)"; fi)"
|
|
echo " Outcome: $(if [[ $deploy_result -eq 0 ]]; then echo -e "${GREEN}PASS${RESET} — port 8443 reachable"; else echo -e "${RED}FAIL${RESET} — phase: ${PHASE}"; fi)"
|
|
echo " Frames: ${FRAME_NUM}"
|
|
echo " Output: ${RUN_DIR}/"
|
|
echo ""
|
|
|
|
# Cleanup
|
|
step "Post-deploy cleanup"
|
|
cleanup_vm
|
|
|
|
if [[ $deploy_result -eq 0 ]]; then
|
|
success "Observational deploy complete — ${FRAME_NUM} frames captured"
|
|
else
|
|
warn "Deploy failed at phase: ${PHASE}"
|
|
warn "Check screenshots and log for diagnosis:"
|
|
warn " ${RUN_DIR}/deploy.log"
|
|
warn " ${RUN_DIR}/frame-*.png"
|
|
fi
|
|
|
|
echo ""
|
|
echo "To view key frames:"
|
|
echo " ls ${RUN_DIR}/frame-*.png"
|
|
echo " # View in image viewer or analyze with Claude Code"
|
|
echo ""
|
|
|
|
exit $deploy_result
|