maarten
/
incus-contrib
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
incus-contrib/bin/diagnostic/observe-deploy

1326 lines
44 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
# observe-deploy - Single-VM IncusOS deploy with rapid console screenshots
#
# Captures the complete IncusOS lifecycle on screen — from ISO boot through
# installation, reboot, first-boot sequence, and port 8443 readiness. Takes
# screenshots every 2 seconds via Proxmox SSH root access (screendump).
#
# Purpose: diagnose boot failures by building a frame-by-frame timeline of
# exactly what IncusOS shows at every stage. NOT a batch test — one VM at
# a time, maximum visibility.
#
# Usage:
# ./observe-deploy # Standard deploy with screenshots
# ./observe-deploy --no-update-yaml # Omit update.yaml from seed
# ./observe-deploy --label run-01 # Custom run label
# ./observe-deploy --runs 10 --label baseline # 10 sequential cycles with stats
# ./observe-deploy --vmid 855 # Use a specific VMID
# ./observe-deploy --interval 1 # 1-second screenshots
# ./observe-deploy --dry-run # Preview without deploying
#
# Output: timestamped directory with PNG frames + deploy.log
# observe-runs/YYYY-MM-DD_HH-MM-SS_<label>/
# frame-001.png
# frame-002.png
# ...
# deploy.log
set -euo pipefail
readonly VERSION="0.1.0"
readonly SCRIPT_NAME="observe-deploy"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SEED_TOOL="${SCRIPT_DIR}/incusos-seed"
# ---------------------------------------------------------------------------
# Defaults
# ---------------------------------------------------------------------------
VMID=850
VM_NAME="observe-deploy"
LABEL=""
RUNS=1 # number of sequential deploy+destroy cycles
NO_UPDATE_YAML=false
NO_VLAN=false
STATIC_IP="" # Static IP in CIDR notation (e.g. 192.168.102.100/22)
VM_EXTRA="" # Extra VM creation overrides (key=value,key=value)
NO_FORCE_REBOOT=false # Skip force_reboot in seed (no SysRq-B reboot after install)
DRY_RUN=false
SCREENSHOT_INTERVAL=2 # seconds between screenshots
INSTALL_TIMEOUT=600 # seconds to wait for install completion
BOOT_TIMEOUT=180 # seconds to wait for port 8443 after disk boot
INSTALL_POLL=5 # blockstat poll interval during install phase
STABLE_THRESHOLD=3 # consecutive stable polls = install done
# Proxmox connection (from proxmox.yaml)
PVE_HOST=""
PVE_NODE=""
PVE_STORAGE=""
PVE_ISO_STORAGE=""
PVE_BRIDGE=""
PVE_POOL=""
PVE_VLAN=""
PVE_GATEWAY=""
PVE_DNS=""
PVE_API_TOKEN_ID=""
# IncusOS ISO
ISO_FILENAME=""
# Run output
RUN_DIR=""
LOG_FILE=""
FRAME_NUM=0
# State tracking
PHASE="init" # init → iso_boot → installing → install_done → transitioning → disk_boot → ready/failed
PREV_WR_BYTES=0
WRITES_STARTED=false
STABLE_COUNT=0
SCRUB_SCHEDULE="" # Filled after port 8443 check: "0 4 * * 0" = healthy, "" = crontab bug
DEPLOY_IP="" # IP of the deployed VM (set during disk_boot phase)
# ---------------------------------------------------------------------------
# Colors
# ---------------------------------------------------------------------------
setup_colors() {
if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[0;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
BOLD='\033[1m'
DIM='\033[2m'
RESET='\033[0m'
else
RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' RESET=''
fi
}
setup_colors
step() { echo -e "${CYAN}[step]${RESET} ${BOLD}$*${RESET}"; }
success() { echo -e "${GREEN}[ok]${RESET} $*"; }
warn() { echo -e "${YELLOW}[warn]${RESET} $*" >&2; }
error() { echo -e "${RED}[error]${RESET} $*" >&2; }
info() { echo -e "${BLUE}[info]${RESET} $*"; }
detail() { echo -e "${DIM} $*${RESET}"; }
# Log to both console and file
log() {
local ts
ts=$(date '+%H:%M:%S.%3N')
local msg="[${ts}] $*"
echo -e "${DIM}${msg}${RESET}"
[[ -n "$LOG_FILE" ]] && echo "$msg" >> "$LOG_FILE"
}
# Log to file only (for high-frequency events)
log_quiet() {
local ts
ts=$(date '+%H:%M:%S.%3N')
[[ -n "$LOG_FILE" ]] && echo "[${ts}] $*" >> "$LOG_FILE"
}
# ---------------------------------------------------------------------------
# Argument parsing
# ---------------------------------------------------------------------------
while [[ $# -gt 0 ]]; do
case "$1" in
--vmid) VMID="${2:?'--vmid requires a number'}"; shift 2 ;;
--label) LABEL="${2:?'--label requires a string'}"; shift 2 ;;
--runs) RUNS="${2:?'--runs requires a number'}"; shift 2 ;;
--no-update-yaml) NO_UPDATE_YAML=true; shift ;;
--no-force-reboot) NO_FORCE_REBOOT=true; shift ;;
--no-vlan) NO_VLAN=true; shift ;;
--ip) STATIC_IP="${2:?'--ip requires ADDR/PREFIX'}"; shift 2 ;;
--vm-extra) VM_EXTRA="${2:?'--vm-extra requires key=value pairs'}"; shift 2 ;;
--interval) SCREENSHOT_INTERVAL="${2:?'--interval requires seconds'}"; shift 2 ;;
--dry-run) DRY_RUN=true; shift ;;
-h|--help)
cat <<EOF
${BOLD}${SCRIPT_NAME}${RESET} v${VERSION} - Observational IncusOS deploy with screenshots
${BOLD}USAGE${RESET}
${SCRIPT_NAME} [OPTIONS]
${BOLD}OPTIONS${RESET}
--vmid N VMID to use (default: 850, must be 850-869)
--label NAME Run label for output directory (default: auto)
--runs N Run N sequential deploy+destroy cycles with stats (default: 1)
--no-update-yaml Omit update.yaml from seed (Group A equivalent)
--no-force-reboot Omit force_reboot from seed (no SysRq-B after install)
--no-vlan Skip VLAN tag (use native LAN for IP detection)
--ip ADDR/PREFIX Static IP for the VM (e.g. 192.168.102.100/22)
Skips ARP detection; uses known IP for health checks
--interval N Screenshot interval in seconds (default: 2)
--dry-run Preview without deploying
-h, --help Show this help
${BOLD}OUTPUT${RESET}
Screenshots and logs are saved to:
observe-runs/<timestamp>_<label>/
frame-001.png ... frame-NNN.png
deploy.log
${BOLD}REQUIREMENTS${RESET}
- proxmox.yaml with Proxmox connection settings
- env file with PROXMOX_TOKEN_SECRET and PROXMOX_ROOT_PASSWORD
- IncusOS ISO already uploaded to Proxmox (use incusos-proxmox --phase upload)
- python3 with PIL/Pillow for PPM→PNG conversion
- sshpass for root SSH to Proxmox host
EOF
exit 0
;;
*) error "Unknown option: $1"; exit 1 ;;
esac
done
# ---------------------------------------------------------------------------
# Load environment
# ---------------------------------------------------------------------------
load_env() {
local dir="$SCRIPT_DIR"
while [[ "$dir" != "/" ]]; do
if [[ -f "${dir}/env" ]]; then
# shellcheck disable=SC1091
source "${dir}/env"
return
fi
dir="$(dirname "$dir")"
done
}
load_env
if [[ -z "${PROXMOX_TOKEN_SECRET:-}" ]]; then
error "PROXMOX_TOKEN_SECRET not set. Create an env file or export it."
exit 1
fi
if [[ -z "${PROXMOX_ROOT_PASSWORD:-}" ]]; then
error "PROXMOX_ROOT_PASSWORD not set. Add it to the env file."
error "Required for SSH root screenshots (screendump)."
exit 1
fi
# ---------------------------------------------------------------------------
# Load proxmox.yaml
# ---------------------------------------------------------------------------
yaml_get() {
local file="$1" key="$2" default="${3:-}"
local val
val=$(awk -F': ' -v k="$key" '$1 == k {print $2; exit}' "$file" 2>/dev/null) || val=""
val="${val#\"}" ; val="${val%\"}"
val="${val#\'}" ; val="${val%\'}"
echo "${val:-$default}"
}
load_proxmox_yaml() {
local yaml_file=""
if [[ -f "${SCRIPT_DIR}/proxmox.yaml" ]]; then
yaml_file="${SCRIPT_DIR}/proxmox.yaml"
elif [[ -f "proxmox.yaml" ]]; then
yaml_file="proxmox.yaml"
else
error "proxmox.yaml not found"
exit 1
fi
PVE_HOST=$(yaml_get "$yaml_file" "host")
PVE_NODE=$(yaml_get "$yaml_file" "node" "pve")
PVE_STORAGE=$(yaml_get "$yaml_file" "storage" "local-lvm")
PVE_ISO_STORAGE=$(yaml_get "$yaml_file" "iso_storage" "local")
PVE_BRIDGE=$(yaml_get "$yaml_file" "bridge" "vmbr0")
PVE_POOL=$(yaml_get "$yaml_file" "pool")
PVE_VLAN=$(yaml_get "$yaml_file" "vlan")
PVE_GATEWAY=$(yaml_get "$yaml_file" "gateway")
PVE_DNS=$(yaml_get "$yaml_file" "dns")
if [[ "$NO_VLAN" == true ]]; then
PVE_VLAN=""
fi
PVE_API_TOKEN_ID=$(yaml_get "$yaml_file" "api_token_id")
if [[ -z "$PVE_HOST" ]]; then
error "proxmox.yaml: 'host' is required"
exit 1
fi
}
# ---------------------------------------------------------------------------
# Proxmox API helpers
# ---------------------------------------------------------------------------
pve_api() {
local method="$1"
local endpoint="$2"
local data="${3:-}"
local content_type="${4:-form}"
local url="https://${PVE_HOST}:8006/api2/json${endpoint}"
local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}"
local curl_args=(-fsSk -H "$auth")
if [[ "$content_type" == "json" ]]; then
curl_args+=(-H "Content-Type: application/json")
fi
case "$method" in
GET) curl "${curl_args[@]}" "$url" ;;
POST) curl "${curl_args[@]}" -X POST ${data:+-d "$data"} "$url" ;;
PUT) curl "${curl_args[@]}" -X PUT ${data:+-d "$data"} "$url" ;;
DELETE) curl "${curl_args[@]}" -X DELETE "$url" ;;
esac
}
pve_vm_exists() {
local vmid="$1"
pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" &>/dev/null
}
pve_vm_status() {
local vmid="$1"
pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" 2>/dev/null | \
python3 -c "import json,sys; print(json.load(sys.stdin)['data']['status'])" 2>/dev/null
}
pve_start_vm() {
local vmid="$1"
pve_api POST "/nodes/${PVE_NODE}/qemu/${vmid}/status/start" &>/dev/null
}
pve_stop_vm() {
local vmid="$1"
pve_api POST "/nodes/${PVE_NODE}/qemu/${vmid}/status/stop" &>/dev/null
}
pve_destroy_vm() {
local vmid="$1"
local destroy_response
destroy_response=$(pve_api DELETE "/nodes/${PVE_NODE}/qemu/${vmid}?purge=1&destroy-unreferenced-disks=1" 2>/dev/null) || return 1
# Wait for async destroy task to complete (avoids VMID reuse issues)
local upid
upid=$(echo "$destroy_response" | python3 -c "import json,sys; print(json.load(sys.stdin).get('data',''))" 2>/dev/null) || upid=""
if [[ -n "$upid" ]]; then
local encoded_upid
encoded_upid=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${upid}', safe=''))" 2>/dev/null)
local wait_elapsed=0
while [[ $wait_elapsed -lt 30 ]]; do
sleep 2
wait_elapsed=$((wait_elapsed + 2))
local task_status
task_status=$(pve_api GET "/nodes/${PVE_NODE}/tasks/${encoded_upid}/status" 2>/dev/null | \
python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status',''))" 2>/dev/null) || task_status=""
if [[ "$task_status" == "stopped" ]]; then
break
fi
done
fi
}
pve_vm_disk_writes() {
local vmid="$1"
pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" 2>/dev/null | \
python3 -c "
import json,sys
d = json.load(sys.stdin).get('data',{})
bs = d.get('blockstat',{}).get('scsi0',{})
print(bs.get('wr_bytes',0))
" 2>/dev/null || echo "0"
}
pve_vm_mac() {
local vmid="$1"
pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null | \
python3 -c "
import json, sys, re
data = json.load(sys.stdin).get('data', {})
net0 = data.get('net0', '')
m = re.search(r'([0-9A-Fa-f:]{17})', net0)
print(m.group(1).lower() if m else '')
" 2>/dev/null
}
pve_vm_ip() {
local vmid="$1"
# If static IP is configured, return the bare IP (strip CIDR prefix)
if [[ -n "$STATIC_IP" ]]; then
echo "${STATIC_IP%%/*}"
return 0
fi
local mac
mac=$(pve_vm_mac "$vmid") || return 1
[[ -z "$mac" ]] && return 1
ip neigh flush nud stale 2>/dev/null || true
local bridge_subnet
bridge_subnet=$(ip -4 addr show | awk '/inet / && !/127\.0\.0/ {print $2; exit}')
if [[ -n "$bridge_subnet" ]]; then
local broadcast
broadcast=$(python3 -c "import ipaddress; print(ipaddress.ip_interface('${bridge_subnet}').network.broadcast_address)" 2>/dev/null) || true
if [[ -n "$broadcast" ]]; then
ping -c 2 -W 1 -b "$broadcast" &>/dev/null || true
sleep 1
fi
fi
local found_ip
found_ip=$(ip neigh show | awk -v mac="$mac" 'tolower($0) ~ tolower(mac) && (/REACHABLE/ || /DELAY/ || /STALE/) {print $1; exit}') || true
if [[ -n "$found_ip" ]] && ping -c 1 -W 1 "$found_ip" &>/dev/null; then
echo "$found_ip"
return 0
fi
# Parallel ping sweep fallback
if [[ -n "$bridge_subnet" ]]; then
local hosts
hosts=$(python3 -c "
import ipaddress
net = ipaddress.ip_interface('${bridge_subnet}').network
for h in net.hosts():
print(h)
" 2>/dev/null)
if [[ -n "$hosts" ]]; then
while IFS= read -r host_ip; do
ping -c 1 -W 1 "$host_ip" &>/dev/null &
done <<< "$hosts"
wait
sleep 1
found_ip=$(ip neigh show | awk -v mac="$mac" 'tolower($0) ~ tolower(mac) && (/REACHABLE/ || /DELAY/) {print $1; exit}') || true
if [[ -n "$found_ip" ]] && ping -c 1 -W 1 "$found_ip" &>/dev/null; then
echo "$found_ip"
return 0
fi
fi
fi
return 1
}
# ---------------------------------------------------------------------------
# scrub_schedule health check (definitive crontab bug indicator)
# ---------------------------------------------------------------------------
check_scrub_schedule() {
# Query the IncusOS REST API for the scrub_schedule field.
# This is the definitive health indicator:
# "0 4 * * 0" → genuine success
# "" → crontab bug hit (even if port 8443 responds)
# unreachable → daemon crashed before API started
#
# Sets global SCRUB_SCHEDULE and returns 0 on success, 1 on bug/failure.
local ip="$1"
local cert="${HOME}/.config/incus/client.crt"
local key="${HOME}/.config/incus/client.key"
if [[ ! -f "$cert" ]] || [[ ! -f "$key" ]]; then
log "No client certificate — skipping scrub_schedule check"
SCRUB_SCHEDULE="unchecked"
return 0
fi
local response
response=$(curl -sk --connect-timeout 5 --max-time 10 \
--cert "$cert" --key "$key" \
"https://${ip}:8443/os/1.0/system/storage" 2>/dev/null) || {
log "Could not query IncusOS storage API at ${ip}"
SCRUB_SCHEDULE="unreachable"
return 1
}
SCRUB_SCHEDULE=$(printf '%s' "$response" | python3 -c "
import sys, json
try:
d = json.load(sys.stdin)
print(d.get('metadata', {}).get('config', {}).get('scrub_schedule', ''))
except:
print('')
" 2>/dev/null) || SCRUB_SCHEDULE=""
if [[ -n "$SCRUB_SCHEDULE" ]]; then
log "scrub_schedule = '${SCRUB_SCHEDULE}' → HEALTHY"
return 0
else
log "scrub_schedule is EMPTY → crontab bug detected"
return 1
fi
}
# ---------------------------------------------------------------------------
# Screenshot capture
# ---------------------------------------------------------------------------
# SSH options for root access to Proxmox (screenshots only)
SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -o LogLevel=ERROR)
capture_screenshot() {
# Captures a single frame from the VM console.
# Returns 0 on success, 1 on failure (VM not running, SSH error, etc.)
local vmid="$1"
local output_png="$2"
local remote_ppm="/tmp/vm-${vmid}-screen.ppm"
local local_ppm="/tmp/vm-${vmid}-screen.ppm"
# Take screenshot via QEMU monitor (screendump)
if ! sshpass -p "$PROXMOX_ROOT_PASSWORD" ssh "${SSH_OPTS[@]}" \
"root@${PVE_HOST}" \
"echo 'screendump ${remote_ppm}' | qm monitor ${vmid}" &>/dev/null; then
return 1
fi
# Retrieve PPM file
if ! sshpass -p "$PROXMOX_ROOT_PASSWORD" scp "${SSH_OPTS[@]}" \
"root@${PVE_HOST}:${remote_ppm}" "$local_ppm" &>/dev/null; then
return 1
fi
# Cleanup remote file
sshpass -p "$PROXMOX_ROOT_PASSWORD" ssh "${SSH_OPTS[@]}" \
"root@${PVE_HOST}" "rm -f ${remote_ppm}" &>/dev/null || true
# Convert PPM → PNG
if ! python3 -c "from PIL import Image; Image.open('${local_ppm}').save('${output_png}')" 2>/dev/null; then
# If PIL not available, keep PPM
mv "$local_ppm" "${output_png%.png}.ppm" 2>/dev/null || true
return 0
fi
rm -f "$local_ppm"
return 0
}
take_frame() {
# Capture a numbered frame and log it
FRAME_NUM=$((FRAME_NUM + 1))
local frame_name
frame_name=$(printf "frame-%03d.png" "$FRAME_NUM")
local frame_path="${RUN_DIR}/${frame_name}"
if capture_screenshot "$VMID" "$frame_path"; then
log_quiet "Frame ${FRAME_NUM}: ${frame_name} (phase=${PHASE})"
return 0
else
log_quiet "Frame ${FRAME_NUM}: FAILED (phase=${PHASE})"
return 1
fi
}
# ---------------------------------------------------------------------------
# Find IncusOS ISO on Proxmox
# ---------------------------------------------------------------------------
find_incusos_iso() {
step "Finding IncusOS ISO on Proxmox"
local iso_list
iso_list=$(pve_api GET "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content?content=iso" 2>/dev/null) || iso_list=""
ISO_FILENAME=$(echo "$iso_list" | python3 -c "
import json, sys
data = json.load(sys.stdin).get('data', [])
isos = [d['volid'].split('/')[-1] for d in data if 'IncusOS_' in d.get('volid', '')]
if isos:
isos.sort()
print(isos[-1])
" 2>/dev/null) || ISO_FILENAME=""
if [[ -z "$ISO_FILENAME" ]]; then
error "No IncusOS ISO found on Proxmox storage '${PVE_ISO_STORAGE}'"
error "Deploy a lab first to upload the ISO, or use incusos-proxmox --phase upload"
exit 1
fi
success "Using ISO: ${ISO_FILENAME}"
}
# ---------------------------------------------------------------------------
# Seed generation
# ---------------------------------------------------------------------------
SEED_ISO=""
SEED_TMPDIR=""
generate_seed() {
step "Generating seed ISO"
SEED_TMPDIR=$(mktemp -d)
local seed_name="seed-${VM_NAME}.iso"
SEED_ISO="$seed_name"
local local_path="${SEED_TMPDIR}/${seed_name}"
if [[ "$NO_UPDATE_YAML" == true ]]; then
info "Omitting update.yaml from seed (Group A equivalent)"
# Generate full seed as tar, extract, remove update.yaml, create ISO
local seed_extra=()
if [[ -n "$STATIC_IP" ]]; then
seed_extra+=(--ip "$STATIC_IP" --gateway "$PVE_GATEWAY")
[[ -n "$PVE_DNS" ]] && seed_extra+=(--dns "$PVE_DNS")
fi
local reboot_flag=()
[[ "$NO_FORCE_REBOOT" != true ]] && reboot_flag=(--force-reboot)
"$SEED_TOOL" --format tar --app incus --output "${SEED_TMPDIR}/seed.tar" \
--force-install "${reboot_flag[@]+"${reboot_flag[@]}"}" --hostname "$VM_NAME" \
--defaults "${seed_extra[@]+"${seed_extra[@]}"}" --quiet &>/dev/null
local seeddir="${SEED_TMPDIR}/seedfiles"
mkdir -p "$seeddir"
tar xf "${SEED_TMPDIR}/seed.tar" -C "$seeddir"
rm -f "${seeddir}/update.yaml"
# Create ISO from modified seed directory
if command -v genisoimage &>/dev/null; then
genisoimage -V SEED_DATA -J -r -o "$local_path" "$seeddir" 2>/dev/null
elif command -v mkisofs &>/dev/null; then
mkisofs -V SEED_DATA -J -r -o "$local_path" "$seeddir" 2>/dev/null
else
error "genisoimage or mkisofs required for ISO creation"
exit 1
fi
# Log seed contents
log "Seed contents (no update.yaml):"
for f in "$seeddir"/*.yaml; do
[[ -f "$f" ]] && log " $(basename "$f")"
done
else
info "Standard seed (with update.yaml, check_frequency=6h)"
local seed_extra=()
if [[ -n "$STATIC_IP" ]]; then
seed_extra+=(--ip "$STATIC_IP" --gateway "$PVE_GATEWAY")
[[ -n "$PVE_DNS" ]] && seed_extra+=(--dns "$PVE_DNS")
fi
local reboot_flag=()
[[ "$NO_FORCE_REBOOT" != true ]] && reboot_flag=(--force-reboot)
"$SEED_TOOL" --format iso --app incus --output "$local_path" \
--force-install "${reboot_flag[@]+"${reboot_flag[@]}"}" --hostname "$VM_NAME" \
--defaults "${seed_extra[@]+"${seed_extra[@]}"}" --quiet &>/dev/null
log "Seed: standard incusos-seed output (force_reboot=$([[ "$NO_FORCE_REBOOT" == true ]] && echo false || echo true))"
fi
success "Seed ISO: ${seed_name}"
# Upload to Proxmox
step "Uploading seed ISO"
local url="https://${PVE_HOST}:8006/api2/json/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/upload"
local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}"
curl -fsSk -H "$auth" \
-F "content=iso" \
-F "filename=@${local_path};filename=${seed_name}" \
"$url" &>/dev/null
success "Seed uploaded"
}
# ---------------------------------------------------------------------------
# VM lifecycle
# ---------------------------------------------------------------------------
create_vm() {
step "Creating VM (VMID ${VMID}, name ${VM_NAME})"
local json_payload
json_payload=$(python3 -c "
import json
payload = {
'vmid': ${VMID},
'name': '${VM_NAME}',
'description': '[incusos-lab:managed] observe-deploy',
'ostype': 'l26',
'bios': 'ovmf',
'machine': 'q35',
'efidisk0': '${PVE_STORAGE}:0,efitype=4m,pre-enrolled-keys=0',
'tpmstate0': '${PVE_STORAGE}:1,version=v2.0',
'cpu': 'host',
'cores': 4,
'memory': 4096,
'balloon': 0,
'scsihw': 'virtio-scsi-pci',
'scsi0': '${PVE_STORAGE}:50,iothread=1',
'ide2': '${PVE_ISO_STORAGE}:iso/${ISO_FILENAME},media=cdrom',
'ide3': '${PVE_ISO_STORAGE}:iso/${SEED_ISO},media=cdrom',
'net0': 'virtio,bridge=${PVE_BRIDGE}' + (',tag=${PVE_VLAN}' if '${PVE_VLAN}' else ''),
'boot': 'order=ide2;scsi0',
'agent': 1,
}
pool = '${PVE_POOL}'
if pool:
payload['pool'] = pool
# Apply --vm-extra overrides (e.g. 'scsi0=local-zfs:50,cache=writethrough,iothread=1')
vm_extra = '${VM_EXTRA}'
if vm_extra:
for pair in vm_extra.split(',,,'): # triple-comma separator for multi-value pairs
if '=' in pair:
k, v = pair.split('=', 1)
# Convert numeric strings to ints for JSON
try:
v = int(v)
except (ValueError, TypeError):
pass
payload[k.strip()] = v
print(json.dumps(payload))
")
local url="https://${PVE_HOST}:8006/api2/json/nodes/${PVE_NODE}/qemu"
local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}"
local create_response
create_response=$(curl -fsSk -H "$auth" -H "Content-Type: application/json" \
-X POST -d "$json_payload" "$url" 2>/dev/null) || {
error "Failed to create VM"
exit 1
}
# Wait for async task
local upid
upid=$(echo "$create_response" | python3 -c "import json,sys; print(json.load(sys.stdin).get('data',''))" 2>/dev/null) || upid=""
if [[ -n "$upid" ]]; then
local encoded_upid
encoded_upid=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${upid}', safe=''))" 2>/dev/null)
local wait_elapsed=0
while [[ $wait_elapsed -lt 60 ]]; do
sleep 2
wait_elapsed=$((wait_elapsed + 2))
local task_status
task_status=$(pve_api GET "/nodes/${PVE_NODE}/tasks/${encoded_upid}/status" 2>/dev/null | \
python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status',''))" 2>/dev/null) || task_status=""
if [[ "$task_status" == "stopped" ]]; then
break
fi
done
fi
success "VM created"
}
cleanup_vm() {
step "Cleaning up VM ${VMID}"
local status
status=$(pve_vm_status "$VMID" 2>/dev/null) || status="unknown"
if [[ "$status" == "running" ]]; then
pve_stop_vm "$VMID" || true
sleep 3
fi
pve_destroy_vm "$VMID" || true
# Remove seed ISO from Proxmox storage
pve_api DELETE "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content/${PVE_ISO_STORAGE}:iso/${SEED_ISO}" &>/dev/null || true
# Remove incus remote
if command -v incus &>/dev/null; then
local current_remote
current_remote=$(incus remote get-default 2>/dev/null) || true
if [[ "$current_remote" == "$VM_NAME" ]]; then
incus remote switch local 2>/dev/null || true
fi
incus remote remove "$VM_NAME" 2>/dev/null || true
fi
# Cleanup temp files
rm -rf "${SEED_TMPDIR:-}" 2>/dev/null || true
rm -f "/tmp/vm-${VMID}-screen.ppm" 2>/dev/null || true
success "Cleanup complete"
}
# ---------------------------------------------------------------------------
# Install monitoring (inline blockstat check)
# ---------------------------------------------------------------------------
check_install_progress() {
# Called periodically during the ISO boot phase to detect when
# installation completes. Updates PHASE when install is done.
# Returns immediately — does not block.
local wr_bytes
wr_bytes=$(pve_vm_disk_writes "$VMID" 2>/dev/null) || wr_bytes=0
if [[ $wr_bytes -gt $PREV_WR_BYTES ]]; then
if [[ "$WRITES_STARTED" != true ]]; then
WRITES_STARTED=true
PHASE="installing"
local wr_mb=$(( wr_bytes / 1048576 ))
log "Install started — disk writes detected (${wr_mb} MiB)"
fi
STABLE_COUNT=0
elif [[ "$WRITES_STARTED" == true ]]; then
STABLE_COUNT=$((STABLE_COUNT + 1))
if [[ $STABLE_COUNT -ge $STABLE_THRESHOLD ]]; then
local wr_mb=$(( wr_bytes / 1048576 ))
log "Install complete — disk activity stopped (${wr_mb} MiB total, ${STABLE_COUNT} stable polls)"
PHASE="install_done"
fi
fi
PREV_WR_BYTES=$wr_bytes
}
# ---------------------------------------------------------------------------
# Main deploy loop
# ---------------------------------------------------------------------------
run_deploy() {
local deploy_start
deploy_start=$(date +%s)
# --- Phase 1: Boot from ISO ---
step "Starting VM (ISO boot + screenshot capture)"
pve_start_vm "$VMID"
PHASE="iso_boot"
log "VM started — phase: iso_boot"
local elapsed=0
local last_blockstat_check=0
# Screenshot + monitor loop during ISO boot and installation
while [[ "$PHASE" == "iso_boot" ]] || [[ "$PHASE" == "installing" ]]; do
take_frame || true
# Check blockstat every INSTALL_POLL seconds (not every screenshot)
local now
now=$(date +%s)
if [[ $((now - last_blockstat_check)) -ge $INSTALL_POLL ]]; then
check_install_progress
last_blockstat_check=$now
fi
sleep "$SCREENSHOT_INTERVAL"
elapsed=$((elapsed + SCREENSHOT_INTERVAL))
# Check if VM stopped on its own (force_reboot caused guest reboot,
# but QEMU uptime doesn't reset — VM stays "running". If it's
# actually stopped, something went wrong.)
local status
status=$(pve_vm_status "$VMID" 2>/dev/null) || status="unknown"
if [[ "$status" == "stopped" ]]; then
log "VM stopped unexpectedly during install"
PHASE="install_done"
fi
if [[ $elapsed -ge $INSTALL_TIMEOUT ]]; then
log "Install timeout after ${INSTALL_TIMEOUT}s"
warn "Installation did not complete within ${INSTALL_TIMEOUT}s"
# Take a final frame for diagnosis
take_frame || true
PHASE="failed"
break
fi
# Progress logging every 30s
if [[ $((elapsed % 30)) -lt $SCREENSHOT_INTERVAL ]]; then
local wr_mb=$(( PREV_WR_BYTES / 1048576 ))
log "Progress: ${elapsed}s elapsed, ${wr_mb} MiB written, phase=${PHASE}, frame=${FRAME_NUM}"
fi
done
if [[ "$PHASE" == "failed" ]]; then
return 1
fi
# --- Phase 2: Transition to disk boot ---
PHASE="transitioning"
log "Transitioning: stop VM, remove install media, start from disk"
# Take a few more frames during transition
take_frame || true
# Stop the VM
step "Stopping VM for media removal"
pve_stop_vm "$VMID" || true
# Wait for VM to actually stop
local stop_wait=0
while [[ $stop_wait -lt 30 ]]; do
local status
status=$(pve_vm_status "$VMID" 2>/dev/null) || status="unknown"
if [[ "$status" == "stopped" ]]; then
break
fi
sleep 2
stop_wait=$((stop_wait + 2))
done
log "VM stopped after ${stop_wait}s"
# Remove install media and set boot order
step "Removing install media (ide2, ide3) and setting boot to scsi0"
pve_api PUT "/nodes/${PVE_NODE}/qemu/${VMID}/config" \
'{"delete":"ide2,ide3","boot":"order=scsi0"}' json &>/dev/null || {
error "Failed to remove install media"
PHASE="failed"
return 1
}
log "Install media removed, boot order set to scsi0"
# --- Phase 3: Boot from disk ---
step "Starting VM from disk (first boot)"
pve_start_vm "$VMID"
PHASE="disk_boot"
log "VM started from disk — phase: disk_boot"
local boot_elapsed=0
local ip_detected=""
local ip_detect_started=false
# Screenshot loop during first boot
while [[ "$PHASE" == "disk_boot" ]]; do
take_frame || true
sleep "$SCREENSHOT_INTERVAL"
boot_elapsed=$((boot_elapsed + SCREENSHOT_INTERVAL))
# After 30s, start trying to detect IP
if [[ $boot_elapsed -ge 30 ]] && [[ "$ip_detect_started" != true ]]; then
ip_detect_started=true
log "Starting IP detection (${boot_elapsed}s into disk boot)"
fi
# Try IP detection every 10s after the initial 30s wait
if [[ "$ip_detect_started" == true ]] && [[ $((boot_elapsed % 10)) -lt $SCREENSHOT_INTERVAL ]]; then
local ip
ip=$(pve_vm_ip "$VMID" 2>/dev/null) || ip=""
if [[ -n "$ip" ]]; then
if [[ -z "$ip_detected" ]]; then
ip_detected="$ip"
log "IP detected: ${ip} (${boot_elapsed}s into disk boot)"
fi
# Check port 8443
if curl -sk --connect-timeout 3 "https://${ip}:8443" &>/dev/null; then
log "Port 8443 responding at ${ip} (${boot_elapsed}s into disk boot)"
DEPLOY_IP="$ip"
PHASE="ready"
break
else
log_quiet "Port 8443 not yet responding at ${ip}"
fi
fi
fi
if [[ $boot_elapsed -ge $BOOT_TIMEOUT ]]; then
log "Boot timeout after ${BOOT_TIMEOUT}s"
warn "Port 8443 not reachable after ${BOOT_TIMEOUT}s"
# Take final diagnostic frames
take_frame || true
PHASE="failed"
break
fi
# Progress logging every 15s
if [[ $((boot_elapsed % 15)) -lt $SCREENSHOT_INTERVAL ]]; then
log "Disk boot: ${boot_elapsed}s elapsed, ip=${ip_detected:-none}, phase=${PHASE}, frame=${FRAME_NUM}"
fi
done
# Take a few final frames after reaching ready state
if [[ "$PHASE" == "ready" ]]; then
for i in 1 2 3; do
sleep "$SCREENSHOT_INTERVAL"
take_frame || true
done
fi
# Definitive health check: query scrub_schedule via IncusOS REST API
if [[ "$PHASE" == "ready" ]] && [[ -n "$DEPLOY_IP" ]]; then
step "Checking scrub_schedule (definitive health indicator)"
if ! check_scrub_schedule "$DEPLOY_IP"; then
PHASE="crontab_bug"
log "Deploy reached port 8443 but scrub_schedule is empty — crontab bug"
fi
fi
local deploy_end
deploy_end=$(date +%s)
local total_seconds=$((deploy_end - deploy_start))
log "Deploy finished: phase=${PHASE}, scrub_schedule=${SCRUB_SCHEDULE:-unset}, total_time=${total_seconds}s, frames=${FRAME_NUM}"
if [[ "$PHASE" == "ready" ]]; then
return 0
else
return 1
fi
}
# ---------------------------------------------------------------------------
# Preflight checks
# ---------------------------------------------------------------------------
preflight() {
step "Preflight checks"
# Check required tools
local missing=()
command -v sshpass &>/dev/null || missing+=("sshpass")
command -v python3 &>/dev/null || missing+=("python3")
command -v curl &>/dev/null || missing+=("curl")
if [[ ${#missing[@]} -gt 0 ]]; then
error "Missing required tools: ${missing[*]}"
exit 1
fi
# Check PIL availability
if ! python3 -c "from PIL import Image" &>/dev/null; then
warn "python3 PIL/Pillow not available — screenshots will be saved as PPM (larger files)"
fi
# Check SSH connectivity
if ! sshpass -p "$PROXMOX_ROOT_PASSWORD" ssh "${SSH_OPTS[@]}" \
"root@${PVE_HOST}" "echo ok" &>/dev/null; then
error "Cannot SSH to root@${PVE_HOST}"
error "Check PROXMOX_ROOT_PASSWORD in env file and SSH access"
exit 1
fi
success "SSH to root@${PVE_HOST}: OK"
# Check VMID range (must be 850-869 per CLAUDE.md)
if [[ $VMID -lt 850 ]] || [[ $VMID -gt 869 ]]; then
error "VMID ${VMID} outside allowed test range (850-869)"
exit 1
fi
# Check VMID not in use
if pve_vm_exists "$VMID"; then
error "VMID ${VMID} already exists on Proxmox"
error "Clean it up first or use --vmid to pick a different one"
exit 1
fi
success "VMID ${VMID} available"
}
# ---------------------------------------------------------------------------
# Setup output directory
# ---------------------------------------------------------------------------
setup_output() {
local timestamp
timestamp=$(date '+%Y-%m-%d_%H-%M-%S')
if [[ -z "$LABEL" ]]; then
if [[ "$NO_UPDATE_YAML" == true ]]; then
LABEL="no-update-yaml"
else
LABEL="standard"
fi
fi
RUN_DIR="${SCRIPT_DIR}/observe-runs/${timestamp}_${LABEL}"
mkdir -p "$RUN_DIR"
LOG_FILE="${RUN_DIR}/deploy.log"
touch "$LOG_FILE"
success "Output directory: ${RUN_DIR}"
}
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
echo -e "${BOLD}${SCRIPT_NAME}${RESET} v${VERSION}"
echo -e "Observational single-VM deploy with rapid console screenshots"
echo ""
if [[ "$DRY_RUN" == true ]]; then
echo -e "${YELLOW}[dry-run] Preview mode — no VMs will be created${RESET}"
echo ""
echo "Configuration:"
echo " VMID: ${VMID}"
echo " VM name: ${VM_NAME}"
echo " Update.yaml: $(if [[ "$NO_UPDATE_YAML" == true ]]; then echo "OMITTED"; else echo "included (6h)"; fi)"
echo " Screenshot interval: ${SCREENSHOT_INTERVAL}s"
echo ""
step "Would load proxmox.yaml"
load_proxmox_yaml
detail "Host: ${PVE_HOST}, Node: ${PVE_NODE}, Storage: ${PVE_STORAGE}"
detail "Pool: ${PVE_POOL}, VLAN: ${PVE_VLAN:-none}, Token: ${PVE_API_TOKEN_ID}"
echo ""
step "Would check:"
detail "- SSH to root@${PVE_HOST} for screenshots"
detail "- VMID ${VMID} availability"
detail "- IncusOS ISO on Proxmox"
detail "- PIL/Pillow for PPM→PNG conversion"
echo ""
step "Would execute:"
detail "1. Generate seed ISO ($(if [[ "$NO_UPDATE_YAML" == true ]]; then echo "no update.yaml"; else echo "standard"; fi))"
detail "2. Upload seed to Proxmox"
detail "3. Create VM ${VMID} (${VM_NAME})"
detail "4. Start VM, begin screenshot loop (every ${SCREENSHOT_INTERVAL}s)"
detail "5. Monitor blockstat for install completion"
detail "6. Stop VM, remove media, start from disk"
detail "7. Continue screenshots until port 8443 responds (or ${BOOT_TIMEOUT}s timeout)"
detail "8. Check scrub_schedule via IncusOS REST API"
detail "9. Save frames + deploy.log"
detail "10. Cleanup VM"
if [[ $RUNS -gt 1 ]]; then
detail "Repeat ${RUNS} times with aggregate statistics"
fi
echo ""
info "Dry run complete"
exit 0
fi
# Load configuration
step "Loading configuration"
load_proxmox_yaml
detail "Host: ${PVE_HOST}, Node: ${PVE_NODE}, Storage: ${PVE_STORAGE}, VLAN: ${PVE_VLAN:-none}"
if [[ -n "$STATIC_IP" ]]; then
detail "Static IP: ${STATIC_IP}, Gateway: ${PVE_GATEWAY:-MISSING}"
if [[ -z "$PVE_GATEWAY" ]]; then
error "--ip requires 'gateway' in proxmox.yaml"
exit 1
fi
fi
# Run preflight checks
preflight
# Find IncusOS ISO
find_incusos_iso
# Trap for clean exit
trap_cleanup() {
echo ""
warn "Interrupted — cleaning up"
cleanup_vm
exit 130
}
trap trap_cleanup INT TERM
# ---------------------------------------------------------------------------
# Run loop (single run or multi-run with --runs N)
# ---------------------------------------------------------------------------
run_single_cycle() {
# Runs one complete deploy+destroy cycle. Returns 0 on success, 1 on failure.
# Sets globals: PHASE, SCRUB_SCHEDULE, DEPLOY_IP, FRAME_NUM
# Reset state for this run
PHASE="init"
PREV_WR_BYTES=0
WRITES_STARTED=false
STABLE_COUNT=0
SCRUB_SCHEDULE=""
DEPLOY_IP=""
FRAME_NUM=0
# Setup per-run output directory
setup_output
# Log configuration
log "=== observe-deploy v${VERSION} ==="
log "VMID: ${VMID}"
log "VM name: ${VM_NAME}"
log "ISO: ${ISO_FILENAME}"
log "Update.yaml: $(if [[ "$NO_UPDATE_YAML" == true ]]; then echo "OMITTED"; else echo "included (6h)"; fi)"
log "Screenshot interval: ${SCREENSHOT_INTERVAL}s"
log "Install timeout: ${INSTALL_TIMEOUT}s"
log "Boot timeout: ${BOOT_TIMEOUT}s"
# Generate and upload seed
generate_seed
# Create VM
create_vm
echo ""
echo -e "${BOLD}Starting observational deploy${RESET}"
echo " Frames saved to: ${RUN_DIR}/"
echo " Log: ${RUN_DIR}/deploy.log"
echo " Press Ctrl+C to abort (VM will be cleaned up)"
echo ""
# Run the deploy with screenshots
local result=0
run_deploy || result=$?
echo ""
echo -e "${BOLD}========================================${RESET}"
echo -e "${BOLD} Deploy Results${RESET}"
echo -e "${BOLD}========================================${RESET}"
echo ""
echo " VMID: ${VMID}"
echo " Seed: $(if [[ "$NO_UPDATE_YAML" == true ]]; then echo "no update.yaml"; else echo "standard (check_frequency=6h)"; fi)"
if [[ $result -eq 0 ]]; then
echo -e " Outcome: ${GREEN}PASS${RESET} — port 8443 reachable, scrub_schedule='${SCRUB_SCHEDULE}'"
elif [[ "$PHASE" == "crontab_bug" ]]; then
echo -e " Outcome: ${RED}FAIL${RESET} — crontab bug (port 8443 up, scrub_schedule empty)"
else
echo -e " Outcome: ${RED}FAIL${RESET} — phase: ${PHASE}"
fi
echo " Frames: ${FRAME_NUM}"
echo " Output: ${RUN_DIR}/"
echo ""
# Cleanup
step "Post-deploy cleanup"
cleanup_vm
return $result
}
if [[ $RUNS -le 1 ]]; then
# Single-run mode (original behavior)
deploy_result=0
run_single_cycle || deploy_result=$?
if [[ $deploy_result -eq 0 ]]; then
success "Observational deploy complete — ${FRAME_NUM} frames captured"
else
warn "Deploy failed at phase: ${PHASE}"
warn "Check screenshots and log for diagnosis:"
warn " ${RUN_DIR}/deploy.log"
warn " ${RUN_DIR}/frame-*.png"
fi
echo ""
echo "To view key frames:"
echo " ls ${RUN_DIR}/frame-*.png"
echo " # View in image viewer or analyze with Claude Code"
echo ""
exit $deploy_result
else
# Multi-run mode: N sequential deploy+destroy cycles with aggregate stats
echo ""
echo -e "${BOLD}Multi-run mode: ${RUNS} sequential deploy cycles${RESET}"
echo ""
# Compute base label before the loop (LABEL may be empty at this point)
if [[ -z "$LABEL" ]]; then
if [[ "$NO_UPDATE_YAML" == true ]]; then
BASE_LABEL="no-update-yaml"
else
BASE_LABEL="standard"
fi
else
BASE_LABEL="$LABEL"
fi
total_pass=0
total_fail=0
total_crontab_bug=0
total_timeout=0
total_time=0
run_results=() # Array of "PASS|FAIL|BUG" per run
overall_start=$(date +%s)
run_num=0
while [[ $run_num -lt $RUNS ]]; do
run_num=$((run_num + 1))
echo ""
echo -e "${BOLD}╔══════════════════════════════════════╗${RESET}"
echo -e "${BOLD}║ Run ${run_num}/${RUNS}${RESET}"
echo -e "${BOLD}╚══════════════════════════════════════╝${RESET}"
echo ""
# Set per-run label
LABEL="${BASE_LABEL}_run-$(printf '%02d' "$run_num")"
local_start=$(date +%s)
deploy_result=0
run_single_cycle || deploy_result=$?
local_end=$(date +%s)
local_duration=$((local_end - local_start))
total_time=$((total_time + local_duration))
if [[ $deploy_result -eq 0 ]]; then
total_pass=$((total_pass + 1))
run_results+=("PASS")
success "Run ${run_num}/${RUNS}: PASS (${local_duration}s)"
elif [[ "$PHASE" == "crontab_bug" ]]; then
total_fail=$((total_fail + 1))
total_crontab_bug=$((total_crontab_bug + 1))
run_results+=("BUG")
warn "Run ${run_num}/${RUNS}: FAIL — crontab bug (${local_duration}s)"
else
total_fail=$((total_fail + 1))
total_timeout=$((total_timeout + 1))
run_results+=("FAIL")
warn "Run ${run_num}/${RUNS}: FAIL — ${PHASE} (${local_duration}s)"
fi
# Brief pause between runs to let Proxmox settle
if [[ $run_num -lt $RUNS ]]; then
info "Pausing 10s before next run..."
sleep 10
fi
done
overall_end=$(date +%s)
overall_duration=$((overall_end - overall_start))
avg_time=$((total_time / RUNS))
echo ""
echo -e "${BOLD}╔══════════════════════════════════════════════════╗${RESET}"
echo -e "${BOLD}║ Multi-Run Summary (${RUNS} cycles)${RESET}"
echo -e "${BOLD}╚══════════════════════════════════════════════════╝${RESET}"
echo ""
echo -e " ${GREEN}PASS:${RESET} ${total_pass}/${RUNS}"
echo -e " ${RED}FAIL:${RESET} ${total_fail}/${RUNS}"
if [[ $total_crontab_bug -gt 0 ]]; then
echo -e " ${RED}crontab bug:${RESET} ${total_crontab_bug}"
fi
if [[ $total_timeout -gt 0 ]]; then
echo -e " ${RED}timeout/other:${RESET} ${total_timeout}"
fi
echo ""
echo " Avg time: ${avg_time}s per run"
echo " Total time: ${overall_duration}s"
echo ""
# Per-run results
echo " Per-run: ${run_results[*]}"
echo ""
# Success rate
if [[ $RUNS -gt 0 ]]; then
pct=$((total_pass * 100 / RUNS))
echo -e " Success rate: ${pct}%"
if [[ $pct -ge 95 ]]; then
echo -e " ${GREEN}Target met (≥95%)${RESET}"
else
echo -e " ${RED}Below target (<95%)${RESET}"
fi
fi
echo ""
# Output directories
echo " Output: ${SCRIPT_DIR}/observe-runs/*${BASE_LABEL}*"
echo ""
if [[ $total_fail -gt 0 ]]; then
exit 1
fi
exit 0
fi
Reference in New Issue View Git Blame Copy Permalink