579 lines
22 KiB
Bash
Executable File
579 lines
22 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# investigate-parallel - Parallel IncusOS boot investigation with smart phase detection
|
|
#
|
|
# Runs N VMs simultaneously, monitors each independently, probes API on success.
|
|
# Uses aggressive port polling + screenshot capture for phase detection.
|
|
#
|
|
# Usage:
|
|
# ./investigate-parallel --count 4 --iso old --label baseline
|
|
# ./investigate-parallel --count 4 --iso new --label version-mismatch
|
|
# ./investigate-parallel --count 4 --iso /tmp/IncusOS-oc.iso --label oc-iso
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
SEED_TOOL="${SCRIPT_DIR}/incusos-seed"
|
|
PVE_API="${SCRIPT_DIR}/pve-api"
|
|
|
|
# Defaults
|
|
COUNT=4
|
|
START_VMID=850
|
|
ISO_MODE="old" # old, new, or file path
|
|
LABEL=""
|
|
DRY_RUN=false
|
|
BOOT_TIMEOUT=240 # longer for version-mismatch OS update path
|
|
|
|
# Proxmox config
|
|
PVE_HOST=""
|
|
PVE_NODE=""
|
|
PVE_STORAGE=""
|
|
PVE_ISO_STORAGE=""
|
|
PVE_BRIDGE=""
|
|
PVE_POOL=""
|
|
|
|
# ISO
|
|
ISO_FILENAME=""
|
|
ISO_VERSION=""
|
|
|
|
# Results
|
|
RESULTS_DIR=""
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Colors
|
|
# ---------------------------------------------------------------------------
|
|
|
|
if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then
|
|
RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[0;33m'
|
|
CYAN='\033[0;36m' BOLD='\033[1m' DIM='\033[2m' RESET='\033[0m'
|
|
else
|
|
RED='' GREEN='' YELLOW='' CYAN='' BOLD='' DIM='' RESET=''
|
|
fi
|
|
|
|
step() { echo -e "${CYAN}[step]${RESET} ${BOLD}$*${RESET}"; }
|
|
success() { echo -e "${GREEN}[ok]${RESET} $*"; }
|
|
warn() { echo -e "${YELLOW}[warn]${RESET} $*" >&2; }
|
|
error() { echo -e "${RED}[error]${RESET} $*" >&2; }
|
|
info() { echo -e "${DIM}[info]${RESET} $*"; }
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Argument parsing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--count) COUNT="${2:?'--count requires N'}"; shift 2 ;;
|
|
--iso) ISO_MODE="${2:?'--iso requires old|new|path'}"; shift 2 ;;
|
|
--label) LABEL="${2:?'--label requires string'}"; shift 2 ;;
|
|
--start-vmid) START_VMID="${2:?'--start-vmid requires N'}"; shift 2 ;;
|
|
--dry-run) DRY_RUN=true; shift ;;
|
|
--timeout) BOOT_TIMEOUT="${2:?'--timeout requires seconds'}"; shift 2 ;;
|
|
-h|--help)
|
|
echo "Usage: investigate-parallel [OPTIONS]"
|
|
echo " --count N Parallel VMs (default: 4, max 8)"
|
|
echo " --iso MODE old (202602200553), new (latest CDN), or file path"
|
|
echo " --label NAME Results label"
|
|
echo " --start-vmid N First VMID (default: 850)"
|
|
echo " --timeout N Boot timeout seconds (default: 240)"
|
|
echo " --dry-run Preview only"
|
|
exit 0
|
|
;;
|
|
*) error "Unknown: $1"; exit 1 ;;
|
|
esac
|
|
done
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Load environment & config
|
|
# ---------------------------------------------------------------------------
|
|
|
|
load_env() {
|
|
local dir="$SCRIPT_DIR"
|
|
while [[ "$dir" != "/" ]]; do
|
|
[[ -f "${dir}/env" ]] && { source "${dir}/env"; return; }
|
|
dir="$(dirname "$dir")"
|
|
done
|
|
}
|
|
load_env
|
|
|
|
[[ -z "${PROXMOX_TOKEN_SECRET:-}" ]] && { error "PROXMOX_TOKEN_SECRET not set"; exit 1; }
|
|
[[ -z "${PROXMOX_ROOT_PASSWORD:-}" ]] && { error "PROXMOX_ROOT_PASSWORD not set"; exit 1; }
|
|
|
|
yaml_get() {
|
|
local val
|
|
val=$(awk -F': ' -v k="$1" '$1 == k {print $2; exit}' "${SCRIPT_DIR}/proxmox.yaml" 2>/dev/null) || val=""
|
|
val="${val#\"}" ; val="${val%\"}" ; val="${val#\'}" ; val="${val%\'}"
|
|
echo "${val:-${2:-}}"
|
|
}
|
|
|
|
PVE_HOST=$(yaml_get host)
|
|
PVE_NODE=$(yaml_get node pve)
|
|
PVE_STORAGE=$(yaml_get storage local-lvm)
|
|
PVE_ISO_STORAGE=$(yaml_get iso_storage local)
|
|
PVE_BRIDGE=$(yaml_get bridge vmbr0)
|
|
PVE_POOL=$(yaml_get pool)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Screenshot helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -o LogLevel=ERROR)
|
|
|
|
take_screenshot() {
|
|
local vmid="$1" output="$2"
|
|
local rppm="/tmp/vm-${vmid}-screen.ppm"
|
|
sshpass -p "$PROXMOX_ROOT_PASSWORD" ssh "${SSH_OPTS[@]}" "root@${PVE_HOST}" \
|
|
"echo 'screendump ${rppm}' | qm monitor ${vmid}" &>/dev/null || return 1
|
|
sshpass -p "$PROXMOX_ROOT_PASSWORD" scp "${SSH_OPTS[@]}" \
|
|
"root@${PVE_HOST}:${rppm}" "/tmp/vm-${vmid}-screen.ppm" &>/dev/null || return 1
|
|
sshpass -p "$PROXMOX_ROOT_PASSWORD" ssh "${SSH_OPTS[@]}" "root@${PVE_HOST}" \
|
|
"rm -f ${rppm}" &>/dev/null || true
|
|
python3 -c "from PIL import Image; Image.open('/tmp/vm-${vmid}-screen.ppm').save('${output}')" 2>/dev/null || {
|
|
mv "/tmp/vm-${vmid}-screen.ppm" "${output%.png}.ppm" 2>/dev/null || true
|
|
}
|
|
rm -f "/tmp/vm-${vmid}-screen.ppm" 2>/dev/null || true
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ISO resolution
|
|
# ---------------------------------------------------------------------------
|
|
|
|
resolve_iso() {
|
|
case "$ISO_MODE" in
|
|
old)
|
|
ISO_FILENAME="IncusOS_202602200553.iso"
|
|
ISO_VERSION="202602200553"
|
|
;;
|
|
new)
|
|
# Find latest on Proxmox
|
|
local latest
|
|
latest=$("$PVE_API" "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content?content=iso" 2>/dev/null | python3 -c "
|
|
import json, sys
|
|
data = json.load(sys.stdin).get('data', [])
|
|
isos = [d['volid'].split('/')[-1] for d in data if 'IncusOS_' in d.get('volid','') and '-oc' not in d['volid'].split('/')[-1].lower()]
|
|
isos.sort(reverse=True)
|
|
print(isos[0] if isos else '')
|
|
" 2>/dev/null)
|
|
if [[ -z "$latest" ]]; then
|
|
error "No IncusOS ISO found on Proxmox"; exit 1
|
|
fi
|
|
ISO_FILENAME="$latest"
|
|
ISO_VERSION=$(echo "$latest" | sed -n 's/IncusOS_\([0-9]*\)\..*/\1/p')
|
|
;;
|
|
*)
|
|
# File path — must already be uploaded
|
|
ISO_FILENAME="$(basename "$ISO_MODE")"
|
|
ISO_VERSION=$(echo "$ISO_FILENAME" | sed -n 's/.*_\([0-9]*\)\..*/\1/p')
|
|
[[ -z "$ISO_VERSION" ]] && ISO_VERSION="custom"
|
|
;;
|
|
esac
|
|
success "ISO: ${ISO_FILENAME} (version ${ISO_VERSION})"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Single VM lifecycle (called as background job)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
deploy_single_vm() {
|
|
local vmid="$1"
|
|
local vm_name="$2"
|
|
local run_dir="$3"
|
|
local seed_iso="$4"
|
|
local result_file="${run_dir}/result.txt"
|
|
local log_file="${run_dir}/deploy.log"
|
|
|
|
_log() { echo "[$(date '+%H:%M:%S.%3N')] $*" >> "$log_file"; }
|
|
|
|
# Error trap: write FAIL result on any uncaught error
|
|
trap '_log "ERROR at line $LINENO: exit code $?"; echo "ERROR|line_${LINENO}|0|N/A|N/A|N/A" > "$result_file"' ERR
|
|
|
|
_log "=== VM ${vmid} (${vm_name}) ==="
|
|
_log "ISO: ${ISO_FILENAME}, seed: ${seed_iso}"
|
|
|
|
# Create VM
|
|
local json_payload
|
|
json_payload=$(python3 -c "
|
|
import json
|
|
p = {
|
|
'vmid': ${vmid}, 'name': '${vm_name}',
|
|
'description': '[incusos-lab:managed] investigate-parallel',
|
|
'ostype': 'l26', 'bios': 'ovmf', 'machine': 'q35',
|
|
'efidisk0': '${PVE_STORAGE}:0,efitype=4m,pre-enrolled-keys=0',
|
|
'tpmstate0': '${PVE_STORAGE}:1,version=v2.0',
|
|
'cpu': 'host', 'cores': 4, 'memory': 4096, 'balloon': 0,
|
|
'scsihw': 'virtio-scsi-pci', 'scsi0': '${PVE_STORAGE}:50,iothread=1',
|
|
'ide2': '${PVE_ISO_STORAGE}:iso/${ISO_FILENAME},media=cdrom',
|
|
'ide3': '${PVE_ISO_STORAGE}:iso/${seed_iso},media=cdrom',
|
|
'net0': 'virtio,bridge=${PVE_BRIDGE}',
|
|
'boot': 'order=ide2;scsi0', 'agent': 1,
|
|
}
|
|
pool = '${PVE_POOL}'
|
|
if pool: p['pool'] = pool
|
|
print(json.dumps(p))
|
|
")
|
|
|
|
"$PVE_API" "/nodes/${PVE_NODE}/qemu" POST "$json_payload" >/dev/null 2>&1 || {
|
|
echo "ERROR|create_failed|0|N/A|N/A|N/A" > "$result_file"; return
|
|
}
|
|
sleep 15 # Wait for create task (allocating EFI, TPM, 50G disk)
|
|
_log "VM created"
|
|
|
|
# Start VM (ISO boot)
|
|
"$PVE_API" "/nodes/${PVE_NODE}/qemu/${vmid}/status/start" POST '{}' >/dev/null 2>&1 || {
|
|
_log "ERROR: Failed to start VM"; echo "ERROR|start_failed|0|N/A|N/A|N/A" > "$result_file"; return
|
|
}
|
|
_log "VM started"
|
|
|
|
local frame=0
|
|
|
|
# Monitor install via blockstat (screenshots every 15s to avoid SSH contention)
|
|
local elapsed=0 prev_wr=0 writes_started=false stable=0
|
|
while [[ $elapsed -lt 300 ]]; do
|
|
sleep 5; elapsed=$((elapsed + 5))
|
|
# Screenshot every 15s during install
|
|
if [[ $((elapsed % 15)) -lt 6 ]]; then
|
|
take_screenshot "$vmid" "${run_dir}/frame-$(printf '%03d' $((++frame))).png" 2>/dev/null || true
|
|
fi
|
|
|
|
local wr
|
|
wr=$("$PVE_API" "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" 2>/dev/null | python3 -c "
|
|
import json,sys; d=json.load(sys.stdin).get('data',{}); print(d.get('blockstat',{}).get('scsi0',{}).get('wr_bytes',0))" 2>/dev/null) || wr=0
|
|
|
|
if [[ $wr -gt $prev_wr ]]; then
|
|
[[ "$writes_started" != true ]] && { writes_started=true; _log "Install writes started"; }
|
|
stable=0
|
|
elif [[ "$writes_started" == true ]]; then
|
|
stable=$((stable + 1))
|
|
[[ $stable -ge 3 ]] && { _log "Install complete ($(( wr / 1048576 )) MiB)"; break; }
|
|
fi
|
|
prev_wr=$wr
|
|
|
|
# Check if VM stopped
|
|
local st
|
|
st=$("$PVE_API" "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin)['data']['status'])" 2>/dev/null) || st="unknown"
|
|
[[ "$st" == "stopped" ]] && { _log "VM stopped during install"; break; }
|
|
done
|
|
|
|
# Take screenshot showing install-complete/media-error state
|
|
take_screenshot "$vmid" "${run_dir}/frame-$(printf '%03d' $((++frame))).png" 2>/dev/null || true
|
|
|
|
# Transition: stop, remove media, boot from disk
|
|
"$PVE_API" "/nodes/${PVE_NODE}/qemu/${vmid}/status/stop" POST '{}' >/dev/null 2>&1 || true
|
|
sleep 8
|
|
# Wait for stop to complete
|
|
for _ in 1 2 3 4 5; do
|
|
local vm_st
|
|
vm_st=$("$PVE_API" "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin)['data']['status'])" 2>/dev/null) || vm_st="unknown"
|
|
[[ "$vm_st" == "stopped" ]] && break
|
|
sleep 3
|
|
done
|
|
"$PVE_API" "/nodes/${PVE_NODE}/qemu/${vmid}/config" "PUT" '{"delete":"ide2,ide3","boot":"order=scsi0"}' >/dev/null 2>&1 || {
|
|
_log "ERROR: Failed to remove media"; echo "ERROR|media_remove|0|N/A|N/A|N/A" > "$result_file"; return
|
|
}
|
|
_log "Media removed"
|
|
|
|
"$PVE_API" "/nodes/${PVE_NODE}/qemu/${vmid}/status/start" POST '{}' >/dev/null 2>&1 || {
|
|
_log "ERROR: Failed to start from disk"; echo "ERROR|disk_start|0|N/A|N/A|N/A" > "$result_file"; return
|
|
}
|
|
_log "Disk boot started"
|
|
local boot_start
|
|
boot_start=$(date +%s)
|
|
|
|
# Poll for port 8443 with periodic screenshots
|
|
local ip="" boot_elapsed=0 port_up=false
|
|
while [[ $boot_elapsed -lt $BOOT_TIMEOUT ]]; do
|
|
sleep 5; boot_elapsed=$(( $(date +%s) - boot_start ))
|
|
# Screenshot every 15s during boot
|
|
if [[ $((boot_elapsed % 15)) -lt 6 ]]; then
|
|
take_screenshot "$vmid" "${run_dir}/frame-$(printf '%03d' $((++frame))).png" 2>/dev/null || true
|
|
fi
|
|
|
|
# Try to find IP (after 15s)
|
|
if [[ $boot_elapsed -ge 15 ]] && [[ -z "$ip" ]]; then
|
|
# Get MAC from Proxmox config (only once)
|
|
if [[ -z "${mac:-}" ]]; then
|
|
mac=$("$PVE_API" "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null | python3 -c "
|
|
import json, sys, re
|
|
net0 = json.load(sys.stdin).get('data',{}).get('net0','')
|
|
m = re.search(r'([0-9A-Fa-f:]{17})', net0)
|
|
print(m.group(1).lower() if m else '')
|
|
" 2>/dev/null) || mac=""
|
|
[[ -n "$mac" ]] && _log "MAC: ${mac}"
|
|
fi
|
|
if [[ -n "${mac:-}" ]]; then
|
|
# Check ARP table (don't flush — other VMs are using it)
|
|
ip=$(ip neigh show | awk -v mac="$mac" 'tolower($0) ~ tolower(mac) {print $1; exit}') || ip=""
|
|
# If not found, do a targeted ARP probe: ping a range of likely DHCP IPs
|
|
if [[ -z "$ip" ]]; then
|
|
# Only sweep every 30 seconds to avoid overload
|
|
local now_s
|
|
now_s=$(date +%s)
|
|
if [[ $(( now_s - ${_last_sweep:-0} )) -ge 25 ]]; then
|
|
_last_sweep=$now_s
|
|
_log "Running ARP sweep"
|
|
# Sweep in smaller batches to avoid overwhelming
|
|
for octet in $(seq 1 254); do
|
|
ping -c 1 -W 1 "192.168.1.${octet}" &>/dev/null &
|
|
done
|
|
wait 2>/dev/null || true
|
|
sleep 2
|
|
fi
|
|
ip=$(ip neigh show | awk -v mac="$mac" 'tolower($0) ~ tolower(mac) {print $1; exit}') || ip=""
|
|
fi
|
|
if [[ -n "$ip" ]]; then
|
|
if ping -c 1 -W 2 "$ip" &>/dev/null; then
|
|
_log "IP detected: ${ip}"
|
|
else
|
|
_log "IP ${ip} not responding, retrying"
|
|
ip=""
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Check port 8443
|
|
if [[ -n "$ip" ]]; then
|
|
if curl -sk --connect-timeout 3 "https://${ip}:8443" &>/dev/null; then
|
|
port_up=true
|
|
_log "Port 8443 UP at ${ip} (${boot_elapsed}s)"
|
|
break
|
|
fi
|
|
fi
|
|
|
|
[[ $((boot_elapsed % 30)) -lt 6 ]] && _log "Disk boot: ${boot_elapsed}s, ip=${ip:-none}"
|
|
done
|
|
|
|
# Take final screenshot
|
|
take_screenshot "$vmid" "${run_dir}/frame-$(printf '%03d' $((++frame))).png" 2>/dev/null || true
|
|
|
|
# Probe API if successful
|
|
local scrub="N/A" freq="N/A" osver="N/A"
|
|
if [[ "$port_up" == true ]] && [[ -n "$ip" ]]; then
|
|
# Add incus remote and probe
|
|
incus remote add "$vm_name" "https://${ip}:8443" --accept-certificate 2>/dev/null || true
|
|
local storage_json update_json system_json
|
|
storage_json=$(incus query "${vm_name}:/os/1.0/system/storage" 2>/dev/null) || storage_json='{}'
|
|
update_json=$(incus query "${vm_name}:/os/1.0/system/update" 2>/dev/null) || update_json='{}'
|
|
system_json=$(incus query "${vm_name}:/os/1.0" 2>/dev/null) || system_json='{}'
|
|
|
|
scrub=$(echo "$storage_json" | python3 -c "import json,sys; print(json.load(sys.stdin).get('config',{}).get('scrub_schedule','EMPTY'))" 2>/dev/null) || scrub="error"
|
|
freq=$(echo "$update_json" | python3 -c "import json,sys; print(json.load(sys.stdin).get('config',{}).get('check_frequency','EMPTY'))" 2>/dev/null) || freq="error"
|
|
osver=$(echo "$system_json" | python3 -c "import json,sys; print(json.load(sys.stdin).get('environment',{}).get('os_version','EMPTY'))" 2>/dev/null) || osver="error"
|
|
|
|
_log "API: scrub='${scrub}' freq='${freq}' os=${osver}"
|
|
|
|
# Save full probe data
|
|
echo "$storage_json" > "${run_dir}/api-storage.json" 2>/dev/null || true
|
|
echo "$update_json" > "${run_dir}/api-update.json" 2>/dev/null || true
|
|
echo "$system_json" > "${run_dir}/api-system.json" 2>/dev/null || true
|
|
|
|
# Cleanup remote
|
|
local cur_remote
|
|
cur_remote=$(incus remote get-default 2>/dev/null) || true
|
|
[[ "$cur_remote" == "$vm_name" ]] && incus remote switch local 2>/dev/null || true
|
|
incus remote remove "$vm_name" 2>/dev/null || true
|
|
fi
|
|
|
|
# Determine outcome:
|
|
# - PASS = port up AND scrub_schedule is set (genuine success)
|
|
# - CRONTAB_BUG = port up but scrub_schedule empty (detected during error window)
|
|
# - FAIL = port never came up (could be bug + bad timing, or IP detection failure)
|
|
local outcome="FAIL"
|
|
if [[ "$port_up" == true ]]; then
|
|
if [[ -n "$scrub" ]] && [[ "$scrub" != "" ]] && [[ "$scrub" != "EMPTY" ]] && [[ "$scrub" != "N/A" ]]; then
|
|
outcome="PASS"
|
|
else
|
|
outcome="CRONTAB_BUG"
|
|
fi
|
|
fi
|
|
|
|
echo "${outcome}|${boot_elapsed}|${scrub}|${freq}|${osver}|${ip:-none}" > "$result_file"
|
|
_log "RESULT: ${outcome} boot=${boot_elapsed}s scrub='${scrub}' freq='${freq}' os=${osver}"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cleanup single VM
|
|
# ---------------------------------------------------------------------------
|
|
|
|
cleanup_vm() {
|
|
local vmid="$1"
|
|
local seed_iso="$2"
|
|
"$PVE_API" "/nodes/${PVE_NODE}/qemu/${vmid}/status/stop" POST >/dev/null 2>&1 || true
|
|
sleep 3
|
|
"$PVE_API" "/nodes/${PVE_NODE}/qemu/${vmid}?purge=1&destroy-unreferenced-disks=1" DELETE >/dev/null 2>&1 || true
|
|
"$PVE_API" "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content/${PVE_ISO_STORAGE}:iso/${seed_iso}" DELETE >/dev/null 2>&1 || true
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main
|
|
# ---------------------------------------------------------------------------
|
|
|
|
echo -e "${BOLD}investigate-parallel${RESET} — Parallel IncusOS boot investigation"
|
|
echo ""
|
|
|
|
[[ -z "$LABEL" ]] && LABEL="${ISO_MODE}"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
echo " Count: ${COUNT}, ISO: ${ISO_MODE}, VMIDs: ${START_VMID}-$((START_VMID + COUNT - 1))"
|
|
exit 0
|
|
fi
|
|
|
|
step "Resolving ISO"
|
|
resolve_iso
|
|
|
|
RESULTS_DIR="${SCRIPT_DIR}/observe-runs/parallel_$(date '+%Y-%m-%d_%H-%M-%S')_${LABEL}"
|
|
mkdir -p "$RESULTS_DIR"
|
|
success "Output: ${RESULTS_DIR}/"
|
|
|
|
# Generate seeds and create run dirs
|
|
step "Generating ${COUNT} seeds"
|
|
declare -a VM_NAMES=() VM_SEEDS=() VM_DIRS=()
|
|
for ((i=0; i<COUNT; i++)); do
|
|
vmid=$((START_VMID + i))
|
|
vm_name="inv-${vmid}"
|
|
seed_name="seed-inv-${vmid}.iso"
|
|
run_dir="${RESULTS_DIR}/vm-${vmid}"
|
|
mkdir -p "$run_dir"
|
|
|
|
VM_NAMES+=("$vm_name")
|
|
VM_SEEDS+=("$seed_name")
|
|
VM_DIRS+=("$run_dir")
|
|
|
|
local_seed="/tmp/${seed_name}"
|
|
"$SEED_TOOL" --format iso --app incus --output "$local_seed" \
|
|
--force-install --force-reboot --hostname "$vm_name" \
|
|
--defaults --quiet &>/dev/null
|
|
|
|
# Upload seed (use Python to avoid bash ! quoting issues with API token)
|
|
url="https://${PVE_HOST}:8006/api2/json/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/upload"
|
|
python3 -c "
|
|
import subprocess, os, sys
|
|
# Read token_id from proxmox.yaml
|
|
token_id = ''
|
|
yaml_file = '${SCRIPT_DIR}/proxmox.yaml'
|
|
for line in open(yaml_file):
|
|
line = line.strip()
|
|
if line.startswith('api_token_id:'):
|
|
token_id = line.split(': ', 1)[1].strip().strip(\"'\\\"\")
|
|
secret = os.environ.get('PROXMOX_TOKEN_SECRET', '')
|
|
if not token_id or not secret:
|
|
sys.exit(1)
|
|
r = subprocess.run([
|
|
'curl', '-fsSk',
|
|
'-H', 'Authorization: PVEAPIToken=' + token_id + '=' + secret,
|
|
'-F', 'content=iso',
|
|
'-F', 'filename=@${local_seed};filename=${seed_name}',
|
|
'${url}'
|
|
], capture_output=True)
|
|
sys.exit(r.returncode)
|
|
" 2>/dev/null
|
|
rm -f "$local_seed"
|
|
info "Seed $((i+1))/${COUNT}: ${seed_name}"
|
|
done
|
|
success "All seeds uploaded"
|
|
|
|
# Check VMIDs are free
|
|
step "Checking VMIDs"
|
|
for ((i=0; i<COUNT; i++)); do
|
|
vmid=$((START_VMID + i))
|
|
if "$PVE_API" "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" >/dev/null 2>&1; then
|
|
error "VMID ${vmid} already in use"; exit 1
|
|
fi
|
|
done
|
|
success "VMIDs ${START_VMID}-$((START_VMID + COUNT - 1)) available"
|
|
|
|
echo ""
|
|
echo -e "${BOLD}Launching ${COUNT} VMs in parallel${RESET}"
|
|
echo " ISO: ${ISO_FILENAME} (${ISO_VERSION})"
|
|
echo " VMIDs: ${START_VMID}-$((START_VMID + COUNT - 1))"
|
|
echo " Boot timeout: ${BOOT_TIMEOUT}s"
|
|
echo ""
|
|
|
|
# Trap for cleanup
|
|
trap_cleanup() {
|
|
echo ""
|
|
warn "Interrupted — cleaning up all VMs"
|
|
for ((i=0; i<COUNT; i++)); do
|
|
cleanup_vm "$((START_VMID + i))" "${VM_SEEDS[$i]}" &
|
|
done
|
|
wait
|
|
exit 130
|
|
}
|
|
trap trap_cleanup INT TERM
|
|
|
|
# Launch all VMs in parallel
|
|
pids=()
|
|
for ((i=0; i<COUNT; i++)); do
|
|
vmid=$((START_VMID + i))
|
|
( set +e; deploy_single_vm "$vmid" "${VM_NAMES[$i]}" "${VM_DIRS[$i]}" "${VM_SEEDS[$i]}" 2>"${VM_DIRS[$i]}/stderr.log" ) &
|
|
pids+=($!)
|
|
info "Started VM ${vmid} (PID $!)"
|
|
done
|
|
|
|
# Wait for all to complete
|
|
for pid in "${pids[@]}"; do
|
|
wait "$pid" || true
|
|
done
|
|
|
|
echo ""
|
|
echo -e "${BOLD}════════════════════════════════════════════════${RESET}"
|
|
echo -e "${BOLD} Results (ISO: ${ISO_FILENAME})${RESET}"
|
|
echo -e "${BOLD}════════════════════════════════════════════════${RESET}"
|
|
echo ""
|
|
|
|
pass=0 fail=0 crontab_bug=0
|
|
for ((i=0; i<COUNT; i++)); do
|
|
vmid=$((START_VMID + i))
|
|
rfile="${VM_DIRS[$i]}/result.txt"
|
|
if [[ -f "$rfile" ]]; then
|
|
IFS='|' read -r outcome boot_s scrub freq osver vm_ip < "$rfile"
|
|
if [[ "$outcome" == "PASS" ]]; then
|
|
marker="${GREEN}PASS${RESET}"; pass=$((pass + 1))
|
|
elif [[ "$outcome" == "CRONTAB_BUG" ]]; then
|
|
marker="${YELLOW}CRONTAB_BUG${RESET}"; crontab_bug=$((crontab_bug + 1))
|
|
else
|
|
marker="${RED}FAIL${RESET}"; fail=$((fail + 1))
|
|
fi
|
|
echo -e " ${vmid}: ${marker} boot=${boot_s}s scrub='${scrub}' freq='${freq}' os=${osver} ip=${vm_ip}"
|
|
else
|
|
echo -e " ${vmid}: ${RED}NO RESULT${RESET}"
|
|
fail=$((fail + 1))
|
|
fi
|
|
done
|
|
|
|
echo ""
|
|
echo -e " Total: ${GREEN}${pass} PASS${RESET}, ${YELLOW}${crontab_bug} CRONTAB_BUG${RESET}, ${RED}${fail} FAIL/NO_IP${RESET} (of ${COUNT})"
|
|
total_bad=$((crontab_bug + fail))
|
|
[[ $total_bad -gt 0 ]] && echo -e " Bug rate: ${YELLOW}$(( crontab_bug * 100 / COUNT ))%${RESET} confirmed, ${RED}$(( fail * 100 / COUNT ))%${RESET} unknown (IP detection failure)"
|
|
|
|
# Write summary
|
|
{
|
|
echo "# investigate-parallel results"
|
|
echo "# $(date -Iseconds)"
|
|
echo "# ISO: ${ISO_FILENAME} (${ISO_VERSION})"
|
|
echo "# Count: ${COUNT}, VMIDs: ${START_VMID}-$((START_VMID + COUNT - 1))"
|
|
echo "#"
|
|
echo "# vmid|outcome|boot_s|scrub_schedule|check_frequency|os_version|ip"
|
|
for ((i=0; i<COUNT; i++)); do
|
|
vmid=$((START_VMID + i))
|
|
rfile="${VM_DIRS[$i]}/result.txt"
|
|
[[ -f "$rfile" ]] && echo "${vmid}|$(cat "$rfile")"
|
|
done
|
|
} > "${RESULTS_DIR}/results.txt"
|
|
|
|
echo ""
|
|
|
|
# Cleanup all VMs
|
|
step "Cleaning up"
|
|
for ((i=0; i<COUNT; i++)); do
|
|
cleanup_vm "$((START_VMID + i))" "${VM_SEEDS[$i]}" &
|
|
done
|
|
wait
|
|
sleep 3
|
|
success "All cleaned up"
|
|
|
|
echo ""
|
|
echo "Screenshots: ${RESULTS_DIR}/vm-*/frame-*.png"
|
|
echo "Results: ${RESULTS_DIR}/results.txt"
|
|
echo ""
|
|
|
|
exit $fail
|