maarten
/
incus-contrib
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
incus-contrib/bin/diagnostic/investigate-boot

1007 lines
36 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
# investigate-boot - Systematic IncusOS boot failure investigation
#
# Runs repeated deploy/destroy cycles with observe-deploy-style screenshot
# capture, PLUS probes the IncusOS REST API after each deploy to capture
# state.txt contents. Records structured results for analysis.
#
# Supports different ISO sources to test the version mismatch hypothesis:
# --iso-source cdn Use latest CDN ISO (downloads if not on Proxmox)
# --iso-source old Use IncusOS_202602200553.iso (already on Proxmox)
# --iso-source FILE Use a specific ISO file path
#
# Usage:
# ./investigate-boot --runs 5 --iso-source old --label old-iso
# ./investigate-boot --runs 5 --iso-source cdn --label new-iso
# ./investigate-boot --runs 5 --iso-source /tmp/IncusOS-oc.iso --label oc-iso
# ./investigate-boot --dry-run
set -euo pipefail
readonly VERSION="0.1.0"
readonly SCRIPT_NAME="investigate-boot"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SEED_TOOL="${SCRIPT_DIR}/incusos-seed"
# ---------------------------------------------------------------------------
# Defaults
# ---------------------------------------------------------------------------
VMID=850
VM_NAME="investigate-boot"
RUNS=5
ISO_SOURCE="old" # "cdn", "old", or a file path
LABEL=""
DRY_RUN=false
SCREENSHOT_INTERVAL=2
INSTALL_TIMEOUT=600
BOOT_TIMEOUT=180
INSTALL_POLL=5
STABLE_THRESHOLD=3
APPLY_DEFAULTS=true # true for standalone, false for OC-destined nodes
# Proxmox connection (from proxmox.yaml)
PVE_HOST=""
PVE_NODE=""
PVE_STORAGE=""
PVE_ISO_STORAGE=""
PVE_BRIDGE=""
PVE_POOL=""
PVE_API_TOKEN_ID=""
# ISO tracking
ISO_FILENAME=""
ISO_VERSION=""
ISO_UPLOADED_BY_US=false
# Run output
RESULTS_DIR=""
RUN_DIR=""
LOG_FILE=""
FRAME_NUM=0
# State tracking
PHASE="init"
PREV_WR_BYTES=0
WRITES_STARTED=false
STABLE_COUNT=0
# Results accumulator
declare -a RUN_OUTCOMES=()
declare -a RUN_TIMES=()
declare -a RUN_SCRUB_SCHEDULES=()
declare -a RUN_CHECK_FREQUENCIES=()
declare -a RUN_OS_VERSIONS=()
declare -a RUN_DIRS=()
# ---------------------------------------------------------------------------
# Colors
# ---------------------------------------------------------------------------
setup_colors() {
if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then
RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[0;33m'
BLUE='\033[0;34m' CYAN='\033[0;36m' BOLD='\033[1m'
DIM='\033[2m' RESET='\033[0m'
else
RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' RESET=''
fi
}
setup_colors
step() { echo -e "${CYAN}[step]${RESET} ${BOLD}$*${RESET}"; }
success() { echo -e "${GREEN}[ok]${RESET} $*"; }
warn() { echo -e "${YELLOW}[warn]${RESET} $*" >&2; }
error() { echo -e "${RED}[error]${RESET} $*" >&2; }
info() { echo -e "${BLUE}[info]${RESET} $*"; }
detail() { echo -e "${DIM} $*${RESET}"; }
log() {
local ts
ts=$(date '+%H:%M:%S.%3N')
local msg="[${ts}] $*"
echo -e "${DIM}${msg}${RESET}"
[[ -n "${LOG_FILE:-}" ]] && echo "$msg" >> "$LOG_FILE"
}
log_quiet() {
local ts
ts=$(date '+%H:%M:%S.%3N')
[[ -n "${LOG_FILE:-}" ]] && echo "[${ts}] $*" >> "$LOG_FILE"
}
# ---------------------------------------------------------------------------
# Argument parsing
# ---------------------------------------------------------------------------
while [[ $# -gt 0 ]]; do
case "$1" in
--vmid) VMID="${2:?'--vmid requires a number'}"; shift 2 ;;
--runs) RUNS="${2:?'--runs requires a number'}"; shift 2 ;;
--iso-source) ISO_SOURCE="${2:?'--iso-source requires cdn|old|path'}"; shift 2 ;;
--label) LABEL="${2:?'--label requires a string'}"; shift 2 ;;
--no-defaults) APPLY_DEFAULTS=false; shift ;;
--interval) SCREENSHOT_INTERVAL="${2:?'--interval requires seconds'}"; shift 2 ;;
--dry-run) DRY_RUN=true; shift ;;
-h|--help)
cat <<EOF
${BOLD}${SCRIPT_NAME}${RESET} v${VERSION} - Systematic IncusOS boot failure investigation
${BOLD}USAGE${RESET}
${SCRIPT_NAME} [OPTIONS]
${BOLD}OPTIONS${RESET}
--runs N Number of deploy/destroy cycles (default: 5)
--iso-source SRC ISO source: "cdn" (latest), "old" (202602200553), or file path
--vmid N VMID to use (default: 850, must be 850-869)
--label NAME Label for results directory
--no-defaults Use apply_defaults: false in seed
--interval N Screenshot interval in seconds (default: 2)
--dry-run Preview without deploying
-h, --help Show this help
${BOLD}EXAMPLES${RESET}
${SCRIPT_NAME} --runs 5 --iso-source old --label baseline-old
${SCRIPT_NAME} --runs 5 --iso-source cdn --label baseline-new
${SCRIPT_NAME} --runs 5 --iso-source /tmp/IncusOS-oc.iso --label oc-iso
${BOLD}OUTPUT${RESET}
Results saved to: observe-runs/investigation_<timestamp>_<label>/
run-01/frame-*.png, deploy.log
run-02/...
results.txt (structured results matrix)
summary.txt (human-readable summary)
EOF
exit 0
;;
*) error "Unknown option: $1"; exit 1 ;;
esac
done
# ---------------------------------------------------------------------------
# Load environment
# ---------------------------------------------------------------------------
load_env() {
[[ -n "${PROXMOX_TOKEN_SECRET:-}" ]] && return
# Check envs/*/env in repo root
local repo_root
repo_root=$(cd "$SCRIPT_DIR/../.." && pwd)
for env_file in "${repo_root}"/envs/*/env; do
if [[ -f "$env_file" ]]; then
# shellcheck disable=SC1091
source "$env_file"
[[ -n "${PROXMOX_TOKEN_SECRET:-}" ]] && return
fi
done
# Walk up from script dir (backward compat)
local dir="$SCRIPT_DIR"
while [[ "$dir" != "/" ]]; do
if [[ -f "${dir}/env" ]]; then
# shellcheck disable=SC1091
source "${dir}/env"
return
fi
dir="$(dirname "$dir")"
done
}
load_env
if [[ -z "${PROXMOX_TOKEN_SECRET:-}" ]]; then
error "PROXMOX_TOKEN_SECRET not set. Create an env file or export it."
exit 1
fi
if [[ -z "${PROXMOX_ROOT_PASSWORD:-}" ]]; then
error "PROXMOX_ROOT_PASSWORD not set. Add it to the env file."
exit 1
fi
# ---------------------------------------------------------------------------
# Load proxmox.yaml
# ---------------------------------------------------------------------------
yaml_get() {
local file="$1" key="$2" default="${3:-}"
local val
val=$(awk -F': ' -v k="$key" '$1 == k {print $2; exit}' "$file" 2>/dev/null) || val=""
val="${val#\"}" ; val="${val%\"}"
val="${val#\'}" ; val="${val%\'}"
echo "${val:-$default}"
}
load_proxmox_yaml() {
local yaml_file=""
if [[ -f "${SCRIPT_DIR}/proxmox.yaml" ]]; then
yaml_file="${SCRIPT_DIR}/proxmox.yaml"
elif [[ -f "proxmox.yaml" ]]; then
yaml_file="proxmox.yaml"
else
error "proxmox.yaml not found"
exit 1
fi
PVE_HOST=$(yaml_get "$yaml_file" "host")
PVE_NODE=$(yaml_get "$yaml_file" "node" "pve")
PVE_STORAGE=$(yaml_get "$yaml_file" "storage" "local-lvm")
PVE_ISO_STORAGE=$(yaml_get "$yaml_file" "iso_storage" "local")
PVE_BRIDGE=$(yaml_get "$yaml_file" "bridge" "vmbr0")
PVE_POOL=$(yaml_get "$yaml_file" "pool")
PVE_API_TOKEN_ID=$(yaml_get "$yaml_file" "api_token_id")
if [[ -z "$PVE_HOST" ]]; then
error "proxmox.yaml: 'host' is required"
exit 1
fi
}
# ---------------------------------------------------------------------------
# Proxmox API helpers
# ---------------------------------------------------------------------------
pve_api() {
local method="$1" endpoint="$2" data="${3:-}" content_type="${4:-form}"
local url="https://${PVE_HOST}:8006/api2/json${endpoint}"
local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}"
local curl_args=(-fsSk -H "$auth")
[[ "$content_type" == "json" ]] && curl_args+=(-H "Content-Type: application/json")
case "$method" in
GET) curl "${curl_args[@]}" "$url" ;;
POST) curl "${curl_args[@]}" -X POST ${data:+-d "$data"} "$url" ;;
PUT) curl "${curl_args[@]}" -X PUT ${data:+-d "$data"} "$url" ;;
DELETE) curl "${curl_args[@]}" -X DELETE "$url" ;;
esac
}
pve_vm_exists() { pve_api GET "/nodes/${PVE_NODE}/qemu/$1/status/current" &>/dev/null; }
pve_vm_status() { pve_api GET "/nodes/${PVE_NODE}/qemu/$1/status/current" 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin)['data']['status'])" 2>/dev/null; }
pve_start_vm() { pve_api POST "/nodes/${PVE_NODE}/qemu/$1/status/start" &>/dev/null; }
pve_stop_vm() { pve_api POST "/nodes/${PVE_NODE}/qemu/$1/status/stop" &>/dev/null; }
pve_destroy_vm() {
local vmid="$1"
local resp
resp=$(pve_api DELETE "/nodes/${PVE_NODE}/qemu/${vmid}?purge=1&destroy-unreferenced-disks=1" 2>/dev/null) || return 1
local upid
upid=$(echo "$resp" | python3 -c "import json,sys; print(json.load(sys.stdin).get('data',''))" 2>/dev/null) || upid=""
if [[ -n "$upid" ]]; then
local enc
enc=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${upid}', safe=''))" 2>/dev/null)
local w=0
while [[ $w -lt 30 ]]; do
sleep 2; w=$((w + 2))
local st
st=$(pve_api GET "/nodes/${PVE_NODE}/tasks/${enc}/status" 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status',''))" 2>/dev/null) || st=""
[[ "$st" == "stopped" ]] && break
done
fi
}
pve_vm_disk_writes() {
pve_api GET "/nodes/${PVE_NODE}/qemu/$1/status/current" 2>/dev/null | python3 -c "
import json,sys
d = json.load(sys.stdin).get('data',{})
print(d.get('blockstat',{}).get('scsi0',{}).get('wr_bytes',0))
" 2>/dev/null || echo "0"
}
pve_vm_mac() {
pve_api GET "/nodes/${PVE_NODE}/qemu/$1/config" 2>/dev/null | python3 -c "
import json, sys, re
net0 = json.load(sys.stdin).get('data',{}).get('net0','')
m = re.search(r'([0-9A-Fa-f:]{17})', net0)
print(m.group(1).lower() if m else '')
" 2>/dev/null
}
pve_vm_ip() {
local vmid="$1"
local mac
mac=$(pve_vm_mac "$vmid") || return 1
[[ -z "$mac" ]] && return 1
ip neigh flush nud stale 2>/dev/null || true
local bridge_subnet
bridge_subnet=$(ip -4 addr show | awk '/inet / && !/127\.0\.0/ {print $2; exit}')
if [[ -n "$bridge_subnet" ]]; then
local broadcast
broadcast=$(python3 -c "import ipaddress; print(ipaddress.ip_interface('${bridge_subnet}').network.broadcast_address)" 2>/dev/null) || true
[[ -n "${broadcast:-}" ]] && { ping -c 2 -W 1 -b "$broadcast" &>/dev/null || true; sleep 1; }
fi
local found_ip
found_ip=$(ip neigh show | awk -v mac="$mac" 'tolower($0) ~ tolower(mac) && (/REACHABLE/ || /DELAY/ || /STALE/) {print $1; exit}') || true
if [[ -n "$found_ip" ]] && ping -c 1 -W 1 "$found_ip" &>/dev/null; then
echo "$found_ip"; return 0
fi
if [[ -n "$bridge_subnet" ]]; then
local hosts
hosts=$(python3 -c "
import ipaddress
for h in ipaddress.ip_interface('${bridge_subnet}').network.hosts():
print(h)
" 2>/dev/null)
if [[ -n "$hosts" ]]; then
while IFS= read -r hip; do ping -c 1 -W 1 "$hip" &>/dev/null & done <<< "$hosts"
wait; sleep 1
found_ip=$(ip neigh show | awk -v mac="$mac" 'tolower($0) ~ tolower(mac) && (/REACHABLE/ || /DELAY/) {print $1; exit}') || true
if [[ -n "$found_ip" ]] && ping -c 1 -W 1 "$found_ip" &>/dev/null; then
echo "$found_ip"; return 0
fi
fi
fi
return 1
}
# ---------------------------------------------------------------------------
# Screenshot capture
# ---------------------------------------------------------------------------
SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -o LogLevel=ERROR)
capture_screenshot() {
local vmid="$1" output_png="$2"
local rppm="/tmp/vm-${vmid}-screen.ppm" lppm="/tmp/vm-${vmid}-screen.ppm"
sshpass -p "$PROXMOX_ROOT_PASSWORD" ssh "${SSH_OPTS[@]}" "root@${PVE_HOST}" \
"echo 'screendump ${rppm}' | qm monitor ${vmid}" &>/dev/null || return 1
sshpass -p "$PROXMOX_ROOT_PASSWORD" scp "${SSH_OPTS[@]}" \
"root@${PVE_HOST}:${rppm}" "$lppm" &>/dev/null || return 1
sshpass -p "$PROXMOX_ROOT_PASSWORD" ssh "${SSH_OPTS[@]}" \
"root@${PVE_HOST}" "rm -f ${rppm}" &>/dev/null || true
python3 -c "from PIL import Image; Image.open('${lppm}').save('${output_png}')" 2>/dev/null || {
mv "$lppm" "${output_png%.png}.ppm" 2>/dev/null || true; return 0
}
rm -f "$lppm"; return 0
}
take_frame() {
FRAME_NUM=$((FRAME_NUM + 1))
local fname
fname=$(printf "frame-%03d.png" "$FRAME_NUM")
if capture_screenshot "$VMID" "${RUN_DIR}/${fname}"; then
log_quiet "Frame ${FRAME_NUM}: ${fname} (phase=${PHASE})"
else
log_quiet "Frame ${FRAME_NUM}: FAILED (phase=${PHASE})"
fi
}
# ---------------------------------------------------------------------------
# ISO management
# ---------------------------------------------------------------------------
resolve_iso() {
step "Resolving ISO source: ${ISO_SOURCE}"
case "$ISO_SOURCE" in
old)
ISO_FILENAME="IncusOS_202602200553.iso"
ISO_VERSION="202602200553"
# Check it exists on Proxmox
local iso_list
iso_list=$(pve_api GET "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content?content=iso" 2>/dev/null) || iso_list=""
if ! echo "$iso_list" | python3 -c "import json,sys; data=json.load(sys.stdin).get('data',[]); assert any('${ISO_FILENAME}' in d.get('volid','') for d in data)" 2>/dev/null; then
error "ISO ${ISO_FILENAME} not found on Proxmox. Upload it first."
exit 1
fi
success "Using existing ISO: ${ISO_FILENAME} (version ${ISO_VERSION})"
;;
cdn)
# Get latest version from CDN
local index_json
index_json=$(curl -fsSk "https://images.linuxcontainers.org/os/index.json" 2>/dev/null) || {
error "Failed to fetch CDN index"; exit 1
}
ISO_VERSION=$(echo "$index_json" | python3 -c "
import json, sys
data = json.load(sys.stdin)
versions = sorted(data.get('versions', {}).keys(), reverse=True)
print(versions[0] if versions else '')
" 2>/dev/null)
[[ -z "$ISO_VERSION" ]] && { error "Could not determine latest CDN version"; exit 1; }
ISO_FILENAME="IncusOS_${ISO_VERSION}.iso"
info "Latest CDN version: ${ISO_VERSION}"
# Check if already on Proxmox
local iso_list
iso_list=$(pve_api GET "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content?content=iso" 2>/dev/null) || iso_list=""
if echo "$iso_list" | python3 -c "import json,sys; data=json.load(sys.stdin).get('data',[]); assert any('${ISO_FILENAME}' in d.get('volid','') for d in data)" 2>/dev/null; then
success "ISO already on Proxmox: ${ISO_FILENAME}"
else
step "Downloading ISO from CDN"
local dl_url="https://images.linuxcontainers.org/os/${ISO_VERSION}/x86_64/IncusOS_${ISO_VERSION}.iso.gz"
local tmp_gz="/tmp/${ISO_FILENAME}.gz"
local tmp_iso="/tmp/${ISO_FILENAME}"
curl -fSk -o "$tmp_gz" "$dl_url" 2>&1 | tail -1 || {
error "Failed to download ${dl_url}"; exit 1
}
info "Decompressing..."
gunzip -f "$tmp_gz"
step "Uploading to Proxmox"
local url="https://${PVE_HOST}:8006/api2/json/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/upload"
local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}"
curl -fsSk -H "$auth" -F "content=iso" \
-F "filename=@${tmp_iso};filename=${ISO_FILENAME}" "$url" &>/dev/null || {
error "Failed to upload ISO"; exit 1
}
rm -f "$tmp_iso"
ISO_UPLOADED_BY_US=true
success "Uploaded: ${ISO_FILENAME}"
fi
;;
*)
# File path
if [[ ! -f "$ISO_SOURCE" ]]; then
error "ISO file not found: ${ISO_SOURCE}"; exit 1
fi
local basename
basename=$(basename "$ISO_SOURCE")
ISO_FILENAME="$basename"
ISO_VERSION=$(echo "$basename" | sed -n 's/.*_\([0-9]*\)\..*/\1/p')
[[ -z "$ISO_VERSION" ]] && ISO_VERSION="custom"
# Upload to Proxmox
step "Uploading custom ISO: ${basename}"
local url="https://${PVE_HOST}:8006/api2/json/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/upload"
local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}"
curl -fsSk -H "$auth" -F "content=iso" \
-F "filename=@${ISO_SOURCE};filename=${basename}" "$url" &>/dev/null || {
error "Failed to upload ISO"; exit 1
}
ISO_UPLOADED_BY_US=true
success "Using custom ISO: ${ISO_FILENAME} (version ${ISO_VERSION})"
;;
esac
}
# ---------------------------------------------------------------------------
# Seed generation
# ---------------------------------------------------------------------------
SEED_ISO=""
SEED_TMPDIR=""
generate_seed() {
step "Generating seed ISO"
SEED_TMPDIR=$(mktemp -d)
local seed_name="seed-${VM_NAME}.iso"
SEED_ISO="$seed_name"
local local_path="${SEED_TMPDIR}/${seed_name}"
local defaults_flag=""
if [[ "$APPLY_DEFAULTS" == true ]]; then
defaults_flag="--defaults"
else
defaults_flag="--no-defaults"
fi
"$SEED_TOOL" --format iso --app incus --output "$local_path" \
--force-install --force-reboot --hostname "$VM_NAME" \
$defaults_flag --quiet &>/dev/null
success "Seed ISO: ${seed_name} (apply_defaults=${APPLY_DEFAULTS})"
step "Uploading seed ISO"
local url="https://${PVE_HOST}:8006/api2/json/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/upload"
local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}"
curl -fsSk -H "$auth" -F "content=iso" \
-F "filename=@${local_path};filename=${seed_name}" "$url" &>/dev/null
success "Seed uploaded"
}
# ---------------------------------------------------------------------------
# VM lifecycle
# ---------------------------------------------------------------------------
create_vm() {
local json_payload
json_payload=$(python3 -c "
import json
payload = {
'vmid': ${VMID},
'name': '${VM_NAME}',
'description': '[incusos-lab:managed] investigate-boot',
'ostype': 'l26',
'bios': 'ovmf',
'machine': 'q35',
'efidisk0': '${PVE_STORAGE}:0,efitype=4m,pre-enrolled-keys=0',
'tpmstate0': '${PVE_STORAGE}:1,version=v2.0',
'cpu': 'host',
'cores': 4,
'memory': 4096,
'balloon': 0,
'scsihw': 'virtio-scsi-pci',
'scsi0': '${PVE_STORAGE}:50,iothread=1',
'ide2': '${PVE_ISO_STORAGE}:iso/${ISO_FILENAME},media=cdrom',
'ide3': '${PVE_ISO_STORAGE}:iso/${SEED_ISO},media=cdrom',
'net0': 'virtio,bridge=${PVE_BRIDGE}',
'boot': 'order=ide2;scsi0',
'agent': 1,
}
pool = '${PVE_POOL}'
if pool:
payload['pool'] = pool
print(json.dumps(payload))
")
local url="https://${PVE_HOST}:8006/api2/json/nodes/${PVE_NODE}/qemu"
local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}"
local resp
resp=$(curl -fsSk -H "$auth" -H "Content-Type: application/json" \
-X POST -d "$json_payload" "$url" 2>/dev/null) || { error "Failed to create VM"; return 1; }
# Wait for async task
local upid
upid=$(echo "$resp" | python3 -c "import json,sys; print(json.load(sys.stdin).get('data',''))" 2>/dev/null) || upid=""
if [[ -n "$upid" ]]; then
local enc
enc=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${upid}', safe=''))" 2>/dev/null)
local w=0
while [[ $w -lt 60 ]]; do
sleep 2; w=$((w + 2))
local st
st=$(pve_api GET "/nodes/${PVE_NODE}/tasks/${enc}/status" 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status',''))" 2>/dev/null) || st=""
[[ "$st" == "stopped" ]] && break
done
fi
}
cleanup_vm() {
local status
status=$(pve_vm_status "$VMID" 2>/dev/null) || status="unknown"
[[ "$status" == "running" ]] && { pve_stop_vm "$VMID" || true; sleep 3; }
pve_destroy_vm "$VMID" || true
pve_api DELETE "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content/${PVE_ISO_STORAGE}:iso/${SEED_ISO}" &>/dev/null || true
if command -v incus &>/dev/null; then
local cur
cur=$(incus remote get-default 2>/dev/null) || true
[[ "$cur" == "$VM_NAME" ]] && { incus remote switch local 2>/dev/null || true; }
incus remote remove "$VM_NAME" 2>/dev/null || true
fi
rm -rf "${SEED_TMPDIR:-}" 2>/dev/null || true
rm -f "/tmp/vm-${VMID}-screen.ppm" 2>/dev/null || true
}
# ---------------------------------------------------------------------------
# IncusOS API probing
# ---------------------------------------------------------------------------
probe_incusos_api() {
# Probe the IncusOS REST API via the Incus /os/ proxy.
# Requires that the VM has been added as an incus remote.
local ip="$1"
local probe_file="${RUN_DIR}/api-probe.json"
local cert_file="${HOME}/.config/incus/client.crt"
local key_file="${HOME}/.config/incus/client.key"
if [[ ! -f "$cert_file" ]] || [[ ! -f "$key_file" ]]; then
log "API probe: no client cert available"
echo '{"error": "no client cert"}' > "$probe_file"
return
fi
log "Probing IncusOS API at ${ip}..."
# Add as temporary incus remote
incus remote add "$VM_NAME" "https://${ip}:8443" --accept-certificate 2>/dev/null || true
# Probe storage config (scrub_schedule)
local storage_json
storage_json=$(incus query "${VM_NAME}:/os/1.0/system/storage" 2>/dev/null) || storage_json='{"error":"query failed"}'
# Probe update config (check_frequency)
local update_json
update_json=$(incus query "${VM_NAME}:/os/1.0/system/update" 2>/dev/null) || update_json='{"error":"query failed"}'
# Probe system info (os_version, hostname, uptime)
local system_json
system_json=$(incus query "${VM_NAME}:/os/1.0" 2>/dev/null) || system_json='{"error":"query failed"}'
# Write combined probe results
python3 -c "
import json, sys
storage = json.loads('''${storage_json}''')
update = json.loads('''${update_json}''')
system = json.loads('''${system_json}''')
result = {
'storage': storage,
'update': update,
'system': system,
'scrub_schedule': storage.get('config', {}).get('scrub_schedule', ''),
'check_frequency': update.get('config', {}).get('check_frequency', ''),
'os_version': system.get('environment', {}).get('os_version', ''),
'hostname': system.get('environment', {}).get('hostname', ''),
'uptime': system.get('environment', {}).get('uptime', 0),
}
print(json.dumps(result, indent=2))
" > "$probe_file" 2>/dev/null || echo '{"error":"probe parse failed"}' > "$probe_file"
# Extract key values for summary
local scrub check_freq os_ver
scrub=$(python3 -c "import json; d=json.load(open('${probe_file}')); print(d.get('scrub_schedule',''))" 2>/dev/null) || scrub=""
check_freq=$(python3 -c "import json; d=json.load(open('${probe_file}')); print(d.get('check_frequency',''))" 2>/dev/null) || check_freq=""
os_ver=$(python3 -c "import json; d=json.load(open('${probe_file}')); print(d.get('os_version',''))" 2>/dev/null) || os_ver=""
log "API probe results:"
log " scrub_schedule: '${scrub}'"
log " check_frequency: '${check_freq}'"
log " os_version: '${os_ver}'"
# Store for summary
RUN_SCRUB_SCHEDULES+=("$scrub")
RUN_CHECK_FREQUENCIES+=("$check_freq")
RUN_OS_VERSIONS+=("$os_ver")
}
# ---------------------------------------------------------------------------
# Install monitoring
# ---------------------------------------------------------------------------
check_install_progress() {
local wr_bytes
wr_bytes=$(pve_vm_disk_writes "$VMID" 2>/dev/null) || wr_bytes=0
if [[ $wr_bytes -gt $PREV_WR_BYTES ]]; then
if [[ "$WRITES_STARTED" != true ]]; then
WRITES_STARTED=true
PHASE="installing"
log "Install started — disk writes detected"
fi
STABLE_COUNT=0
elif [[ "$WRITES_STARTED" == true ]]; then
STABLE_COUNT=$((STABLE_COUNT + 1))
if [[ $STABLE_COUNT -ge $STABLE_THRESHOLD ]]; then
local wr_mb=$(( wr_bytes / 1048576 ))
log "Install complete — ${wr_mb} MiB written, ${STABLE_COUNT} stable polls"
PHASE="install_done"
fi
fi
PREV_WR_BYTES=$wr_bytes
}
# ---------------------------------------------------------------------------
# Single deploy cycle
# ---------------------------------------------------------------------------
run_single_deploy() {
local run_num="$1"
local deploy_start
deploy_start=$(date +%s)
# Reset state
PHASE="init"
PREV_WR_BYTES=0
WRITES_STARTED=false
STABLE_COUNT=0
FRAME_NUM=0
# Setup run directory
RUN_DIR="${RESULTS_DIR}/run-$(printf '%02d' "$run_num")"
mkdir -p "$RUN_DIR"
LOG_FILE="${RUN_DIR}/deploy.log"
touch "$LOG_FILE"
log "=== Run ${run_num}/${RUNS} ==="
log "ISO: ${ISO_FILENAME} (version ${ISO_VERSION})"
log "apply_defaults: ${APPLY_DEFAULTS}"
# Generate seed (fresh each time)
generate_seed
# Create VM
step "Run ${run_num}: Creating VM (VMID ${VMID})"
create_vm || { error "Failed to create VM"; return 1; }
success "VM created"
# --- Phase 1: Boot from ISO ---
step "Run ${run_num}: Starting VM (ISO boot)"
pve_start_vm "$VMID"
PHASE="iso_boot"
log "VM started — phase: iso_boot"
local elapsed=0
local last_bs_check=0
while [[ "$PHASE" == "iso_boot" ]] || [[ "$PHASE" == "installing" ]]; do
take_frame || true
local now; now=$(date +%s)
if [[ $((now - last_bs_check)) -ge $INSTALL_POLL ]]; then
check_install_progress
last_bs_check=$now
fi
sleep "$SCREENSHOT_INTERVAL"
elapsed=$((elapsed + SCREENSHOT_INTERVAL))
local status; status=$(pve_vm_status "$VMID" 2>/dev/null) || status="unknown"
[[ "$status" == "stopped" ]] && { log "VM stopped during install"; PHASE="install_done"; }
if [[ $elapsed -ge $INSTALL_TIMEOUT ]]; then
warn "Install timeout after ${INSTALL_TIMEOUT}s"
take_frame || true
PHASE="failed"
break
fi
if [[ $((elapsed % 30)) -lt $SCREENSHOT_INTERVAL ]]; then
local wr_mb=$(( PREV_WR_BYTES / 1048576 ))
log "Progress: ${elapsed}s, ${wr_mb} MiB, phase=${PHASE}"
fi
done
[[ "$PHASE" == "failed" ]] && return 1
# --- Phase 2: Transition ---
PHASE="transitioning"
take_frame || true
step "Run ${run_num}: Stopping VM, removing media"
pve_stop_vm "$VMID" || true
local sw=0
while [[ $sw -lt 30 ]]; do
local st; st=$(pve_vm_status "$VMID" 2>/dev/null) || st="unknown"
[[ "$st" == "stopped" ]] && break
sleep 2; sw=$((sw + 2))
done
pve_api PUT "/nodes/${PVE_NODE}/qemu/${VMID}/config" \
'{"delete":"ide2,ide3","boot":"order=scsi0"}' json &>/dev/null || {
error "Failed to remove install media"; PHASE="failed"; return 1
}
log "Media removed, boot=scsi0"
# --- Phase 3: Boot from disk ---
step "Run ${run_num}: Starting from disk"
pve_start_vm "$VMID"
PHASE="disk_boot"
log "Disk boot started"
local boot_elapsed=0
local ip_detected=""
local ip_started=false
while [[ "$PHASE" == "disk_boot" ]]; do
take_frame || true
sleep "$SCREENSHOT_INTERVAL"
boot_elapsed=$((boot_elapsed + SCREENSHOT_INTERVAL))
if [[ $boot_elapsed -ge 30 ]] && [[ "$ip_started" != true ]]; then
ip_started=true
log "Starting IP detection"
fi
if [[ "$ip_started" == true ]] && [[ $((boot_elapsed % 10)) -lt $SCREENSHOT_INTERVAL ]]; then
local ip
ip=$(pve_vm_ip "$VMID" 2>/dev/null) || ip=""
if [[ -n "$ip" ]]; then
[[ -z "$ip_detected" ]] && { ip_detected="$ip"; log "IP: ${ip}"; }
if curl -sk --connect-timeout 3 "https://${ip}:8443" &>/dev/null; then
log "Port 8443 UP at ${ip} (${boot_elapsed}s)"
PHASE="ready"
break
fi
fi
fi
if [[ $boot_elapsed -ge $BOOT_TIMEOUT ]]; then
warn "Boot timeout after ${BOOT_TIMEOUT}s"
take_frame || true
PHASE="failed"
break
fi
if [[ $((boot_elapsed % 15)) -lt $SCREENSHOT_INTERVAL ]]; then
log "Disk boot: ${boot_elapsed}s, ip=${ip_detected:-none}, phase=${PHASE}"
fi
done
# Take final frames
if [[ "$PHASE" == "ready" ]]; then
for _ in 1 2 3; do sleep "$SCREENSHOT_INTERVAL"; take_frame || true; done
fi
local deploy_end; deploy_end=$(date +%s)
local total=$((deploy_end - deploy_start))
log "Result: phase=${PHASE}, total=${total}s, frames=${FRAME_NUM}"
# --- Phase 4: API probing ---
if [[ "$PHASE" == "ready" ]] && [[ -n "$ip_detected" ]]; then
probe_incusos_api "$ip_detected"
else
RUN_SCRUB_SCHEDULES+=("N/A")
RUN_CHECK_FREQUENCIES+=("N/A")
RUN_OS_VERSIONS+=("N/A")
fi
RUN_TIMES+=("$total")
RUN_DIRS+=("$RUN_DIR")
if [[ "$PHASE" == "ready" ]]; then
RUN_OUTCOMES+=("PASS")
return 0
else
RUN_OUTCOMES+=("FAIL")
return 1
fi
}
# ---------------------------------------------------------------------------
# Preflight
# ---------------------------------------------------------------------------
preflight() {
step "Preflight checks"
local missing=()
command -v sshpass &>/dev/null || missing+=("sshpass")
command -v python3 &>/dev/null || missing+=("python3")
command -v curl &>/dev/null || missing+=("curl")
[[ ${#missing[@]} -gt 0 ]] && { error "Missing: ${missing[*]}"; exit 1; }
python3 -c "from PIL import Image" &>/dev/null || warn "PIL not available — PPM screenshots"
if ! sshpass -p "$PROXMOX_ROOT_PASSWORD" ssh "${SSH_OPTS[@]}" "root@${PVE_HOST}" "echo ok" &>/dev/null; then
error "Cannot SSH to root@${PVE_HOST}"; exit 1
fi
success "SSH OK"
if [[ $VMID -lt 850 ]] || [[ $VMID -gt 869 ]]; then
error "VMID ${VMID} outside test range (850-869)"; exit 1
fi
success "VMID ${VMID} in range"
}
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
echo -e "${BOLD}${SCRIPT_NAME}${RESET} v${VERSION}"
echo "Systematic IncusOS boot failure investigation"
echo ""
if [[ -z "$LABEL" ]]; then
case "$ISO_SOURCE" in
cdn) LABEL="cdn-latest" ;;
old) LABEL="old-202602200553" ;;
*) LABEL="custom-iso" ;;
esac
fi
if [[ "$DRY_RUN" == true ]]; then
echo -e "${YELLOW}[dry-run] Preview mode${RESET}"
echo ""
echo "Configuration:"
echo " Runs: ${RUNS}"
echo " ISO source: ${ISO_SOURCE}"
echo " VMID: ${VMID}"
echo " Label: ${LABEL}"
echo " apply_defaults: ${APPLY_DEFAULTS}"
echo " Screenshots: every ${SCREENSHOT_INTERVAL}s"
echo ""
load_proxmox_yaml
detail "Host: ${PVE_HOST}, Node: ${PVE_NODE}"
echo ""
exit 0
fi
# Setup
step "Loading configuration"
load_proxmox_yaml
detail "Host: ${PVE_HOST}, Node: ${PVE_NODE}, Storage: ${PVE_STORAGE}"
preflight
resolve_iso
# Setup results directory
RESULTS_DIR="${SCRIPT_DIR}/observe-runs/investigation_$(date '+%Y-%m-%d_%H-%M-%S')_${LABEL}"
mkdir -p "$RESULTS_DIR"
success "Results: ${RESULTS_DIR}/"
echo ""
echo -e "${BOLD}Investigation plan:${RESET}"
echo " Runs: ${RUNS}"
echo " ISO: ${ISO_FILENAME} (version ${ISO_VERSION})"
echo " apply_defaults: ${APPLY_DEFAULTS}"
echo " VMID: ${VMID}"
echo ""
# Trap for cleanup
trap_cleanup() {
echo ""
warn "Interrupted — cleaning up"
cleanup_vm 2>/dev/null || true
exit 130
}
trap trap_cleanup INT TERM
# Run the investigation cycles
for ((run=1; run<=RUNS; run++)); do
echo ""
echo -e "${BOLD}════════════════════════════════════════${RESET}"
echo -e "${BOLD} Run ${run}/${RUNS}${RESET}"
echo -e "${BOLD}════════════════════════════════════════${RESET}"
echo ""
# Ensure VMID is free
if pve_vm_exists "$VMID"; then
warn "VMID ${VMID} exists — cleaning up from previous run"
cleanup_vm
sleep 5
fi
run_result=0
run_single_deploy "$run" || run_result=$?
echo ""
if [[ $run_result -eq 0 ]]; then
echo -e " Result: ${GREEN}PASS${RESET}"
else
echo -e " Result: ${RED}FAIL${RESET} (phase: ${PHASE})"
fi
# Cleanup
step "Cleanup run ${run}"
cleanup_vm
sleep 3 # Brief pause between runs
done
# ---------------------------------------------------------------------------
# Results summary
# ---------------------------------------------------------------------------
echo ""
echo ""
echo -e "${BOLD}╔══════════════════════════════════════════╗${RESET}"
echo -e "${BOLD}║ Investigation Results Summary ║${RESET}"
echo -e "${BOLD}╚══════════════════════════════════════════╝${RESET}"
echo ""
echo "ISO: ${ISO_FILENAME} (version ${ISO_VERSION})"
echo "ISO source: ${ISO_SOURCE}"
echo "apply_defaults: ${APPLY_DEFAULTS}"
echo "Runs: ${RUNS}"
echo ""
pass_count=0
fail_count=0
for ((i=0; i<${#RUN_OUTCOMES[@]}; i++)); do
local_run=$((i + 1))
outcome="${RUN_OUTCOMES[$i]}"
time_s="${RUN_TIMES[$i]:-?}"
scrub="${RUN_SCRUB_SCHEDULES[$i]:-?}"
freq="${RUN_CHECK_FREQUENCIES[$i]:-?}"
osver="${RUN_OS_VERSIONS[$i]:-?}"
marker=""
case "$outcome" in
PASS) marker="${GREEN}PASS${RESET}"; pass_count=$((pass_count + 1)) ;;
FAIL) marker="${RED}FAIL${RESET}"; fail_count=$((fail_count + 1)) ;;
esac
echo -e " Run $(printf '%2d' $local_run): ${marker} time=${time_s}s scrub='${scrub}' freq='${freq}' os=${osver}"
done
echo ""
echo -e " Total: ${GREEN}${pass_count} PASS${RESET}, ${RED}${fail_count} FAIL${RESET} (of ${RUNS})"
if [[ $fail_count -gt 0 ]]; then
pct=$(( fail_count * 100 / RUNS ))
echo -e " Failure rate: ${RED}${pct}%${RESET}"
fi
echo ""
# Write structured results file
results_file="${RESULTS_DIR}/results.txt"
{
echo "# investigate-boot results"
echo "# Generated: $(date -Iseconds)"
echo "# ISO: ${ISO_FILENAME} (version ${ISO_VERSION})"
echo "# ISO source: ${ISO_SOURCE}"
echo "# apply_defaults: ${APPLY_DEFAULTS}"
echo "# Runs: ${RUNS}"
echo "#"
echo "# run|outcome|time_s|scrub_schedule|check_frequency|os_version|run_dir"
for ((i=0; i<${#RUN_OUTCOMES[@]}; i++)); do
echo "$((i+1))|${RUN_OUTCOMES[$i]}|${RUN_TIMES[$i]:-?}|${RUN_SCRUB_SCHEDULES[$i]:-?}|${RUN_CHECK_FREQUENCIES[$i]:-?}|${RUN_OS_VERSIONS[$i]:-?}|${RUN_DIRS[$i]:-?}"
done
} > "$results_file"
success "Results written to: ${results_file}"
echo ""
exit $fail_count
Reference in New Issue View Git Blame Copy Permalink