#!/usr/bin/env bash
# investigate-boot - Systematic IncusOS boot failure investigation
#
# Runs repeated deploy/destroy cycles with observe-deploy-style screenshot
# capture, PLUS probes the IncusOS REST API after each deploy to capture
# state.txt contents. Records structured results for analysis.
#
# Supports different ISO sources to test the version mismatch hypothesis:
#   --iso-source cdn       Use latest CDN ISO (downloads if not on Proxmox)
#   --iso-source old       Use IncusOS_202602200553.iso (already on Proxmox)
#   --iso-source FILE      Use a specific ISO file path
#
# Usage:
#   ./investigate-boot --runs 5 --iso-source old --label old-iso
#   ./investigate-boot --runs 5 --iso-source cdn --label new-iso
#   ./investigate-boot --runs 5 --iso-source /tmp/IncusOS-oc.iso --label oc-iso
#   ./investigate-boot --dry-run

set -euo pipefail

readonly VERSION="0.1.0"
readonly SCRIPT_NAME="investigate-boot"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SEED_TOOL="${SCRIPT_DIR}/incusos-seed"

# ---------------------------------------------------------------------------
# Defaults
# ---------------------------------------------------------------------------

VMID=850
VM_NAME="investigate-boot"
RUNS=5
ISO_SOURCE="old"   # "cdn", "old", or a file path
LABEL=""
DRY_RUN=false
SCREENSHOT_INTERVAL=2
INSTALL_TIMEOUT=600
BOOT_TIMEOUT=180
INSTALL_POLL=5
STABLE_THRESHOLD=3
APPLY_DEFAULTS=true  # true for standalone, false for OC-destined nodes

# Proxmox connection (from proxmox.yaml)
PVE_HOST=""
PVE_NODE=""
PVE_STORAGE=""
PVE_ISO_STORAGE=""
PVE_BRIDGE=""
PVE_POOL=""
PVE_API_TOKEN_ID=""

# ISO tracking
ISO_FILENAME=""
ISO_VERSION=""
ISO_UPLOADED_BY_US=false

# Run output
RESULTS_DIR=""
RUN_DIR=""
LOG_FILE=""
FRAME_NUM=0

# State tracking
PHASE="init"
PREV_WR_BYTES=0
WRITES_STARTED=false
STABLE_COUNT=0

# Results accumulator
declare -a RUN_OUTCOMES=()
declare -a RUN_TIMES=()
declare -a RUN_SCRUB_SCHEDULES=()
declare -a RUN_CHECK_FREQUENCIES=()
declare -a RUN_OS_VERSIONS=()
declare -a RUN_DIRS=()

# ---------------------------------------------------------------------------
# Colors
# ---------------------------------------------------------------------------

setup_colors() {
    if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then
        RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[0;33m'
        BLUE='\033[0;34m' CYAN='\033[0;36m' BOLD='\033[1m'
        DIM='\033[2m' RESET='\033[0m'
    else
        RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' RESET=''
    fi
}
setup_colors

step()    { echo -e "${CYAN}[step]${RESET}    ${BOLD}$*${RESET}"; }
success() { echo -e "${GREEN}[ok]${RESET}      $*"; }
warn()    { echo -e "${YELLOW}[warn]${RESET}    $*" >&2; }
error()   { echo -e "${RED}[error]${RESET}   $*" >&2; }
info()    { echo -e "${BLUE}[info]${RESET}    $*"; }
detail()  { echo -e "${DIM}          $*${RESET}"; }

log() {
    local ts
    ts=$(date '+%H:%M:%S.%3N')
    local msg="[${ts}] $*"
    echo -e "${DIM}${msg}${RESET}"
    [[ -n "${LOG_FILE:-}" ]] && echo "$msg" >> "$LOG_FILE"
}

log_quiet() {
    local ts
    ts=$(date '+%H:%M:%S.%3N')
    [[ -n "${LOG_FILE:-}" ]] && echo "[${ts}] $*" >> "$LOG_FILE"
}

# ---------------------------------------------------------------------------
# Argument parsing
# ---------------------------------------------------------------------------

while [[ $# -gt 0 ]]; do
    case "$1" in
        --vmid)        VMID="${2:?'--vmid requires a number'}"; shift 2 ;;
        --runs)        RUNS="${2:?'--runs requires a number'}"; shift 2 ;;
        --iso-source)  ISO_SOURCE="${2:?'--iso-source requires cdn|old|path'}"; shift 2 ;;
        --label)       LABEL="${2:?'--label requires a string'}"; shift 2 ;;
        --no-defaults) APPLY_DEFAULTS=false; shift ;;
        --interval)    SCREENSHOT_INTERVAL="${2:?'--interval requires seconds'}"; shift 2 ;;
        --dry-run)     DRY_RUN=true; shift ;;
        -h|--help)
            cat <<EOF
${BOLD}${SCRIPT_NAME}${RESET} v${VERSION} - Systematic IncusOS boot failure investigation

${BOLD}USAGE${RESET}
    ${SCRIPT_NAME} [OPTIONS]

${BOLD}OPTIONS${RESET}
    --runs N            Number of deploy/destroy cycles (default: 5)
    --iso-source SRC    ISO source: "cdn" (latest), "old" (202602200553), or file path
    --vmid N            VMID to use (default: 850, must be 850-869)
    --label NAME        Label for results directory
    --no-defaults       Use apply_defaults: false in seed
    --interval N        Screenshot interval in seconds (default: 2)
    --dry-run           Preview without deploying
    -h, --help          Show this help

${BOLD}EXAMPLES${RESET}
    ${SCRIPT_NAME} --runs 5 --iso-source old --label baseline-old
    ${SCRIPT_NAME} --runs 5 --iso-source cdn --label baseline-new
    ${SCRIPT_NAME} --runs 5 --iso-source /tmp/IncusOS-oc.iso --label oc-iso

${BOLD}OUTPUT${RESET}
    Results saved to: observe-runs/investigation_<timestamp>_<label>/
      run-01/frame-*.png, deploy.log
      run-02/...
      results.txt    (structured results matrix)
      summary.txt    (human-readable summary)
EOF
            exit 0
            ;;
        *) error "Unknown option: $1"; exit 1 ;;
    esac
done

# ---------------------------------------------------------------------------
# Load environment
# ---------------------------------------------------------------------------

load_env() {
    local dir="$SCRIPT_DIR"
    while [[ "$dir" != "/" ]]; do
        if [[ -f "${dir}/env" ]]; then
            # shellcheck disable=SC1091
            source "${dir}/env"
            return
        fi
        dir="$(dirname "$dir")"
    done
}
load_env

if [[ -z "${PROXMOX_TOKEN_SECRET:-}" ]]; then
    error "PROXMOX_TOKEN_SECRET not set. Create an env file or export it."
    exit 1
fi

if [[ -z "${PROXMOX_ROOT_PASSWORD:-}" ]]; then
    error "PROXMOX_ROOT_PASSWORD not set. Add it to the env file."
    exit 1
fi

# ---------------------------------------------------------------------------
# Load proxmox.yaml
# ---------------------------------------------------------------------------

yaml_get() {
    local file="$1" key="$2" default="${3:-}"
    local val
    val=$(awk -F': ' -v k="$key" '$1 == k {print $2; exit}' "$file" 2>/dev/null) || val=""
    val="${val#\"}" ; val="${val%\"}"
    val="${val#\'}" ; val="${val%\'}"
    echo "${val:-$default}"
}

load_proxmox_yaml() {
    local yaml_file=""
    if [[ -f "${SCRIPT_DIR}/proxmox.yaml" ]]; then
        yaml_file="${SCRIPT_DIR}/proxmox.yaml"
    elif [[ -f "proxmox.yaml" ]]; then
        yaml_file="proxmox.yaml"
    else
        error "proxmox.yaml not found"
        exit 1
    fi

    PVE_HOST=$(yaml_get "$yaml_file" "host")
    PVE_NODE=$(yaml_get "$yaml_file" "node" "pve")
    PVE_STORAGE=$(yaml_get "$yaml_file" "storage" "local-lvm")
    PVE_ISO_STORAGE=$(yaml_get "$yaml_file" "iso_storage" "local")
    PVE_BRIDGE=$(yaml_get "$yaml_file" "bridge" "vmbr0")
    PVE_POOL=$(yaml_get "$yaml_file" "pool")
    PVE_API_TOKEN_ID=$(yaml_get "$yaml_file" "api_token_id")

    if [[ -z "$PVE_HOST" ]]; then
        error "proxmox.yaml: 'host' is required"
        exit 1
    fi
}

# ---------------------------------------------------------------------------
# Proxmox API helpers
# ---------------------------------------------------------------------------

pve_api() {
    local method="$1" endpoint="$2" data="${3:-}" content_type="${4:-form}"
    local url="https://${PVE_HOST}:8006/api2/json${endpoint}"
    local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}"
    local curl_args=(-fsSk -H "$auth")
    [[ "$content_type" == "json" ]] && curl_args+=(-H "Content-Type: application/json")
    case "$method" in
        GET)    curl "${curl_args[@]}" "$url" ;;
        POST)   curl "${curl_args[@]}" -X POST ${data:+-d "$data"} "$url" ;;
        PUT)    curl "${curl_args[@]}" -X PUT ${data:+-d "$data"} "$url" ;;
        DELETE) curl "${curl_args[@]}" -X DELETE "$url" ;;
    esac
}

pve_vm_exists() { pve_api GET "/nodes/${PVE_NODE}/qemu/$1/status/current" &>/dev/null; }
pve_vm_status() { pve_api GET "/nodes/${PVE_NODE}/qemu/$1/status/current" 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin)['data']['status'])" 2>/dev/null; }
pve_start_vm()  { pve_api POST "/nodes/${PVE_NODE}/qemu/$1/status/start" &>/dev/null; }
pve_stop_vm()   { pve_api POST "/nodes/${PVE_NODE}/qemu/$1/status/stop" &>/dev/null; }

pve_destroy_vm() {
    local vmid="$1"
    local resp
    resp=$(pve_api DELETE "/nodes/${PVE_NODE}/qemu/${vmid}?purge=1&destroy-unreferenced-disks=1" 2>/dev/null) || return 1
    local upid
    upid=$(echo "$resp" | python3 -c "import json,sys; print(json.load(sys.stdin).get('data',''))" 2>/dev/null) || upid=""
    if [[ -n "$upid" ]]; then
        local enc
        enc=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${upid}', safe=''))" 2>/dev/null)
        local w=0
        while [[ $w -lt 30 ]]; do
            sleep 2; w=$((w + 2))
            local st
            st=$(pve_api GET "/nodes/${PVE_NODE}/tasks/${enc}/status" 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status',''))" 2>/dev/null) || st=""
            [[ "$st" == "stopped" ]] && break
        done
    fi
}

pve_vm_disk_writes() {
    pve_api GET "/nodes/${PVE_NODE}/qemu/$1/status/current" 2>/dev/null | python3 -c "
import json,sys
d = json.load(sys.stdin).get('data',{})
print(d.get('blockstat',{}).get('scsi0',{}).get('wr_bytes',0))
" 2>/dev/null || echo "0"
}

pve_vm_mac() {
    pve_api GET "/nodes/${PVE_NODE}/qemu/$1/config" 2>/dev/null | python3 -c "
import json, sys, re
net0 = json.load(sys.stdin).get('data',{}).get('net0','')
m = re.search(r'([0-9A-Fa-f:]{17})', net0)
print(m.group(1).lower() if m else '')
" 2>/dev/null
}

pve_vm_ip() {
    local vmid="$1"
    local mac
    mac=$(pve_vm_mac "$vmid") || return 1
    [[ -z "$mac" ]] && return 1
    ip neigh flush nud stale 2>/dev/null || true
    local bridge_subnet
    bridge_subnet=$(ip -4 addr show | awk '/inet / && !/127\.0\.0/ {print $2; exit}')
    if [[ -n "$bridge_subnet" ]]; then
        local broadcast
        broadcast=$(python3 -c "import ipaddress; print(ipaddress.ip_interface('${bridge_subnet}').network.broadcast_address)" 2>/dev/null) || true
        [[ -n "${broadcast:-}" ]] && { ping -c 2 -W 1 -b "$broadcast" &>/dev/null || true; sleep 1; }
    fi
    local found_ip
    found_ip=$(ip neigh show | awk -v mac="$mac" 'tolower($0) ~ tolower(mac) && (/REACHABLE/ || /DELAY/ || /STALE/) {print $1; exit}') || true
    if [[ -n "$found_ip" ]] && ping -c 1 -W 1 "$found_ip" &>/dev/null; then
        echo "$found_ip"; return 0
    fi
    if [[ -n "$bridge_subnet" ]]; then
        local hosts
        hosts=$(python3 -c "
import ipaddress
for h in ipaddress.ip_interface('${bridge_subnet}').network.hosts():
    print(h)
" 2>/dev/null)
        if [[ -n "$hosts" ]]; then
            while IFS= read -r hip; do ping -c 1 -W 1 "$hip" &>/dev/null & done <<< "$hosts"
            wait; sleep 1
            found_ip=$(ip neigh show | awk -v mac="$mac" 'tolower($0) ~ tolower(mac) && (/REACHABLE/ || /DELAY/) {print $1; exit}') || true
            if [[ -n "$found_ip" ]] && ping -c 1 -W 1 "$found_ip" &>/dev/null; then
                echo "$found_ip"; return 0
            fi
        fi
    fi
    return 1
}

# ---------------------------------------------------------------------------
# Screenshot capture
# ---------------------------------------------------------------------------

SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -o LogLevel=ERROR)

capture_screenshot() {
    local vmid="$1" output_png="$2"
    local rppm="/tmp/vm-${vmid}-screen.ppm" lppm="/tmp/vm-${vmid}-screen.ppm"
    sshpass -p "$PROXMOX_ROOT_PASSWORD" ssh "${SSH_OPTS[@]}" "root@${PVE_HOST}" \
        "echo 'screendump ${rppm}' | qm monitor ${vmid}" &>/dev/null || return 1
    sshpass -p "$PROXMOX_ROOT_PASSWORD" scp "${SSH_OPTS[@]}" \
        "root@${PVE_HOST}:${rppm}" "$lppm" &>/dev/null || return 1
    sshpass -p "$PROXMOX_ROOT_PASSWORD" ssh "${SSH_OPTS[@]}" \
        "root@${PVE_HOST}" "rm -f ${rppm}" &>/dev/null || true
    python3 -c "from PIL import Image; Image.open('${lppm}').save('${output_png}')" 2>/dev/null || {
        mv "$lppm" "${output_png%.png}.ppm" 2>/dev/null || true; return 0
    }
    rm -f "$lppm"; return 0
}

take_frame() {
    FRAME_NUM=$((FRAME_NUM + 1))
    local fname
    fname=$(printf "frame-%03d.png" "$FRAME_NUM")
    if capture_screenshot "$VMID" "${RUN_DIR}/${fname}"; then
        log_quiet "Frame ${FRAME_NUM}: ${fname} (phase=${PHASE})"
    else
        log_quiet "Frame ${FRAME_NUM}: FAILED (phase=${PHASE})"
    fi
}

# ---------------------------------------------------------------------------
# ISO management
# ---------------------------------------------------------------------------

resolve_iso() {
    step "Resolving ISO source: ${ISO_SOURCE}"

    case "$ISO_SOURCE" in
        old)
            ISO_FILENAME="IncusOS_202602200553.iso"
            ISO_VERSION="202602200553"
            # Check it exists on Proxmox
            local iso_list
            iso_list=$(pve_api GET "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content?content=iso" 2>/dev/null) || iso_list=""
            if ! echo "$iso_list" | python3 -c "import json,sys; data=json.load(sys.stdin).get('data',[]); assert any('${ISO_FILENAME}' in d.get('volid','') for d in data)" 2>/dev/null; then
                error "ISO ${ISO_FILENAME} not found on Proxmox. Upload it first."
                exit 1
            fi
            success "Using existing ISO: ${ISO_FILENAME} (version ${ISO_VERSION})"
            ;;
        cdn)
            # Get latest version from CDN
            local index_json
            index_json=$(curl -fsSk "https://images.linuxcontainers.org/os/index.json" 2>/dev/null) || {
                error "Failed to fetch CDN index"; exit 1
            }
            ISO_VERSION=$(echo "$index_json" | python3 -c "
import json, sys
data = json.load(sys.stdin)
versions = sorted(data.get('versions', {}).keys(), reverse=True)
print(versions[0] if versions else '')
" 2>/dev/null)
            [[ -z "$ISO_VERSION" ]] && { error "Could not determine latest CDN version"; exit 1; }
            ISO_FILENAME="IncusOS_${ISO_VERSION}.iso"
            info "Latest CDN version: ${ISO_VERSION}"

            # Check if already on Proxmox
            local iso_list
            iso_list=$(pve_api GET "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content?content=iso" 2>/dev/null) || iso_list=""
            if echo "$iso_list" | python3 -c "import json,sys; data=json.load(sys.stdin).get('data',[]); assert any('${ISO_FILENAME}' in d.get('volid','') for d in data)" 2>/dev/null; then
                success "ISO already on Proxmox: ${ISO_FILENAME}"
            else
                step "Downloading ISO from CDN"
                local dl_url="https://images.linuxcontainers.org/os/${ISO_VERSION}/x86_64/IncusOS_${ISO_VERSION}.iso.gz"
                local tmp_gz="/tmp/${ISO_FILENAME}.gz"
                local tmp_iso="/tmp/${ISO_FILENAME}"
                curl -fSk -o "$tmp_gz" "$dl_url" 2>&1 | tail -1 || {
                    error "Failed to download ${dl_url}"; exit 1
                }
                info "Decompressing..."
                gunzip -f "$tmp_gz"
                step "Uploading to Proxmox"
                local url="https://${PVE_HOST}:8006/api2/json/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/upload"
                local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}"
                curl -fsSk -H "$auth" -F "content=iso" \
                    -F "filename=@${tmp_iso};filename=${ISO_FILENAME}" "$url" &>/dev/null || {
                    error "Failed to upload ISO"; exit 1
                }
                rm -f "$tmp_iso"
                ISO_UPLOADED_BY_US=true
                success "Uploaded: ${ISO_FILENAME}"
            fi
            ;;
        *)
            # File path
            if [[ ! -f "$ISO_SOURCE" ]]; then
                error "ISO file not found: ${ISO_SOURCE}"; exit 1
            fi
            local basename
            basename=$(basename "$ISO_SOURCE")
            ISO_FILENAME="$basename"
            ISO_VERSION=$(echo "$basename" | sed -n 's/.*_\([0-9]*\)\..*/\1/p')
            [[ -z "$ISO_VERSION" ]] && ISO_VERSION="custom"

            # Upload to Proxmox
            step "Uploading custom ISO: ${basename}"
            local url="https://${PVE_HOST}:8006/api2/json/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/upload"
            local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}"
            curl -fsSk -H "$auth" -F "content=iso" \
                -F "filename=@${ISO_SOURCE};filename=${basename}" "$url" &>/dev/null || {
                error "Failed to upload ISO"; exit 1
            }
            ISO_UPLOADED_BY_US=true
            success "Using custom ISO: ${ISO_FILENAME} (version ${ISO_VERSION})"
            ;;
    esac
}

# ---------------------------------------------------------------------------
# Seed generation
# ---------------------------------------------------------------------------

SEED_ISO=""
SEED_TMPDIR=""

generate_seed() {
    step "Generating seed ISO"
    SEED_TMPDIR=$(mktemp -d)
    local seed_name="seed-${VM_NAME}.iso"
    SEED_ISO="$seed_name"
    local local_path="${SEED_TMPDIR}/${seed_name}"

    local defaults_flag=""
    if [[ "$APPLY_DEFAULTS" == true ]]; then
        defaults_flag="--defaults"
    else
        defaults_flag="--no-defaults"
    fi

    "$SEED_TOOL" --format iso --app incus --output "$local_path" \
        --force-install --force-reboot --hostname "$VM_NAME" \
        $defaults_flag --quiet &>/dev/null

    success "Seed ISO: ${seed_name} (apply_defaults=${APPLY_DEFAULTS})"

    step "Uploading seed ISO"
    local url="https://${PVE_HOST}:8006/api2/json/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/upload"
    local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}"
    curl -fsSk -H "$auth" -F "content=iso" \
        -F "filename=@${local_path};filename=${seed_name}" "$url" &>/dev/null
    success "Seed uploaded"
}

# ---------------------------------------------------------------------------
# VM lifecycle
# ---------------------------------------------------------------------------

create_vm() {
    local json_payload
    json_payload=$(python3 -c "
import json
payload = {
    'vmid': ${VMID},
    'name': '${VM_NAME}',
    'description': '[incusos-lab:managed] investigate-boot',
    'ostype': 'l26',
    'bios': 'ovmf',
    'machine': 'q35',
    'efidisk0': '${PVE_STORAGE}:0,efitype=4m,pre-enrolled-keys=0',
    'tpmstate0': '${PVE_STORAGE}:1,version=v2.0',
    'cpu': 'host',
    'cores': 4,
    'memory': 4096,
    'balloon': 0,
    'scsihw': 'virtio-scsi-pci',
    'scsi0': '${PVE_STORAGE}:50,iothread=1',
    'ide2': '${PVE_ISO_STORAGE}:iso/${ISO_FILENAME},media=cdrom',
    'ide3': '${PVE_ISO_STORAGE}:iso/${SEED_ISO},media=cdrom',
    'net0': 'virtio,bridge=${PVE_BRIDGE}',
    'boot': 'order=ide2;scsi0',
    'agent': 1,
}
pool = '${PVE_POOL}'
if pool:
    payload['pool'] = pool
print(json.dumps(payload))
")

    local url="https://${PVE_HOST}:8006/api2/json/nodes/${PVE_NODE}/qemu"
    local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}"
    local resp
    resp=$(curl -fsSk -H "$auth" -H "Content-Type: application/json" \
        -X POST -d "$json_payload" "$url" 2>/dev/null) || { error "Failed to create VM"; return 1; }

    # Wait for async task
    local upid
    upid=$(echo "$resp" | python3 -c "import json,sys; print(json.load(sys.stdin).get('data',''))" 2>/dev/null) || upid=""
    if [[ -n "$upid" ]]; then
        local enc
        enc=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${upid}', safe=''))" 2>/dev/null)
        local w=0
        while [[ $w -lt 60 ]]; do
            sleep 2; w=$((w + 2))
            local st
            st=$(pve_api GET "/nodes/${PVE_NODE}/tasks/${enc}/status" 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status',''))" 2>/dev/null) || st=""
            [[ "$st" == "stopped" ]] && break
        done
    fi
}

cleanup_vm() {
    local status
    status=$(pve_vm_status "$VMID" 2>/dev/null) || status="unknown"
    [[ "$status" == "running" ]] && { pve_stop_vm "$VMID" || true; sleep 3; }
    pve_destroy_vm "$VMID" || true
    pve_api DELETE "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content/${PVE_ISO_STORAGE}:iso/${SEED_ISO}" &>/dev/null || true
    if command -v incus &>/dev/null; then
        local cur
        cur=$(incus remote get-default 2>/dev/null) || true
        [[ "$cur" == "$VM_NAME" ]] && { incus remote switch local 2>/dev/null || true; }
        incus remote remove "$VM_NAME" 2>/dev/null || true
    fi
    rm -rf "${SEED_TMPDIR:-}" 2>/dev/null || true
    rm -f "/tmp/vm-${VMID}-screen.ppm" 2>/dev/null || true
}

# ---------------------------------------------------------------------------
# IncusOS API probing
# ---------------------------------------------------------------------------

probe_incusos_api() {
    # Probe the IncusOS REST API via the Incus /os/ proxy.
    # Requires that the VM has been added as an incus remote.
    local ip="$1"
    local probe_file="${RUN_DIR}/api-probe.json"
    local cert_file="${HOME}/.config/incus/client.crt"
    local key_file="${HOME}/.config/incus/client.key"

    if [[ ! -f "$cert_file" ]] || [[ ! -f "$key_file" ]]; then
        log "API probe: no client cert available"
        echo '{"error": "no client cert"}' > "$probe_file"
        return
    fi

    log "Probing IncusOS API at ${ip}..."

    # Add as temporary incus remote
    incus remote add "$VM_NAME" "https://${ip}:8443" --accept-certificate 2>/dev/null || true

    # Probe storage config (scrub_schedule)
    local storage_json
    storage_json=$(incus query "${VM_NAME}:/os/1.0/system/storage" 2>/dev/null) || storage_json='{"error":"query failed"}'

    # Probe update config (check_frequency)
    local update_json
    update_json=$(incus query "${VM_NAME}:/os/1.0/system/update" 2>/dev/null) || update_json='{"error":"query failed"}'

    # Probe system info (os_version, hostname, uptime)
    local system_json
    system_json=$(incus query "${VM_NAME}:/os/1.0" 2>/dev/null) || system_json='{"error":"query failed"}'

    # Write combined probe results
    python3 -c "
import json, sys
storage = json.loads('''${storage_json}''')
update = json.loads('''${update_json}''')
system = json.loads('''${system_json}''')
result = {
    'storage': storage,
    'update': update,
    'system': system,
    'scrub_schedule': storage.get('config', {}).get('scrub_schedule', ''),
    'check_frequency': update.get('config', {}).get('check_frequency', ''),
    'os_version': system.get('environment', {}).get('os_version', ''),
    'hostname': system.get('environment', {}).get('hostname', ''),
    'uptime': system.get('environment', {}).get('uptime', 0),
}
print(json.dumps(result, indent=2))
" > "$probe_file" 2>/dev/null || echo '{"error":"probe parse failed"}' > "$probe_file"

    # Extract key values for summary
    local scrub check_freq os_ver
    scrub=$(python3 -c "import json; d=json.load(open('${probe_file}')); print(d.get('scrub_schedule',''))" 2>/dev/null) || scrub=""
    check_freq=$(python3 -c "import json; d=json.load(open('${probe_file}')); print(d.get('check_frequency',''))" 2>/dev/null) || check_freq=""
    os_ver=$(python3 -c "import json; d=json.load(open('${probe_file}')); print(d.get('os_version',''))" 2>/dev/null) || os_ver=""

    log "API probe results:"
    log "  scrub_schedule: '${scrub}'"
    log "  check_frequency: '${check_freq}'"
    log "  os_version: '${os_ver}'"

    # Store for summary
    RUN_SCRUB_SCHEDULES+=("$scrub")
    RUN_CHECK_FREQUENCIES+=("$check_freq")
    RUN_OS_VERSIONS+=("$os_ver")
}

# ---------------------------------------------------------------------------
# Install monitoring
# ---------------------------------------------------------------------------

check_install_progress() {
    local wr_bytes
    wr_bytes=$(pve_vm_disk_writes "$VMID" 2>/dev/null) || wr_bytes=0
    if [[ $wr_bytes -gt $PREV_WR_BYTES ]]; then
        if [[ "$WRITES_STARTED" != true ]]; then
            WRITES_STARTED=true
            PHASE="installing"
            log "Install started — disk writes detected"
        fi
        STABLE_COUNT=0
    elif [[ "$WRITES_STARTED" == true ]]; then
        STABLE_COUNT=$((STABLE_COUNT + 1))
        if [[ $STABLE_COUNT -ge $STABLE_THRESHOLD ]]; then
            local wr_mb=$(( wr_bytes / 1048576 ))
            log "Install complete — ${wr_mb} MiB written, ${STABLE_COUNT} stable polls"
            PHASE="install_done"
        fi
    fi
    PREV_WR_BYTES=$wr_bytes
}

# ---------------------------------------------------------------------------
# Single deploy cycle
# ---------------------------------------------------------------------------

run_single_deploy() {
    local run_num="$1"
    local deploy_start
    deploy_start=$(date +%s)

    # Reset state
    PHASE="init"
    PREV_WR_BYTES=0
    WRITES_STARTED=false
    STABLE_COUNT=0
    FRAME_NUM=0

    # Setup run directory
    RUN_DIR="${RESULTS_DIR}/run-$(printf '%02d' "$run_num")"
    mkdir -p "$RUN_DIR"
    LOG_FILE="${RUN_DIR}/deploy.log"
    touch "$LOG_FILE"

    log "=== Run ${run_num}/${RUNS} ==="
    log "ISO: ${ISO_FILENAME} (version ${ISO_VERSION})"
    log "apply_defaults: ${APPLY_DEFAULTS}"

    # Generate seed (fresh each time)
    generate_seed

    # Create VM
    step "Run ${run_num}: Creating VM (VMID ${VMID})"
    create_vm || { error "Failed to create VM"; return 1; }
    success "VM created"

    # --- Phase 1: Boot from ISO ---
    step "Run ${run_num}: Starting VM (ISO boot)"
    pve_start_vm "$VMID"
    PHASE="iso_boot"
    log "VM started — phase: iso_boot"

    local elapsed=0
    local last_bs_check=0

    while [[ "$PHASE" == "iso_boot" ]] || [[ "$PHASE" == "installing" ]]; do
        take_frame || true
        local now; now=$(date +%s)
        if [[ $((now - last_bs_check)) -ge $INSTALL_POLL ]]; then
            check_install_progress
            last_bs_check=$now
        fi
        sleep "$SCREENSHOT_INTERVAL"
        elapsed=$((elapsed + SCREENSHOT_INTERVAL))
        local status; status=$(pve_vm_status "$VMID" 2>/dev/null) || status="unknown"
        [[ "$status" == "stopped" ]] && { log "VM stopped during install"; PHASE="install_done"; }
        if [[ $elapsed -ge $INSTALL_TIMEOUT ]]; then
            warn "Install timeout after ${INSTALL_TIMEOUT}s"
            take_frame || true
            PHASE="failed"
            break
        fi
        if [[ $((elapsed % 30)) -lt $SCREENSHOT_INTERVAL ]]; then
            local wr_mb=$(( PREV_WR_BYTES / 1048576 ))
            log "Progress: ${elapsed}s, ${wr_mb} MiB, phase=${PHASE}"
        fi
    done

    [[ "$PHASE" == "failed" ]] && return 1

    # --- Phase 2: Transition ---
    PHASE="transitioning"
    take_frame || true
    step "Run ${run_num}: Stopping VM, removing media"
    pve_stop_vm "$VMID" || true
    local sw=0
    while [[ $sw -lt 30 ]]; do
        local st; st=$(pve_vm_status "$VMID" 2>/dev/null) || st="unknown"
        [[ "$st" == "stopped" ]] && break
        sleep 2; sw=$((sw + 2))
    done
    pve_api PUT "/nodes/${PVE_NODE}/qemu/${VMID}/config" \
        '{"delete":"ide2,ide3","boot":"order=scsi0"}' json &>/dev/null || {
        error "Failed to remove install media"; PHASE="failed"; return 1
    }
    log "Media removed, boot=scsi0"

    # --- Phase 3: Boot from disk ---
    step "Run ${run_num}: Starting from disk"
    pve_start_vm "$VMID"
    PHASE="disk_boot"
    log "Disk boot started"

    local boot_elapsed=0
    local ip_detected=""
    local ip_started=false

    while [[ "$PHASE" == "disk_boot" ]]; do
        take_frame || true
        sleep "$SCREENSHOT_INTERVAL"
        boot_elapsed=$((boot_elapsed + SCREENSHOT_INTERVAL))

        if [[ $boot_elapsed -ge 30 ]] && [[ "$ip_started" != true ]]; then
            ip_started=true
            log "Starting IP detection"
        fi

        if [[ "$ip_started" == true ]] && [[ $((boot_elapsed % 10)) -lt $SCREENSHOT_INTERVAL ]]; then
            local ip
            ip=$(pve_vm_ip "$VMID" 2>/dev/null) || ip=""
            if [[ -n "$ip" ]]; then
                [[ -z "$ip_detected" ]] && { ip_detected="$ip"; log "IP: ${ip}"; }
                if curl -sk --connect-timeout 3 "https://${ip}:8443" &>/dev/null; then
                    log "Port 8443 UP at ${ip} (${boot_elapsed}s)"
                    PHASE="ready"
                    break
                fi
            fi
        fi

        if [[ $boot_elapsed -ge $BOOT_TIMEOUT ]]; then
            warn "Boot timeout after ${BOOT_TIMEOUT}s"
            take_frame || true
            PHASE="failed"
            break
        fi

        if [[ $((boot_elapsed % 15)) -lt $SCREENSHOT_INTERVAL ]]; then
            log "Disk boot: ${boot_elapsed}s, ip=${ip_detected:-none}, phase=${PHASE}"
        fi
    done

    # Take final frames
    if [[ "$PHASE" == "ready" ]]; then
        for _ in 1 2 3; do sleep "$SCREENSHOT_INTERVAL"; take_frame || true; done
    fi

    local deploy_end; deploy_end=$(date +%s)
    local total=$((deploy_end - deploy_start))

    log "Result: phase=${PHASE}, total=${total}s, frames=${FRAME_NUM}"

    # --- Phase 4: API probing ---
    if [[ "$PHASE" == "ready" ]] && [[ -n "$ip_detected" ]]; then
        probe_incusos_api "$ip_detected"
    else
        RUN_SCRUB_SCHEDULES+=("N/A")
        RUN_CHECK_FREQUENCIES+=("N/A")
        RUN_OS_VERSIONS+=("N/A")
    fi

    RUN_TIMES+=("$total")
    RUN_DIRS+=("$RUN_DIR")

    if [[ "$PHASE" == "ready" ]]; then
        RUN_OUTCOMES+=("PASS")
        return 0
    else
        RUN_OUTCOMES+=("FAIL")
        return 1
    fi
}

# ---------------------------------------------------------------------------
# Preflight
# ---------------------------------------------------------------------------

preflight() {
    step "Preflight checks"
    local missing=()
    command -v sshpass &>/dev/null || missing+=("sshpass")
    command -v python3 &>/dev/null || missing+=("python3")
    command -v curl &>/dev/null || missing+=("curl")
    [[ ${#missing[@]} -gt 0 ]] && { error "Missing: ${missing[*]}"; exit 1; }

    python3 -c "from PIL import Image" &>/dev/null || warn "PIL not available — PPM screenshots"

    if ! sshpass -p "$PROXMOX_ROOT_PASSWORD" ssh "${SSH_OPTS[@]}" "root@${PVE_HOST}" "echo ok" &>/dev/null; then
        error "Cannot SSH to root@${PVE_HOST}"; exit 1
    fi
    success "SSH OK"

    if [[ $VMID -lt 850 ]] || [[ $VMID -gt 869 ]]; then
        error "VMID ${VMID} outside test range (850-869)"; exit 1
    fi
    success "VMID ${VMID} in range"
}

# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

echo -e "${BOLD}${SCRIPT_NAME}${RESET} v${VERSION}"
echo "Systematic IncusOS boot failure investigation"
echo ""

if [[ -z "$LABEL" ]]; then
    case "$ISO_SOURCE" in
        cdn) LABEL="cdn-latest" ;;
        old) LABEL="old-202602200553" ;;
        *)   LABEL="custom-iso" ;;
    esac
fi

if [[ "$DRY_RUN" == true ]]; then
    echo -e "${YELLOW}[dry-run] Preview mode${RESET}"
    echo ""
    echo "Configuration:"
    echo "  Runs:           ${RUNS}"
    echo "  ISO source:     ${ISO_SOURCE}"
    echo "  VMID:           ${VMID}"
    echo "  Label:          ${LABEL}"
    echo "  apply_defaults: ${APPLY_DEFAULTS}"
    echo "  Screenshots:    every ${SCREENSHOT_INTERVAL}s"
    echo ""
    load_proxmox_yaml
    detail "Host: ${PVE_HOST}, Node: ${PVE_NODE}"
    echo ""
    exit 0
fi

# Setup
step "Loading configuration"
load_proxmox_yaml
detail "Host: ${PVE_HOST}, Node: ${PVE_NODE}, Storage: ${PVE_STORAGE}"

preflight

resolve_iso

# Setup results directory
RESULTS_DIR="${SCRIPT_DIR}/observe-runs/investigation_$(date '+%Y-%m-%d_%H-%M-%S')_${LABEL}"
mkdir -p "$RESULTS_DIR"
success "Results: ${RESULTS_DIR}/"

echo ""
echo -e "${BOLD}Investigation plan:${RESET}"
echo "  Runs:           ${RUNS}"
echo "  ISO:            ${ISO_FILENAME} (version ${ISO_VERSION})"
echo "  apply_defaults: ${APPLY_DEFAULTS}"
echo "  VMID:           ${VMID}"
echo ""

# Trap for cleanup
trap_cleanup() {
    echo ""
    warn "Interrupted — cleaning up"
    cleanup_vm 2>/dev/null || true
    exit 130
}
trap trap_cleanup INT TERM

# Run the investigation cycles
for ((run=1; run<=RUNS; run++)); do
    echo ""
    echo -e "${BOLD}════════════════════════════════════════${RESET}"
    echo -e "${BOLD}  Run ${run}/${RUNS}${RESET}"
    echo -e "${BOLD}════════════════════════════════════════${RESET}"
    echo ""

    # Ensure VMID is free
    if pve_vm_exists "$VMID"; then
        warn "VMID ${VMID} exists — cleaning up from previous run"
        cleanup_vm
        sleep 5
    fi

    run_result=0
    run_single_deploy "$run" || run_result=$?

    echo ""
    if [[ $run_result -eq 0 ]]; then
        echo -e "  Result: ${GREEN}PASS${RESET}"
    else
        echo -e "  Result: ${RED}FAIL${RESET} (phase: ${PHASE})"
    fi

    # Cleanup
    step "Cleanup run ${run}"
    cleanup_vm
    sleep 3  # Brief pause between runs
done

# ---------------------------------------------------------------------------
# Results summary
# ---------------------------------------------------------------------------

echo ""
echo ""
echo -e "${BOLD}╔══════════════════════════════════════════╗${RESET}"
echo -e "${BOLD}║     Investigation Results Summary        ║${RESET}"
echo -e "${BOLD}╚══════════════════════════════════════════╝${RESET}"
echo ""
echo "ISO: ${ISO_FILENAME} (version ${ISO_VERSION})"
echo "ISO source: ${ISO_SOURCE}"
echo "apply_defaults: ${APPLY_DEFAULTS}"
echo "Runs: ${RUNS}"
echo ""

pass_count=0
fail_count=0

for ((i=0; i<${#RUN_OUTCOMES[@]}; i++)); do
    local_run=$((i + 1))
    outcome="${RUN_OUTCOMES[$i]}"
    time_s="${RUN_TIMES[$i]:-?}"
    scrub="${RUN_SCRUB_SCHEDULES[$i]:-?}"
    freq="${RUN_CHECK_FREQUENCIES[$i]:-?}"
    osver="${RUN_OS_VERSIONS[$i]:-?}"

    marker=""
    case "$outcome" in
        PASS) marker="${GREEN}PASS${RESET}"; pass_count=$((pass_count + 1)) ;;
        FAIL) marker="${RED}FAIL${RESET}"; fail_count=$((fail_count + 1)) ;;
    esac

    echo -e "  Run $(printf '%2d' $local_run): ${marker}  time=${time_s}s  scrub='${scrub}'  freq='${freq}'  os=${osver}"
done

echo ""
echo -e "  Total: ${GREEN}${pass_count} PASS${RESET}, ${RED}${fail_count} FAIL${RESET} (of ${RUNS})"

if [[ $fail_count -gt 0 ]]; then
    pct=$(( fail_count * 100 / RUNS ))
    echo -e "  Failure rate: ${RED}${pct}%${RESET}"
fi

echo ""

# Write structured results file
results_file="${RESULTS_DIR}/results.txt"
{
    echo "# investigate-boot results"
    echo "# Generated: $(date -Iseconds)"
    echo "# ISO: ${ISO_FILENAME} (version ${ISO_VERSION})"
    echo "# ISO source: ${ISO_SOURCE}"
    echo "# apply_defaults: ${APPLY_DEFAULTS}"
    echo "# Runs: ${RUNS}"
    echo "#"
    echo "# run|outcome|time_s|scrub_schedule|check_frequency|os_version|run_dir"
    for ((i=0; i<${#RUN_OUTCOMES[@]}; i++)); do
        echo "$((i+1))|${RUN_OUTCOMES[$i]}|${RUN_TIMES[$i]:-?}|${RUN_SCRUB_SCHEDULES[$i]:-?}|${RUN_CHECK_FREQUENCIES[$i]:-?}|${RUN_OS_VERSIONS[$i]:-?}|${RUN_DIRS[$i]:-?}"
    done
} > "$results_file"

success "Results written to: ${results_file}"
echo ""

exit $fail_count