#!/usr/bin/env bash
# investigate-parallel - Parallel IncusOS boot investigation with smart phase detection
#
# Runs N VMs simultaneously, monitors each independently, probes API on success.
# Uses aggressive port polling + screenshot capture for phase detection.
#
# Usage:
#   ./investigate-parallel --count 4 --iso old --label baseline
#   ./investigate-parallel --count 4 --iso new --label version-mismatch
#   ./investigate-parallel --count 4 --iso /tmp/IncusOS-oc.iso --label oc-iso

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SEED_TOOL="${SCRIPT_DIR}/incusos-seed"
PVE_API="${SCRIPT_DIR}/pve-api"

# Defaults
COUNT=4
START_VMID=850
ISO_MODE="old"    # old, new, or file path
LABEL=""
DRY_RUN=false
BOOT_TIMEOUT=240  # longer for version-mismatch OS update path

# Proxmox config
PVE_HOST=""
PVE_NODE=""
PVE_STORAGE=""
PVE_ISO_STORAGE=""
PVE_BRIDGE=""
PVE_POOL=""

# ISO
ISO_FILENAME=""
ISO_VERSION=""

# Results
RESULTS_DIR=""

# ---------------------------------------------------------------------------
# Colors
# ---------------------------------------------------------------------------

if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then
    RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[0;33m'
    CYAN='\033[0;36m' BOLD='\033[1m' DIM='\033[2m' RESET='\033[0m'
else
    RED='' GREEN='' YELLOW='' CYAN='' BOLD='' DIM='' RESET=''
fi

step()    { echo -e "${CYAN}[step]${RESET}    ${BOLD}$*${RESET}"; }
success() { echo -e "${GREEN}[ok]${RESET}      $*"; }
warn()    { echo -e "${YELLOW}[warn]${RESET}    $*" >&2; }
error()   { echo -e "${RED}[error]${RESET}   $*" >&2; }
info()    { echo -e "${DIM}[info]${RESET}    $*"; }

# ---------------------------------------------------------------------------
# Argument parsing
# ---------------------------------------------------------------------------

while [[ $# -gt 0 ]]; do
    case "$1" in
        --count)   COUNT="${2:?'--count requires N'}"; shift 2 ;;
        --iso)     ISO_MODE="${2:?'--iso requires old|new|path'}"; shift 2 ;;
        --label)   LABEL="${2:?'--label requires string'}"; shift 2 ;;
        --start-vmid) START_VMID="${2:?'--start-vmid requires N'}"; shift 2 ;;
        --dry-run) DRY_RUN=true; shift ;;
        --timeout) BOOT_TIMEOUT="${2:?'--timeout requires seconds'}"; shift 2 ;;
        -h|--help)
            echo "Usage: investigate-parallel [OPTIONS]"
            echo "  --count N       Parallel VMs (default: 4, max 8)"
            echo "  --iso MODE      old (202602200553), new (latest CDN), or file path"
            echo "  --label NAME    Results label"
            echo "  --start-vmid N  First VMID (default: 850)"
            echo "  --timeout N     Boot timeout seconds (default: 240)"
            echo "  --dry-run       Preview only"
            exit 0
            ;;
        *) error "Unknown: $1"; exit 1 ;;
    esac
done

# ---------------------------------------------------------------------------
# Load environment & config
# ---------------------------------------------------------------------------

load_env() {
    local dir="$SCRIPT_DIR"
    while [[ "$dir" != "/" ]]; do
        [[ -f "${dir}/env" ]] && { source "${dir}/env"; return; }
        dir="$(dirname "$dir")"
    done
}
load_env

[[ -z "${PROXMOX_TOKEN_SECRET:-}" ]] && { error "PROXMOX_TOKEN_SECRET not set"; exit 1; }
[[ -z "${PROXMOX_ROOT_PASSWORD:-}" ]] && { error "PROXMOX_ROOT_PASSWORD not set"; exit 1; }

yaml_get() {
    local val
    val=$(awk -F': ' -v k="$1" '$1 == k {print $2; exit}' "${SCRIPT_DIR}/proxmox.yaml" 2>/dev/null) || val=""
    val="${val#\"}" ; val="${val%\"}" ; val="${val#\'}" ; val="${val%\'}"
    echo "${val:-${2:-}}"
}

PVE_HOST=$(yaml_get host)
PVE_NODE=$(yaml_get node pve)
PVE_STORAGE=$(yaml_get storage local-lvm)
PVE_ISO_STORAGE=$(yaml_get iso_storage local)
PVE_BRIDGE=$(yaml_get bridge vmbr0)
PVE_POOL=$(yaml_get pool)

# ---------------------------------------------------------------------------
# Screenshot helpers
# ---------------------------------------------------------------------------

SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -o LogLevel=ERROR)

take_screenshot() {
    local vmid="$1" output="$2"
    local rppm="/tmp/vm-${vmid}-screen.ppm"
    sshpass -p "$PROXMOX_ROOT_PASSWORD" ssh "${SSH_OPTS[@]}" "root@${PVE_HOST}" \
        "echo 'screendump ${rppm}' | qm monitor ${vmid}" &>/dev/null || return 1
    sshpass -p "$PROXMOX_ROOT_PASSWORD" scp "${SSH_OPTS[@]}" \
        "root@${PVE_HOST}:${rppm}" "/tmp/vm-${vmid}-screen.ppm" &>/dev/null || return 1
    sshpass -p "$PROXMOX_ROOT_PASSWORD" ssh "${SSH_OPTS[@]}" "root@${PVE_HOST}" \
        "rm -f ${rppm}" &>/dev/null || true
    python3 -c "from PIL import Image; Image.open('/tmp/vm-${vmid}-screen.ppm').save('${output}')" 2>/dev/null || {
        mv "/tmp/vm-${vmid}-screen.ppm" "${output%.png}.ppm" 2>/dev/null || true
    }
    rm -f "/tmp/vm-${vmid}-screen.ppm" 2>/dev/null || true
}

# ---------------------------------------------------------------------------
# ISO resolution
# ---------------------------------------------------------------------------

resolve_iso() {
    case "$ISO_MODE" in
        old)
            ISO_FILENAME="IncusOS_202602200553.iso"
            ISO_VERSION="202602200553"
            ;;
        new)
            # Find latest on Proxmox
            local latest
            latest=$("$PVE_API" "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content?content=iso" 2>/dev/null | python3 -c "
import json, sys
data = json.load(sys.stdin).get('data', [])
isos = [d['volid'].split('/')[-1] for d in data if 'IncusOS_' in d.get('volid','') and '-oc' not in d['volid'].split('/')[-1].lower()]
isos.sort(reverse=True)
print(isos[0] if isos else '')
" 2>/dev/null)
            if [[ -z "$latest" ]]; then
                error "No IncusOS ISO found on Proxmox"; exit 1
            fi
            ISO_FILENAME="$latest"
            ISO_VERSION=$(echo "$latest" | sed -n 's/IncusOS_\([0-9]*\)\..*/\1/p')
            ;;
        *)
            # File path — must already be uploaded
            ISO_FILENAME="$(basename "$ISO_MODE")"
            ISO_VERSION=$(echo "$ISO_FILENAME" | sed -n 's/.*_\([0-9]*\)\..*/\1/p')
            [[ -z "$ISO_VERSION" ]] && ISO_VERSION="custom"
            ;;
    esac
    success "ISO: ${ISO_FILENAME} (version ${ISO_VERSION})"
}

# ---------------------------------------------------------------------------
# Single VM lifecycle (called as background job)
# ---------------------------------------------------------------------------

deploy_single_vm() {
    local vmid="$1"
    local vm_name="$2"
    local run_dir="$3"
    local seed_iso="$4"
    local result_file="${run_dir}/result.txt"
    local log_file="${run_dir}/deploy.log"

    _log() { echo "[$(date '+%H:%M:%S.%3N')] $*" >> "$log_file"; }

    # Error trap: write FAIL result on any uncaught error
    trap '_log "ERROR at line $LINENO: exit code $?"; echo "ERROR|line_${LINENO}|0|N/A|N/A|N/A" > "$result_file"' ERR

    _log "=== VM ${vmid} (${vm_name}) ==="
    _log "ISO: ${ISO_FILENAME}, seed: ${seed_iso}"

    # Create VM
    local json_payload
    json_payload=$(python3 -c "
import json
p = {
    'vmid': ${vmid}, 'name': '${vm_name}',
    'description': '[incusos-lab:managed] investigate-parallel',
    'ostype': 'l26', 'bios': 'ovmf', 'machine': 'q35',
    'efidisk0': '${PVE_STORAGE}:0,efitype=4m,pre-enrolled-keys=0',
    'tpmstate0': '${PVE_STORAGE}:1,version=v2.0',
    'cpu': 'host', 'cores': 4, 'memory': 4096, 'balloon': 0,
    'scsihw': 'virtio-scsi-pci', 'scsi0': '${PVE_STORAGE}:50,iothread=1',
    'ide2': '${PVE_ISO_STORAGE}:iso/${ISO_FILENAME},media=cdrom',
    'ide3': '${PVE_ISO_STORAGE}:iso/${seed_iso},media=cdrom',
    'net0': 'virtio,bridge=${PVE_BRIDGE}',
    'boot': 'order=ide2;scsi0', 'agent': 1,
}
pool = '${PVE_POOL}'
if pool: p['pool'] = pool
print(json.dumps(p))
")

    "$PVE_API" "/nodes/${PVE_NODE}/qemu" POST "$json_payload" >/dev/null 2>&1 || {
        echo "ERROR|create_failed|0|N/A|N/A|N/A" > "$result_file"; return
    }
    sleep 15  # Wait for create task (allocating EFI, TPM, 50G disk)
    _log "VM created"

    # Start VM (ISO boot)
    "$PVE_API" "/nodes/${PVE_NODE}/qemu/${vmid}/status/start" POST '{}' >/dev/null 2>&1 || {
        _log "ERROR: Failed to start VM"; echo "ERROR|start_failed|0|N/A|N/A|N/A" > "$result_file"; return
    }
    _log "VM started"

    local frame=0

    # Monitor install via blockstat (screenshots every 15s to avoid SSH contention)
    local elapsed=0 prev_wr=0 writes_started=false stable=0
    while [[ $elapsed -lt 300 ]]; do
        sleep 5; elapsed=$((elapsed + 5))
        # Screenshot every 15s during install
        if [[ $((elapsed % 15)) -lt 6 ]]; then
            take_screenshot "$vmid" "${run_dir}/frame-$(printf '%03d' $((++frame))).png" 2>/dev/null || true
        fi

        local wr
        wr=$("$PVE_API" "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" 2>/dev/null | python3 -c "
import json,sys; d=json.load(sys.stdin).get('data',{}); print(d.get('blockstat',{}).get('scsi0',{}).get('wr_bytes',0))" 2>/dev/null) || wr=0

        if [[ $wr -gt $prev_wr ]]; then
            [[ "$writes_started" != true ]] && { writes_started=true; _log "Install writes started"; }
            stable=0
        elif [[ "$writes_started" == true ]]; then
            stable=$((stable + 1))
            [[ $stable -ge 3 ]] && { _log "Install complete ($(( wr / 1048576 )) MiB)"; break; }
        fi
        prev_wr=$wr

        # Check if VM stopped
        local st
        st=$("$PVE_API" "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin)['data']['status'])" 2>/dev/null) || st="unknown"
        [[ "$st" == "stopped" ]] && { _log "VM stopped during install"; break; }
    done

    # Take screenshot showing install-complete/media-error state
    take_screenshot "$vmid" "${run_dir}/frame-$(printf '%03d' $((++frame))).png" 2>/dev/null || true

    # Transition: stop, remove media, boot from disk
    "$PVE_API" "/nodes/${PVE_NODE}/qemu/${vmid}/status/stop" POST '{}' >/dev/null 2>&1 || true
    sleep 8
    # Wait for stop to complete
    for _ in 1 2 3 4 5; do
        local vm_st
        vm_st=$("$PVE_API" "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin)['data']['status'])" 2>/dev/null) || vm_st="unknown"
        [[ "$vm_st" == "stopped" ]] && break
        sleep 3
    done
    "$PVE_API" "/nodes/${PVE_NODE}/qemu/${vmid}/config" "PUT" '{"delete":"ide2,ide3","boot":"order=scsi0"}' >/dev/null 2>&1 || {
        _log "ERROR: Failed to remove media"; echo "ERROR|media_remove|0|N/A|N/A|N/A" > "$result_file"; return
    }
    _log "Media removed"

    "$PVE_API" "/nodes/${PVE_NODE}/qemu/${vmid}/status/start" POST '{}' >/dev/null 2>&1 || {
        _log "ERROR: Failed to start from disk"; echo "ERROR|disk_start|0|N/A|N/A|N/A" > "$result_file"; return
    }
    _log "Disk boot started"
    local boot_start
    boot_start=$(date +%s)

    # Poll for port 8443 with periodic screenshots
    local ip="" boot_elapsed=0 port_up=false
    while [[ $boot_elapsed -lt $BOOT_TIMEOUT ]]; do
        sleep 5; boot_elapsed=$(( $(date +%s) - boot_start ))
        # Screenshot every 15s during boot
        if [[ $((boot_elapsed % 15)) -lt 6 ]]; then
            take_screenshot "$vmid" "${run_dir}/frame-$(printf '%03d' $((++frame))).png" 2>/dev/null || true
        fi

        # Try to find IP (after 15s)
        if [[ $boot_elapsed -ge 15 ]] && [[ -z "$ip" ]]; then
            # Get MAC from Proxmox config (only once)
            if [[ -z "${mac:-}" ]]; then
                mac=$("$PVE_API" "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null | python3 -c "
import json, sys, re
net0 = json.load(sys.stdin).get('data',{}).get('net0','')
m = re.search(r'([0-9A-Fa-f:]{17})', net0)
print(m.group(1).lower() if m else '')
" 2>/dev/null) || mac=""
                [[ -n "$mac" ]] && _log "MAC: ${mac}"
            fi
            if [[ -n "${mac:-}" ]]; then
                # Check ARP table (don't flush — other VMs are using it)
                ip=$(ip neigh show | awk -v mac="$mac" 'tolower($0) ~ tolower(mac) {print $1; exit}') || ip=""
                # If not found, do a targeted ARP probe: ping a range of likely DHCP IPs
                if [[ -z "$ip" ]]; then
                    # Only sweep every 30 seconds to avoid overload
                    local now_s
                    now_s=$(date +%s)
                    if [[ $(( now_s - ${_last_sweep:-0} )) -ge 25 ]]; then
                        _last_sweep=$now_s
                        _log "Running ARP sweep"
                        # Sweep in smaller batches to avoid overwhelming
                        for octet in $(seq 1 254); do
                            ping -c 1 -W 1 "192.168.1.${octet}" &>/dev/null &
                        done
                        wait 2>/dev/null || true
                        sleep 2
                    fi
                    ip=$(ip neigh show | awk -v mac="$mac" 'tolower($0) ~ tolower(mac) {print $1; exit}') || ip=""
                fi
                if [[ -n "$ip" ]]; then
                    if ping -c 1 -W 2 "$ip" &>/dev/null; then
                        _log "IP detected: ${ip}"
                    else
                        _log "IP ${ip} not responding, retrying"
                        ip=""
                    fi
                fi
            fi
        fi

        # Check port 8443
        if [[ -n "$ip" ]]; then
            if curl -sk --connect-timeout 3 "https://${ip}:8443" &>/dev/null; then
                port_up=true
                _log "Port 8443 UP at ${ip} (${boot_elapsed}s)"
                break
            fi
        fi

        [[ $((boot_elapsed % 30)) -lt 6 ]] && _log "Disk boot: ${boot_elapsed}s, ip=${ip:-none}"
    done

    # Take final screenshot
    take_screenshot "$vmid" "${run_dir}/frame-$(printf '%03d' $((++frame))).png" 2>/dev/null || true

    # Probe API if successful
    local scrub="N/A" freq="N/A" osver="N/A"
    if [[ "$port_up" == true ]] && [[ -n "$ip" ]]; then
        # Add incus remote and probe
        incus remote add "$vm_name" "https://${ip}:8443" --accept-certificate 2>/dev/null || true
        local storage_json update_json system_json
        storage_json=$(incus query "${vm_name}:/os/1.0/system/storage" 2>/dev/null) || storage_json='{}'
        update_json=$(incus query "${vm_name}:/os/1.0/system/update" 2>/dev/null) || update_json='{}'
        system_json=$(incus query "${vm_name}:/os/1.0" 2>/dev/null) || system_json='{}'

        scrub=$(echo "$storage_json" | python3 -c "import json,sys; print(json.load(sys.stdin).get('config',{}).get('scrub_schedule','EMPTY'))" 2>/dev/null) || scrub="error"
        freq=$(echo "$update_json" | python3 -c "import json,sys; print(json.load(sys.stdin).get('config',{}).get('check_frequency','EMPTY'))" 2>/dev/null) || freq="error"
        osver=$(echo "$system_json" | python3 -c "import json,sys; print(json.load(sys.stdin).get('environment',{}).get('os_version','EMPTY'))" 2>/dev/null) || osver="error"

        _log "API: scrub='${scrub}' freq='${freq}' os=${osver}"

        # Save full probe data
        echo "$storage_json" > "${run_dir}/api-storage.json" 2>/dev/null || true
        echo "$update_json" > "${run_dir}/api-update.json" 2>/dev/null || true
        echo "$system_json" > "${run_dir}/api-system.json" 2>/dev/null || true

        # Cleanup remote
        local cur_remote
        cur_remote=$(incus remote get-default 2>/dev/null) || true
        [[ "$cur_remote" == "$vm_name" ]] && incus remote switch local 2>/dev/null || true
        incus remote remove "$vm_name" 2>/dev/null || true
    fi

    # Determine outcome:
    # - PASS = port up AND scrub_schedule is set (genuine success)
    # - CRONTAB_BUG = port up but scrub_schedule empty (detected during error window)
    # - FAIL = port never came up (could be bug + bad timing, or IP detection failure)
    local outcome="FAIL"
    if [[ "$port_up" == true ]]; then
        if [[ -n "$scrub" ]] && [[ "$scrub" != "" ]] && [[ "$scrub" != "EMPTY" ]] && [[ "$scrub" != "N/A" ]]; then
            outcome="PASS"
        else
            outcome="CRONTAB_BUG"
        fi
    fi

    echo "${outcome}|${boot_elapsed}|${scrub}|${freq}|${osver}|${ip:-none}" > "$result_file"
    _log "RESULT: ${outcome} boot=${boot_elapsed}s scrub='${scrub}' freq='${freq}' os=${osver}"
}

# ---------------------------------------------------------------------------
# Cleanup single VM
# ---------------------------------------------------------------------------

cleanup_vm() {
    local vmid="$1"
    local seed_iso="$2"
    "$PVE_API" "/nodes/${PVE_NODE}/qemu/${vmid}/status/stop" POST >/dev/null 2>&1 || true
    sleep 3
    "$PVE_API" "/nodes/${PVE_NODE}/qemu/${vmid}?purge=1&destroy-unreferenced-disks=1" DELETE >/dev/null 2>&1 || true
    "$PVE_API" "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content/${PVE_ISO_STORAGE}:iso/${seed_iso}" DELETE >/dev/null 2>&1 || true
}

# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

echo -e "${BOLD}investigate-parallel${RESET} — Parallel IncusOS boot investigation"
echo ""

[[ -z "$LABEL" ]] && LABEL="${ISO_MODE}"

if [[ "$DRY_RUN" == true ]]; then
    echo "  Count: ${COUNT}, ISO: ${ISO_MODE}, VMIDs: ${START_VMID}-$((START_VMID + COUNT - 1))"
    exit 0
fi

step "Resolving ISO"
resolve_iso

RESULTS_DIR="${SCRIPT_DIR}/observe-runs/parallel_$(date '+%Y-%m-%d_%H-%M-%S')_${LABEL}"
mkdir -p "$RESULTS_DIR"
success "Output: ${RESULTS_DIR}/"

# Generate seeds and create run dirs
step "Generating ${COUNT} seeds"
declare -a VM_NAMES=() VM_SEEDS=() VM_DIRS=()
for ((i=0; i<COUNT; i++)); do
    vmid=$((START_VMID + i))
    vm_name="inv-${vmid}"
    seed_name="seed-inv-${vmid}.iso"
    run_dir="${RESULTS_DIR}/vm-${vmid}"
    mkdir -p "$run_dir"

    VM_NAMES+=("$vm_name")
    VM_SEEDS+=("$seed_name")
    VM_DIRS+=("$run_dir")

    local_seed="/tmp/${seed_name}"
    "$SEED_TOOL" --format iso --app incus --output "$local_seed" \
        --force-install --force-reboot --hostname "$vm_name" \
        --defaults --quiet &>/dev/null

    # Upload seed (use Python to avoid bash ! quoting issues with API token)
    url="https://${PVE_HOST}:8006/api2/json/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/upload"
    python3 -c "
import subprocess, os, sys
# Read token_id from proxmox.yaml
token_id = ''
yaml_file = '${SCRIPT_DIR}/proxmox.yaml'
for line in open(yaml_file):
    line = line.strip()
    if line.startswith('api_token_id:'):
        token_id = line.split(': ', 1)[1].strip().strip(\"'\\\"\")
secret = os.environ.get('PROXMOX_TOKEN_SECRET', '')
if not token_id or not secret:
    sys.exit(1)
r = subprocess.run([
    'curl', '-fsSk',
    '-H', 'Authorization: PVEAPIToken=' + token_id + '=' + secret,
    '-F', 'content=iso',
    '-F', 'filename=@${local_seed};filename=${seed_name}',
    '${url}'
], capture_output=True)
sys.exit(r.returncode)
" 2>/dev/null
    rm -f "$local_seed"
    info "Seed $((i+1))/${COUNT}: ${seed_name}"
done
success "All seeds uploaded"

# Check VMIDs are free
step "Checking VMIDs"
for ((i=0; i<COUNT; i++)); do
    vmid=$((START_VMID + i))
    if "$PVE_API" "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" >/dev/null 2>&1; then
        error "VMID ${vmid} already in use"; exit 1
    fi
done
success "VMIDs ${START_VMID}-$((START_VMID + COUNT - 1)) available"

echo ""
echo -e "${BOLD}Launching ${COUNT} VMs in parallel${RESET}"
echo "  ISO: ${ISO_FILENAME} (${ISO_VERSION})"
echo "  VMIDs: ${START_VMID}-$((START_VMID + COUNT - 1))"
echo "  Boot timeout: ${BOOT_TIMEOUT}s"
echo ""

# Trap for cleanup
trap_cleanup() {
    echo ""
    warn "Interrupted — cleaning up all VMs"
    for ((i=0; i<COUNT; i++)); do
        cleanup_vm "$((START_VMID + i))" "${VM_SEEDS[$i]}" &
    done
    wait
    exit 130
}
trap trap_cleanup INT TERM

# Launch all VMs in parallel
pids=()
for ((i=0; i<COUNT; i++)); do
    vmid=$((START_VMID + i))
    ( set +e; deploy_single_vm "$vmid" "${VM_NAMES[$i]}" "${VM_DIRS[$i]}" "${VM_SEEDS[$i]}" 2>"${VM_DIRS[$i]}/stderr.log" ) &
    pids+=($!)
    info "Started VM ${vmid} (PID $!)"
done

# Wait for all to complete
for pid in "${pids[@]}"; do
    wait "$pid" || true
done

echo ""
echo -e "${BOLD}════════════════════════════════════════════════${RESET}"
echo -e "${BOLD}  Results (ISO: ${ISO_FILENAME})${RESET}"
echo -e "${BOLD}════════════════════════════════════════════════${RESET}"
echo ""

pass=0 fail=0 crontab_bug=0
for ((i=0; i<COUNT; i++)); do
    vmid=$((START_VMID + i))
    rfile="${VM_DIRS[$i]}/result.txt"
    if [[ -f "$rfile" ]]; then
        IFS='|' read -r outcome boot_s scrub freq osver vm_ip < "$rfile"
        if [[ "$outcome" == "PASS" ]]; then
            marker="${GREEN}PASS${RESET}"; pass=$((pass + 1))
        elif [[ "$outcome" == "CRONTAB_BUG" ]]; then
            marker="${YELLOW}CRONTAB_BUG${RESET}"; crontab_bug=$((crontab_bug + 1))
        else
            marker="${RED}FAIL${RESET}"; fail=$((fail + 1))
        fi
        echo -e "  ${vmid}: ${marker}  boot=${boot_s}s  scrub='${scrub}'  freq='${freq}'  os=${osver}  ip=${vm_ip}"
    else
        echo -e "  ${vmid}: ${RED}NO RESULT${RESET}"
        fail=$((fail + 1))
    fi
done

echo ""
echo -e "  Total: ${GREEN}${pass} PASS${RESET}, ${YELLOW}${crontab_bug} CRONTAB_BUG${RESET}, ${RED}${fail} FAIL/NO_IP${RESET} (of ${COUNT})"
total_bad=$((crontab_bug + fail))
[[ $total_bad -gt 0 ]] && echo -e "  Bug rate: ${YELLOW}$(( crontab_bug * 100 / COUNT ))%${RESET} confirmed, ${RED}$(( fail * 100 / COUNT ))%${RESET} unknown (IP detection failure)"

# Write summary
{
    echo "# investigate-parallel results"
    echo "# $(date -Iseconds)"
    echo "# ISO: ${ISO_FILENAME} (${ISO_VERSION})"
    echo "# Count: ${COUNT}, VMIDs: ${START_VMID}-$((START_VMID + COUNT - 1))"
    echo "#"
    echo "# vmid|outcome|boot_s|scrub_schedule|check_frequency|os_version|ip"
    for ((i=0; i<COUNT; i++)); do
        vmid=$((START_VMID + i))
        rfile="${VM_DIRS[$i]}/result.txt"
        [[ -f "$rfile" ]] && echo "${vmid}|$(cat "$rfile")"
    done
} > "${RESULTS_DIR}/results.txt"

echo ""

# Cleanup all VMs
step "Cleaning up"
for ((i=0; i<COUNT; i++)); do
    cleanup_vm "$((START_VMID + i))" "${VM_SEEDS[$i]}" &
done
wait
sleep 3
success "All cleaned up"

echo ""
echo "Screenshots: ${RESULTS_DIR}/vm-*/frame-*.png"
echo "Results:     ${RESULTS_DIR}/results.txt"
echo ""

exit $fail