#!/usr/bin/env bash
# observe-deploy - Single-VM IncusOS deploy with rapid console screenshots
#
# Captures the complete IncusOS lifecycle on screen — from ISO boot through
# installation, reboot, first-boot sequence, and port 8443 readiness. Takes
# screenshots every 2 seconds via Proxmox SSH root access (screendump).
#
# Purpose: diagnose boot failures by building a frame-by-frame timeline of
# exactly what IncusOS shows at every stage. NOT a batch test — one VM at
# a time, maximum visibility.
#
# Usage:
#   ./observe-deploy                          # Standard deploy with screenshots
#   ./observe-deploy --no-update-yaml         # Omit update.yaml from seed
#   ./observe-deploy --label run-01           # Custom run label
#   ./observe-deploy --vmid 855              # Use a specific VMID
#   ./observe-deploy --dry-run               # Preview without deploying
#
# Output: timestamped directory with PNG frames + deploy.log
#   observe-runs/YYYY-MM-DD_HH-MM-SS_<label>/
#     frame-001.png
#     frame-002.png
#     ...
#     deploy.log

set -euo pipefail

readonly VERSION="0.1.0"
readonly SCRIPT_NAME="observe-deploy"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SEED_TOOL="${SCRIPT_DIR}/incusos-seed"

# ---------------------------------------------------------------------------
# Defaults
# ---------------------------------------------------------------------------

VMID=850
VM_NAME="observe-deploy"
LABEL=""
NO_UPDATE_YAML=false
DRY_RUN=false
SCREENSHOT_INTERVAL=2       # seconds between screenshots
INSTALL_TIMEOUT=600         # seconds to wait for install completion
BOOT_TIMEOUT=180            # seconds to wait for port 8443 after disk boot
INSTALL_POLL=5              # blockstat poll interval during install phase
STABLE_THRESHOLD=3          # consecutive stable polls = install done

# Proxmox connection (from proxmox.yaml)
PVE_HOST=""
PVE_NODE=""
PVE_STORAGE=""
PVE_ISO_STORAGE=""
PVE_BRIDGE=""
PVE_POOL=""
PVE_API_TOKEN_ID=""

# IncusOS ISO
ISO_FILENAME=""

# Run output
RUN_DIR=""
LOG_FILE=""
FRAME_NUM=0

# State tracking
PHASE="init"   # init → iso_boot → installing → install_done → transitioning → disk_boot → ready/failed
PREV_WR_BYTES=0
WRITES_STARTED=false
STABLE_COUNT=0

# ---------------------------------------------------------------------------
# Colors
# ---------------------------------------------------------------------------

setup_colors() {
    if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then
        RED='\033[0;31m'
        GREEN='\033[0;32m'
        YELLOW='\033[0;33m'
        BLUE='\033[0;34m'
        CYAN='\033[0;36m'
        BOLD='\033[1m'
        DIM='\033[2m'
        RESET='\033[0m'
    else
        RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' RESET=''
    fi
}
setup_colors

step()    { echo -e "${CYAN}[step]${RESET}    ${BOLD}$*${RESET}"; }
success() { echo -e "${GREEN}[ok]${RESET}      $*"; }
warn()    { echo -e "${YELLOW}[warn]${RESET}    $*" >&2; }
error()   { echo -e "${RED}[error]${RESET}   $*" >&2; }
info()    { echo -e "${BLUE}[info]${RESET}    $*"; }
detail()  { echo -e "${DIM}          $*${RESET}"; }

# Log to both console and file
log() {
    local ts
    ts=$(date '+%H:%M:%S.%3N')
    local msg="[${ts}] $*"
    echo -e "${DIM}${msg}${RESET}"
    [[ -n "$LOG_FILE" ]] && echo "$msg" >> "$LOG_FILE"
}

# Log to file only (for high-frequency events)
log_quiet() {
    local ts
    ts=$(date '+%H:%M:%S.%3N')
    [[ -n "$LOG_FILE" ]] && echo "[${ts}] $*" >> "$LOG_FILE"
}

# ---------------------------------------------------------------------------
# Argument parsing
# ---------------------------------------------------------------------------

while [[ $# -gt 0 ]]; do
    case "$1" in
        --vmid)           VMID="${2:?'--vmid requires a number'}"; shift 2 ;;
        --label)          LABEL="${2:?'--label requires a string'}"; shift 2 ;;
        --no-update-yaml) NO_UPDATE_YAML=true; shift ;;
        --interval)       SCREENSHOT_INTERVAL="${2:?'--interval requires seconds'}"; shift 2 ;;
        --dry-run)        DRY_RUN=true; shift ;;
        -h|--help)
            cat <<EOF
${BOLD}${SCRIPT_NAME}${RESET} v${VERSION} - Observational IncusOS deploy with screenshots

${BOLD}USAGE${RESET}
    ${SCRIPT_NAME} [OPTIONS]

${BOLD}OPTIONS${RESET}
    --vmid N              VMID to use (default: 850, must be 850-869)
    --label NAME          Run label for output directory (default: auto)
    --no-update-yaml      Omit update.yaml from seed (Group A equivalent)
    --interval N          Screenshot interval in seconds (default: 2)
    --dry-run             Preview without deploying
    -h, --help            Show this help

${BOLD}OUTPUT${RESET}
    Screenshots and logs are saved to:
      observe-runs/<timestamp>_<label>/
        frame-001.png ... frame-NNN.png
        deploy.log

${BOLD}REQUIREMENTS${RESET}
    - proxmox.yaml with Proxmox connection settings
    - env file with PROXMOX_TOKEN_SECRET and PROXMOX_ROOT_PASSWORD
    - IncusOS ISO already uploaded to Proxmox (use incusos-proxmox --phase upload)
    - python3 with PIL/Pillow for PPM→PNG conversion
    - sshpass for root SSH to Proxmox host

EOF
            exit 0
            ;;
        *) error "Unknown option: $1"; exit 1 ;;
    esac
done

# ---------------------------------------------------------------------------
# Load environment
# ---------------------------------------------------------------------------

load_env() {
    local dir="$SCRIPT_DIR"
    while [[ "$dir" != "/" ]]; do
        if [[ -f "${dir}/env" ]]; then
            # shellcheck disable=SC1091
            source "${dir}/env"
            return
        fi
        dir="$(dirname "$dir")"
    done
}
load_env

if [[ -z "${PROXMOX_TOKEN_SECRET:-}" ]]; then
    error "PROXMOX_TOKEN_SECRET not set. Create an env file or export it."
    exit 1
fi

if [[ -z "${PROXMOX_ROOT_PASSWORD:-}" ]]; then
    error "PROXMOX_ROOT_PASSWORD not set. Add it to the env file."
    error "Required for SSH root screenshots (screendump)."
    exit 1
fi

# ---------------------------------------------------------------------------
# Load proxmox.yaml
# ---------------------------------------------------------------------------

yaml_get() {
    local file="$1" key="$2" default="${3:-}"
    local val
    val=$(awk -F': ' -v k="$key" '$1 == k {print $2; exit}' "$file" 2>/dev/null) || val=""
    val="${val#\"}" ; val="${val%\"}"
    val="${val#\'}" ; val="${val%\'}"
    echo "${val:-$default}"
}

load_proxmox_yaml() {
    local yaml_file=""
    if [[ -f "${SCRIPT_DIR}/proxmox.yaml" ]]; then
        yaml_file="${SCRIPT_DIR}/proxmox.yaml"
    elif [[ -f "proxmox.yaml" ]]; then
        yaml_file="proxmox.yaml"
    else
        error "proxmox.yaml not found"
        exit 1
    fi

    PVE_HOST=$(yaml_get "$yaml_file" "host")
    PVE_NODE=$(yaml_get "$yaml_file" "node" "pve")
    PVE_STORAGE=$(yaml_get "$yaml_file" "storage" "local-lvm")
    PVE_ISO_STORAGE=$(yaml_get "$yaml_file" "iso_storage" "local")
    PVE_BRIDGE=$(yaml_get "$yaml_file" "bridge" "vmbr0")
    PVE_POOL=$(yaml_get "$yaml_file" "pool")
    PVE_API_TOKEN_ID=$(yaml_get "$yaml_file" "api_token_id")

    if [[ -z "$PVE_HOST" ]]; then
        error "proxmox.yaml: 'host' is required"
        exit 1
    fi
}

# ---------------------------------------------------------------------------
# Proxmox API helpers
# ---------------------------------------------------------------------------

pve_api() {
    local method="$1"
    local endpoint="$2"
    local data="${3:-}"
    local content_type="${4:-form}"

    local url="https://${PVE_HOST}:8006/api2/json${endpoint}"
    local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}"

    local curl_args=(-fsSk -H "$auth")
    if [[ "$content_type" == "json" ]]; then
        curl_args+=(-H "Content-Type: application/json")
    fi

    case "$method" in
        GET)    curl "${curl_args[@]}" "$url" ;;
        POST)   curl "${curl_args[@]}" -X POST ${data:+-d "$data"} "$url" ;;
        PUT)    curl "${curl_args[@]}" -X PUT ${data:+-d "$data"} "$url" ;;
        DELETE) curl "${curl_args[@]}" -X DELETE "$url" ;;
    esac
}

pve_vm_exists() {
    local vmid="$1"
    pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" &>/dev/null
}

pve_vm_status() {
    local vmid="$1"
    pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" 2>/dev/null | \
        python3 -c "import json,sys; print(json.load(sys.stdin)['data']['status'])" 2>/dev/null
}

pve_start_vm() {
    local vmid="$1"
    pve_api POST "/nodes/${PVE_NODE}/qemu/${vmid}/status/start" &>/dev/null
}

pve_stop_vm() {
    local vmid="$1"
    pve_api POST "/nodes/${PVE_NODE}/qemu/${vmid}/status/stop" &>/dev/null
}

pve_destroy_vm() {
    local vmid="$1"
    local destroy_response
    destroy_response=$(pve_api DELETE "/nodes/${PVE_NODE}/qemu/${vmid}?purge=1&destroy-unreferenced-disks=1" 2>/dev/null) || return 1
    # Wait for async destroy task to complete (avoids VMID reuse issues)
    local upid
    upid=$(echo "$destroy_response" | python3 -c "import json,sys; print(json.load(sys.stdin).get('data',''))" 2>/dev/null) || upid=""
    if [[ -n "$upid" ]]; then
        local encoded_upid
        encoded_upid=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${upid}', safe=''))" 2>/dev/null)
        local wait_elapsed=0
        while [[ $wait_elapsed -lt 30 ]]; do
            sleep 2
            wait_elapsed=$((wait_elapsed + 2))
            local task_status
            task_status=$(pve_api GET "/nodes/${PVE_NODE}/tasks/${encoded_upid}/status" 2>/dev/null | \
                python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status',''))" 2>/dev/null) || task_status=""
            if [[ "$task_status" == "stopped" ]]; then
                break
            fi
        done
    fi
}

pve_vm_disk_writes() {
    local vmid="$1"
    pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/status/current" 2>/dev/null | \
        python3 -c "
import json,sys
d = json.load(sys.stdin).get('data',{})
bs = d.get('blockstat',{}).get('scsi0',{})
print(bs.get('wr_bytes',0))
" 2>/dev/null || echo "0"
}

pve_vm_mac() {
    local vmid="$1"
    pve_api GET "/nodes/${PVE_NODE}/qemu/${vmid}/config" 2>/dev/null | \
        python3 -c "
import json, sys, re
data = json.load(sys.stdin).get('data', {})
net0 = data.get('net0', '')
m = re.search(r'([0-9A-Fa-f:]{17})', net0)
print(m.group(1).lower() if m else '')
" 2>/dev/null
}

pve_vm_ip() {
    local vmid="$1"
    local mac
    mac=$(pve_vm_mac "$vmid") || return 1
    [[ -z "$mac" ]] && return 1

    ip neigh flush nud stale 2>/dev/null || true

    local bridge_subnet
    bridge_subnet=$(ip -4 addr show | awk '/inet / && !/127\.0\.0/ {print $2; exit}')

    if [[ -n "$bridge_subnet" ]]; then
        local broadcast
        broadcast=$(python3 -c "import ipaddress; print(ipaddress.ip_interface('${bridge_subnet}').network.broadcast_address)" 2>/dev/null) || true
        if [[ -n "$broadcast" ]]; then
            ping -c 2 -W 1 -b "$broadcast" &>/dev/null || true
            sleep 1
        fi
    fi

    local found_ip
    found_ip=$(ip neigh show | awk -v mac="$mac" 'tolower($0) ~ tolower(mac) && (/REACHABLE/ || /DELAY/ || /STALE/) {print $1; exit}') || true

    if [[ -n "$found_ip" ]] && ping -c 1 -W 1 "$found_ip" &>/dev/null; then
        echo "$found_ip"
        return 0
    fi

    # Parallel ping sweep fallback
    if [[ -n "$bridge_subnet" ]]; then
        local hosts
        hosts=$(python3 -c "
import ipaddress
net = ipaddress.ip_interface('${bridge_subnet}').network
for h in net.hosts():
    print(h)
" 2>/dev/null)
        if [[ -n "$hosts" ]]; then
            while IFS= read -r host_ip; do
                ping -c 1 -W 1 "$host_ip" &>/dev/null &
            done <<< "$hosts"
            wait
            sleep 1

            found_ip=$(ip neigh show | awk -v mac="$mac" 'tolower($0) ~ tolower(mac) && (/REACHABLE/ || /DELAY/) {print $1; exit}') || true

            if [[ -n "$found_ip" ]] && ping -c 1 -W 1 "$found_ip" &>/dev/null; then
                echo "$found_ip"
                return 0
            fi
        fi
    fi

    return 1
}

# ---------------------------------------------------------------------------
# Screenshot capture
# ---------------------------------------------------------------------------

# SSH options for root access to Proxmox (screenshots only)
SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -o LogLevel=ERROR)

capture_screenshot() {
    # Captures a single frame from the VM console.
    # Returns 0 on success, 1 on failure (VM not running, SSH error, etc.)
    local vmid="$1"
    local output_png="$2"

    local remote_ppm="/tmp/vm-${vmid}-screen.ppm"
    local local_ppm="/tmp/vm-${vmid}-screen.ppm"

    # Take screenshot via QEMU monitor (screendump)
    if ! sshpass -p "$PROXMOX_ROOT_PASSWORD" ssh "${SSH_OPTS[@]}" \
        "root@${PVE_HOST}" \
        "echo 'screendump ${remote_ppm}' | qm monitor ${vmid}" &>/dev/null; then
        return 1
    fi

    # Retrieve PPM file
    if ! sshpass -p "$PROXMOX_ROOT_PASSWORD" scp "${SSH_OPTS[@]}" \
        "root@${PVE_HOST}:${remote_ppm}" "$local_ppm" &>/dev/null; then
        return 1
    fi

    # Cleanup remote file
    sshpass -p "$PROXMOX_ROOT_PASSWORD" ssh "${SSH_OPTS[@]}" \
        "root@${PVE_HOST}" "rm -f ${remote_ppm}" &>/dev/null || true

    # Convert PPM → PNG
    if ! python3 -c "from PIL import Image; Image.open('${local_ppm}').save('${output_png}')" 2>/dev/null; then
        # If PIL not available, keep PPM
        mv "$local_ppm" "${output_png%.png}.ppm" 2>/dev/null || true
        return 0
    fi

    rm -f "$local_ppm"
    return 0
}

take_frame() {
    # Capture a numbered frame and log it
    FRAME_NUM=$((FRAME_NUM + 1))
    local frame_name
    frame_name=$(printf "frame-%03d.png" "$FRAME_NUM")
    local frame_path="${RUN_DIR}/${frame_name}"

    if capture_screenshot "$VMID" "$frame_path"; then
        log_quiet "Frame ${FRAME_NUM}: ${frame_name} (phase=${PHASE})"
        return 0
    else
        log_quiet "Frame ${FRAME_NUM}: FAILED (phase=${PHASE})"
        return 1
    fi
}

# ---------------------------------------------------------------------------
# Find IncusOS ISO on Proxmox
# ---------------------------------------------------------------------------

find_incusos_iso() {
    step "Finding IncusOS ISO on Proxmox"
    local iso_list
    iso_list=$(pve_api GET "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content?content=iso" 2>/dev/null) || iso_list=""

    ISO_FILENAME=$(echo "$iso_list" | python3 -c "
import json, sys
data = json.load(sys.stdin).get('data', [])
isos = [d['volid'].split('/')[-1] for d in data if 'IncusOS_' in d.get('volid', '')]
if isos:
    isos.sort()
    print(isos[-1])
" 2>/dev/null) || ISO_FILENAME=""

    if [[ -z "$ISO_FILENAME" ]]; then
        error "No IncusOS ISO found on Proxmox storage '${PVE_ISO_STORAGE}'"
        error "Deploy a lab first to upload the ISO, or use incusos-proxmox --phase upload"
        exit 1
    fi

    success "Using ISO: ${ISO_FILENAME}"
}

# ---------------------------------------------------------------------------
# Seed generation
# ---------------------------------------------------------------------------

SEED_ISO=""
SEED_TMPDIR=""

generate_seed() {
    step "Generating seed ISO"
    SEED_TMPDIR=$(mktemp -d)
    local seed_name="seed-${VM_NAME}.iso"
    SEED_ISO="$seed_name"
    local local_path="${SEED_TMPDIR}/${seed_name}"

    if [[ "$NO_UPDATE_YAML" == true ]]; then
        info "Omitting update.yaml from seed (Group A equivalent)"

        # Generate full seed as tar, extract, remove update.yaml, create ISO
        "$SEED_TOOL" --format tar --app incus --output "${SEED_TMPDIR}/seed.tar" \
            --force-install --force-reboot --hostname "$VM_NAME" \
            --defaults --quiet &>/dev/null

        local seeddir="${SEED_TMPDIR}/seedfiles"
        mkdir -p "$seeddir"
        tar xf "${SEED_TMPDIR}/seed.tar" -C "$seeddir"
        rm -f "${seeddir}/update.yaml"

        # Create ISO from modified seed directory
        if command -v genisoimage &>/dev/null; then
            genisoimage -V SEED_DATA -J -r -o "$local_path" "$seeddir" 2>/dev/null
        elif command -v mkisofs &>/dev/null; then
            mkisofs -V SEED_DATA -J -r -o "$local_path" "$seeddir" 2>/dev/null
        else
            error "genisoimage or mkisofs required for ISO creation"
            exit 1
        fi

        # Log seed contents
        log "Seed contents (no update.yaml):"
        for f in "$seeddir"/*.yaml; do
            [[ -f "$f" ]] && log "  $(basename "$f")"
        done
    else
        info "Standard seed (with update.yaml, check_frequency=6h)"

        "$SEED_TOOL" --format iso --app incus --output "$local_path" \
            --force-install --force-reboot --hostname "$VM_NAME" \
            --defaults --quiet &>/dev/null

        log "Seed: standard incusos-seed output"
    fi

    success "Seed ISO: ${seed_name}"

    # Upload to Proxmox
    step "Uploading seed ISO"
    local url="https://${PVE_HOST}:8006/api2/json/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/upload"
    local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}"

    curl -fsSk -H "$auth" \
        -F "content=iso" \
        -F "filename=@${local_path};filename=${seed_name}" \
        "$url" &>/dev/null

    success "Seed uploaded"
}

# ---------------------------------------------------------------------------
# VM lifecycle
# ---------------------------------------------------------------------------

create_vm() {
    step "Creating VM (VMID ${VMID}, name ${VM_NAME})"

    local json_payload
    json_payload=$(python3 -c "
import json
payload = {
    'vmid': ${VMID},
    'name': '${VM_NAME}',
    'description': '[incusos-lab:managed] observe-deploy',
    'ostype': 'l26',
    'bios': 'ovmf',
    'machine': 'q35',
    'efidisk0': '${PVE_STORAGE}:0,efitype=4m,pre-enrolled-keys=0',
    'tpmstate0': '${PVE_STORAGE}:1,version=v2.0',
    'cpu': 'host',
    'cores': 4,
    'memory': 4096,
    'balloon': 0,
    'scsihw': 'virtio-scsi-pci',
    'scsi0': '${PVE_STORAGE}:50,iothread=1',
    'ide2': '${PVE_ISO_STORAGE}:iso/${ISO_FILENAME},media=cdrom',
    'ide3': '${PVE_ISO_STORAGE}:iso/${SEED_ISO},media=cdrom',
    'net0': 'virtio,bridge=${PVE_BRIDGE}',
    'boot': 'order=ide2;scsi0',
    'agent': 1,
}
pool = '${PVE_POOL}'
if pool:
    payload['pool'] = pool
print(json.dumps(payload))
")

    local url="https://${PVE_HOST}:8006/api2/json/nodes/${PVE_NODE}/qemu"
    local auth="Authorization: PVEAPIToken=${PVE_API_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}"

    local create_response
    create_response=$(curl -fsSk -H "$auth" -H "Content-Type: application/json" \
        -X POST -d "$json_payload" "$url" 2>/dev/null) || {
        error "Failed to create VM"
        exit 1
    }

    # Wait for async task
    local upid
    upid=$(echo "$create_response" | python3 -c "import json,sys; print(json.load(sys.stdin).get('data',''))" 2>/dev/null) || upid=""
    if [[ -n "$upid" ]]; then
        local encoded_upid
        encoded_upid=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${upid}', safe=''))" 2>/dev/null)
        local wait_elapsed=0
        while [[ $wait_elapsed -lt 60 ]]; do
            sleep 2
            wait_elapsed=$((wait_elapsed + 2))
            local task_status
            task_status=$(pve_api GET "/nodes/${PVE_NODE}/tasks/${encoded_upid}/status" 2>/dev/null | \
                python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status',''))" 2>/dev/null) || task_status=""
            if [[ "$task_status" == "stopped" ]]; then
                break
            fi
        done
    fi

    success "VM created"
}

cleanup_vm() {
    step "Cleaning up VM ${VMID}"

    local status
    status=$(pve_vm_status "$VMID" 2>/dev/null) || status="unknown"
    if [[ "$status" == "running" ]]; then
        pve_stop_vm "$VMID" || true
        sleep 3
    fi

    pve_destroy_vm "$VMID" || true

    # Remove seed ISO from Proxmox storage
    pve_api DELETE "/nodes/${PVE_NODE}/storage/${PVE_ISO_STORAGE}/content/${PVE_ISO_STORAGE}:iso/${SEED_ISO}" &>/dev/null || true

    # Remove incus remote
    if command -v incus &>/dev/null; then
        local current_remote
        current_remote=$(incus remote get-default 2>/dev/null) || true
        if [[ "$current_remote" == "$VM_NAME" ]]; then
            incus remote switch local 2>/dev/null || true
        fi
        incus remote remove "$VM_NAME" 2>/dev/null || true
    fi

    # Cleanup temp files
    rm -rf "${SEED_TMPDIR:-}" 2>/dev/null || true
    rm -f "/tmp/vm-${VMID}-screen.ppm" 2>/dev/null || true

    success "Cleanup complete"
}

# ---------------------------------------------------------------------------
# Install monitoring (inline blockstat check)
# ---------------------------------------------------------------------------

check_install_progress() {
    # Called periodically during the ISO boot phase to detect when
    # installation completes. Updates PHASE when install is done.
    # Returns immediately — does not block.

    local wr_bytes
    wr_bytes=$(pve_vm_disk_writes "$VMID" 2>/dev/null) || wr_bytes=0

    if [[ $wr_bytes -gt $PREV_WR_BYTES ]]; then
        if [[ "$WRITES_STARTED" != true ]]; then
            WRITES_STARTED=true
            PHASE="installing"
            local wr_mb=$(( wr_bytes / 1048576 ))
            log "Install started — disk writes detected (${wr_mb} MiB)"
        fi
        STABLE_COUNT=0
    elif [[ "$WRITES_STARTED" == true ]]; then
        STABLE_COUNT=$((STABLE_COUNT + 1))
        if [[ $STABLE_COUNT -ge $STABLE_THRESHOLD ]]; then
            local wr_mb=$(( wr_bytes / 1048576 ))
            log "Install complete — disk activity stopped (${wr_mb} MiB total, ${STABLE_COUNT} stable polls)"
            PHASE="install_done"
        fi
    fi

    PREV_WR_BYTES=$wr_bytes
}

# ---------------------------------------------------------------------------
# Main deploy loop
# ---------------------------------------------------------------------------

run_deploy() {
    local deploy_start
    deploy_start=$(date +%s)

    # --- Phase 1: Boot from ISO ---
    step "Starting VM (ISO boot + screenshot capture)"
    pve_start_vm "$VMID"
    PHASE="iso_boot"
    log "VM started — phase: iso_boot"

    local elapsed=0
    local last_blockstat_check=0

    # Screenshot + monitor loop during ISO boot and installation
    while [[ "$PHASE" == "iso_boot" ]] || [[ "$PHASE" == "installing" ]]; do
        take_frame || true

        # Check blockstat every INSTALL_POLL seconds (not every screenshot)
        local now
        now=$(date +%s)
        if [[ $((now - last_blockstat_check)) -ge $INSTALL_POLL ]]; then
            check_install_progress
            last_blockstat_check=$now
        fi

        sleep "$SCREENSHOT_INTERVAL"
        elapsed=$((elapsed + SCREENSHOT_INTERVAL))

        # Check if VM stopped on its own (force_reboot caused guest reboot,
        # but QEMU uptime doesn't reset — VM stays "running". If it's
        # actually stopped, something went wrong.)
        local status
        status=$(pve_vm_status "$VMID" 2>/dev/null) || status="unknown"
        if [[ "$status" == "stopped" ]]; then
            log "VM stopped unexpectedly during install"
            PHASE="install_done"
        fi

        if [[ $elapsed -ge $INSTALL_TIMEOUT ]]; then
            log "Install timeout after ${INSTALL_TIMEOUT}s"
            warn "Installation did not complete within ${INSTALL_TIMEOUT}s"

            # Take a final frame for diagnosis
            take_frame || true
            PHASE="failed"
            break
        fi

        # Progress logging every 30s
        if [[ $((elapsed % 30)) -lt $SCREENSHOT_INTERVAL ]]; then
            local wr_mb=$(( PREV_WR_BYTES / 1048576 ))
            log "Progress: ${elapsed}s elapsed, ${wr_mb} MiB written, phase=${PHASE}, frame=${FRAME_NUM}"
        fi
    done

    if [[ "$PHASE" == "failed" ]]; then
        return 1
    fi

    # --- Phase 2: Transition to disk boot ---
    PHASE="transitioning"
    log "Transitioning: stop VM, remove install media, start from disk"

    # Take a few more frames during transition
    take_frame || true

    # Stop the VM
    step "Stopping VM for media removal"
    pve_stop_vm "$VMID" || true

    # Wait for VM to actually stop
    local stop_wait=0
    while [[ $stop_wait -lt 30 ]]; do
        local status
        status=$(pve_vm_status "$VMID" 2>/dev/null) || status="unknown"
        if [[ "$status" == "stopped" ]]; then
            break
        fi
        sleep 2
        stop_wait=$((stop_wait + 2))
    done
    log "VM stopped after ${stop_wait}s"

    # Remove install media and set boot order
    step "Removing install media (ide2, ide3) and setting boot to scsi0"
    pve_api PUT "/nodes/${PVE_NODE}/qemu/${VMID}/config" \
        '{"delete":"ide2,ide3","boot":"order=scsi0"}' json &>/dev/null || {
        error "Failed to remove install media"
        PHASE="failed"
        return 1
    }
    log "Install media removed, boot order set to scsi0"

    # --- Phase 3: Boot from disk ---
    step "Starting VM from disk (first boot)"
    pve_start_vm "$VMID"
    PHASE="disk_boot"
    log "VM started from disk — phase: disk_boot"

    local boot_elapsed=0
    local ip_detected=""
    local ip_detect_started=false

    # Screenshot loop during first boot
    while [[ "$PHASE" == "disk_boot" ]]; do
        take_frame || true

        sleep "$SCREENSHOT_INTERVAL"
        boot_elapsed=$((boot_elapsed + SCREENSHOT_INTERVAL))

        # After 30s, start trying to detect IP
        if [[ $boot_elapsed -ge 30 ]] && [[ "$ip_detect_started" != true ]]; then
            ip_detect_started=true
            log "Starting IP detection (${boot_elapsed}s into disk boot)"
        fi

        # Try IP detection every 10s after the initial 30s wait
        if [[ "$ip_detect_started" == true ]] && [[ $((boot_elapsed % 10)) -lt $SCREENSHOT_INTERVAL ]]; then
            local ip
            ip=$(pve_vm_ip "$VMID" 2>/dev/null) || ip=""
            if [[ -n "$ip" ]]; then
                if [[ -z "$ip_detected" ]]; then
                    ip_detected="$ip"
                    log "IP detected: ${ip} (${boot_elapsed}s into disk boot)"
                fi

                # Check port 8443
                if curl -sk --connect-timeout 3 "https://${ip}:8443" &>/dev/null; then
                    log "Port 8443 responding at ${ip} (${boot_elapsed}s into disk boot)"
                    PHASE="ready"
                    break
                else
                    log_quiet "Port 8443 not yet responding at ${ip}"
                fi
            fi
        fi

        if [[ $boot_elapsed -ge $BOOT_TIMEOUT ]]; then
            log "Boot timeout after ${BOOT_TIMEOUT}s"
            warn "Port 8443 not reachable after ${BOOT_TIMEOUT}s"

            # Take final diagnostic frames
            take_frame || true
            PHASE="failed"
            break
        fi

        # Progress logging every 15s
        if [[ $((boot_elapsed % 15)) -lt $SCREENSHOT_INTERVAL ]]; then
            log "Disk boot: ${boot_elapsed}s elapsed, ip=${ip_detected:-none}, phase=${PHASE}, frame=${FRAME_NUM}"
        fi
    done

    # Take a few final frames after reaching ready state
    if [[ "$PHASE" == "ready" ]]; then
        for i in 1 2 3; do
            sleep "$SCREENSHOT_INTERVAL"
            take_frame || true
        done
    fi

    local deploy_end
    deploy_end=$(date +%s)
    local total_seconds=$((deploy_end - deploy_start))

    log "Deploy finished: phase=${PHASE}, total_time=${total_seconds}s, frames=${FRAME_NUM}"

    if [[ "$PHASE" == "ready" ]]; then
        return 0
    else
        return 1
    fi
}

# ---------------------------------------------------------------------------
# Preflight checks
# ---------------------------------------------------------------------------

preflight() {
    step "Preflight checks"

    # Check required tools
    local missing=()
    command -v sshpass &>/dev/null || missing+=("sshpass")
    command -v python3 &>/dev/null || missing+=("python3")
    command -v curl &>/dev/null || missing+=("curl")

    if [[ ${#missing[@]} -gt 0 ]]; then
        error "Missing required tools: ${missing[*]}"
        exit 1
    fi

    # Check PIL availability
    if ! python3 -c "from PIL import Image" &>/dev/null; then
        warn "python3 PIL/Pillow not available — screenshots will be saved as PPM (larger files)"
    fi

    # Check SSH connectivity
    if ! sshpass -p "$PROXMOX_ROOT_PASSWORD" ssh "${SSH_OPTS[@]}" \
        "root@${PVE_HOST}" "echo ok" &>/dev/null; then
        error "Cannot SSH to root@${PVE_HOST}"
        error "Check PROXMOX_ROOT_PASSWORD in env file and SSH access"
        exit 1
    fi
    success "SSH to root@${PVE_HOST}: OK"

    # Check VMID range (must be 850-869 per CLAUDE.md)
    if [[ $VMID -lt 850 ]] || [[ $VMID -gt 869 ]]; then
        error "VMID ${VMID} outside allowed test range (850-869)"
        exit 1
    fi

    # Check VMID not in use
    if pve_vm_exists "$VMID"; then
        error "VMID ${VMID} already exists on Proxmox"
        error "Clean it up first or use --vmid to pick a different one"
        exit 1
    fi
    success "VMID ${VMID} available"
}

# ---------------------------------------------------------------------------
# Setup output directory
# ---------------------------------------------------------------------------

setup_output() {
    local timestamp
    timestamp=$(date '+%Y-%m-%d_%H-%M-%S')

    if [[ -z "$LABEL" ]]; then
        if [[ "$NO_UPDATE_YAML" == true ]]; then
            LABEL="no-update-yaml"
        else
            LABEL="standard"
        fi
    fi

    RUN_DIR="${SCRIPT_DIR}/observe-runs/${timestamp}_${LABEL}"
    mkdir -p "$RUN_DIR"

    LOG_FILE="${RUN_DIR}/deploy.log"
    touch "$LOG_FILE"

    success "Output directory: ${RUN_DIR}"
}

# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

echo -e "${BOLD}${SCRIPT_NAME}${RESET} v${VERSION}"
echo -e "Observational single-VM deploy with rapid console screenshots"
echo ""

if [[ "$DRY_RUN" == true ]]; then
    echo -e "${YELLOW}[dry-run] Preview mode — no VMs will be created${RESET}"
    echo ""
    echo "Configuration:"
    echo "  VMID:              ${VMID}"
    echo "  VM name:           ${VM_NAME}"
    echo "  Update.yaml:       $(if [[ "$NO_UPDATE_YAML" == true ]]; then echo "OMITTED"; else echo "included (6h)"; fi)"
    echo "  Screenshot interval: ${SCREENSHOT_INTERVAL}s"
    echo ""

    step "Would load proxmox.yaml"
    load_proxmox_yaml
    detail "Host: ${PVE_HOST}, Node: ${PVE_NODE}, Storage: ${PVE_STORAGE}"
    detail "Pool: ${PVE_POOL}, Token: ${PVE_API_TOKEN_ID}"

    echo ""
    step "Would check:"
    detail "- SSH to root@${PVE_HOST} for screenshots"
    detail "- VMID ${VMID} availability"
    detail "- IncusOS ISO on Proxmox"
    detail "- PIL/Pillow for PPM→PNG conversion"

    echo ""
    step "Would execute:"
    detail "1. Generate seed ISO ($(if [[ "$NO_UPDATE_YAML" == true ]]; then echo "no update.yaml"; else echo "standard"; fi))"
    detail "2. Upload seed to Proxmox"
    detail "3. Create VM ${VMID} (${VM_NAME})"
    detail "4. Start VM, begin screenshot loop (every ${SCREENSHOT_INTERVAL}s)"
    detail "5. Monitor blockstat for install completion"
    detail "6. Stop VM, remove media, start from disk"
    detail "7. Continue screenshots until port 8443 responds (or ${BOOT_TIMEOUT}s timeout)"
    detail "8. Save frames + deploy.log"
    detail "9. Cleanup VM"

    echo ""
    info "Dry run complete"
    exit 0
fi

# Load configuration
step "Loading configuration"
load_proxmox_yaml
detail "Host: ${PVE_HOST}, Node: ${PVE_NODE}, Storage: ${PVE_STORAGE}"

# Run preflight checks
preflight

# Find IncusOS ISO
find_incusos_iso

# Setup output directory
setup_output

# Log configuration
log "=== observe-deploy v${VERSION} ==="
log "VMID: ${VMID}"
log "VM name: ${VM_NAME}"
log "ISO: ${ISO_FILENAME}"
log "Update.yaml: $(if [[ "$NO_UPDATE_YAML" == true ]]; then echo "OMITTED"; else echo "included (6h)"; fi)"
log "Screenshot interval: ${SCREENSHOT_INTERVAL}s"
log "Install timeout: ${INSTALL_TIMEOUT}s"
log "Boot timeout: ${BOOT_TIMEOUT}s"

# Generate and upload seed
generate_seed

# Create VM
create_vm

echo ""
echo -e "${BOLD}Starting observational deploy${RESET}"
echo "  Frames saved to: ${RUN_DIR}/"
echo "  Log: ${RUN_DIR}/deploy.log"
echo "  Press Ctrl+C to abort (VM will be cleaned up)"
echo ""

# Trap for clean exit
trap_cleanup() {
    echo ""
    warn "Interrupted — cleaning up"
    cleanup_vm
    exit 130
}
trap trap_cleanup INT TERM

# Run the deploy with screenshots
deploy_result=0
run_deploy || deploy_result=$?

echo ""
echo -e "${BOLD}========================================${RESET}"
echo -e "${BOLD}  Deploy Results${RESET}"
echo -e "${BOLD}========================================${RESET}"
echo ""
echo "  VMID:       ${VMID}"
echo "  Seed:       $(if [[ "$NO_UPDATE_YAML" == true ]]; then echo "no update.yaml"; else echo "standard (check_frequency=6h)"; fi)"
echo "  Outcome:    $(if [[ $deploy_result -eq 0 ]]; then echo -e "${GREEN}PASS${RESET} — port 8443 reachable"; else echo -e "${RED}FAIL${RESET} — phase: ${PHASE}"; fi)"
echo "  Frames:     ${FRAME_NUM}"
echo "  Output:     ${RUN_DIR}/"
echo ""

# Cleanup
step "Post-deploy cleanup"
cleanup_vm

if [[ $deploy_result -eq 0 ]]; then
    success "Observational deploy complete — ${FRAME_NUM} frames captured"
else
    warn "Deploy failed at phase: ${PHASE}"
    warn "Check screenshots and log for diagnosis:"
    warn "  ${RUN_DIR}/deploy.log"
    warn "  ${RUN_DIR}/frame-*.png"
fi

echo ""
echo "To view key frames:"
echo "  ls ${RUN_DIR}/frame-*.png"
echo "  # View in image viewer or analyze with Claude Code"
echo ""

exit $deploy_result