#!/usr/bin/env bash
# deploy-awx - Deploy and manage AWX on an Incus cluster
#
# Deploys a Debian 12 VM with K3s + AWX Operator, configures AWX for
# Aether integration (project, inventory, job templates), and provides
# ongoing health checks and healing.
#
# Usage: deploy-awx [OPTIONS]
# Run 'deploy-awx --help' for full usage information.

set -euo pipefail

readonly VERSION="0.1.0"
readonly SCRIPT_NAME="deploy-awx"
readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly MANIFESTS_DIR="${SCRIPT_DIR}/awx-manifests"
readonly AWX_NAMESPACE="awx"

# ---------------------------------------------------------------------------
# Defaults (overridden by config file or CLI flags)
# ---------------------------------------------------------------------------

VM_NAME="awx"
TARGET_REMOTE="oc-node-02"
AWX_IP="192.168.102.161"
AWX_PREFIX="22"
AWX_GATEWAY="192.168.100.1"
AWX_DNS="192.168.100.1"
AWX_CPU="4"
AWX_MEMORY="8GiB"
AWX_DISK="40GiB"
GIT_REPO="ssh://git@192.168.1.200:2222/maarten/incus-contrib.git"
GIT_BRANCH="master"
PLAYBOOK_DIR="playbooks"
AETHER_URL="https://192.168.102.160:8443"
AETHER_CLUSTER_ID="52"

# Runtime flags
DRY_RUN=false
VERBOSE=false
QUIET=false
CONFIG_FILE=""
ACTION=""

# ---------------------------------------------------------------------------
# Color and formatting
# ---------------------------------------------------------------------------

setup_colors() {
    if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then
        RED='\033[0;31m'
        GREEN='\033[0;32m'
        YELLOW='\033[0;33m'
        BLUE='\033[0;34m'
        CYAN='\033[0;36m'
        BOLD='\033[1m'
        DIM='\033[2m'
        RESET='\033[0m'
    else
        RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' RESET=''
    fi
}

info()    { [[ "$QUIET" == true ]] && return; echo -e "${BLUE}[info]${RESET}    $*"; }
success() { [[ "$QUIET" == true ]] && return; echo -e "${GREEN}[ok]${RESET}      $*"; }
warn()    { echo -e "${YELLOW}[warn]${RESET}    $*" >&2; }
error()   { echo -e "${RED}[error]${RESET}   $*" >&2; }
step()    { [[ "$QUIET" == true ]] && return; echo -e "${CYAN}[step]${RESET}    ${BOLD}$*${RESET}"; }
detail()  { [[ "$VERBOSE" != true ]] && return; echo -e "${DIM}          $*${RESET}"; }

dry_run_guard() {
    if [[ "$DRY_RUN" == true ]]; then
        info "(dry-run) $*"
        return 1
    fi
    return 0
}

# ---------------------------------------------------------------------------
# Help
# ---------------------------------------------------------------------------

usage() {
    cat <<EOF
${BOLD}${SCRIPT_NAME}${RESET} v${VERSION} - Deploy and manage AWX on an Incus cluster

${BOLD}USAGE${RESET}
    ${SCRIPT_NAME} [OPTIONS]

${BOLD}ACTIONS${RESET}
    --deploy          Full deploy: create VM, install K3s, deploy AWX, configure
    --status          Check AWX health, K3s pods, Aether connection
    --heal            Restart failed pods, re-sync project, re-validate templates
    --configure       (Re-)configure AWX: project, inventory, templates
    --join-aether     Register AWX endpoint in Aether + bind to cluster
    --cleanup         Destroy the AWX VM
    --doctor          Check prerequisites (incus CLI, curl, python3, kubectl)

${BOLD}OPTIONS${RESET}
    -c, --config FILE     Configuration file (YAML with awx: section)
    -n, --dry-run         Preview actions without executing
    -v, --verbose         Show detailed output
    -q, --quiet           Suppress informational output
    -h, --help            Show this help message

${BOLD}CONFIGURATION${RESET}
    The script uses built-in defaults for the lab environment. Override with
    a YAML config file containing an 'awx:' section:

        awx:
          vm_name: awx
          target_node: oc-node-02
          ip: 192.168.102.161/22
          gateway: 192.168.100.1
          dns: 192.168.100.1
          cpu: 4
          memory: 8GiB
          disk: 40GiB
          git_repo: ssh://git@192.168.1.200:2222/maarten/incus-contrib.git
          git_branch: master
          aether_url: https://192.168.102.160:8443
          aether_cluster_id: 52

${BOLD}EXAMPLES${RESET}
    # Check prerequisites
    ${SCRIPT_NAME} --doctor

    # Full deployment with defaults
    ${SCRIPT_NAME} --deploy

    # Preview deployment without executing
    ${SCRIPT_NAME} --deploy --dry-run

    # Check AWX health
    ${SCRIPT_NAME} --status

    # Heal broken pods
    ${SCRIPT_NAME} --heal

    # Reconfigure AWX (project, templates)
    ${SCRIPT_NAME} --configure

    # Register with Aether
    ${SCRIPT_NAME} --join-aether

    # Destroy AWX VM
    ${SCRIPT_NAME} --cleanup
EOF
}

# ---------------------------------------------------------------------------
# Config parsing (simple YAML — same pattern as other scripts)
# ---------------------------------------------------------------------------

parse_config() {
    local file="$1"
    if [[ ! -f "$file" ]]; then
        error "Config file not found: $file"
        exit 1
    fi

    # Extract awx: section values using grep + sed (no yq dependency)
    local val
    val=$(grep -A50 '^awx:' "$file" | grep '^ *vm_name:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
    [[ -n "$val" ]] && VM_NAME="$val"
    val=$(grep -A50 '^awx:' "$file" | grep '^ *target_node:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
    [[ -n "$val" ]] && TARGET_REMOTE="$val"
    val=$(grep -A50 '^awx:' "$file" | grep '^ *ip:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
    if [[ -n "$val" ]]; then
        AWX_IP="${val%%/*}"
        [[ "$val" == */* ]] && AWX_PREFIX="${val#*/}"
    fi
    val=$(grep -A50 '^awx:' "$file" | grep '^ *gateway:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
    [[ -n "$val" ]] && AWX_GATEWAY="$val"
    val=$(grep -A50 '^awx:' "$file" | grep '^ *dns:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
    [[ -n "$val" ]] && AWX_DNS="$val"
    val=$(grep -A50 '^awx:' "$file" | grep '^ *cpu:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
    [[ -n "$val" ]] && AWX_CPU="$val"
    val=$(grep -A50 '^awx:' "$file" | grep '^ *memory:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
    [[ -n "$val" ]] && AWX_MEMORY="$val"
    val=$(grep -A50 '^awx:' "$file" | grep '^ *disk:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
    [[ -n "$val" ]] && AWX_DISK="$val"
    val=$(grep -A50 '^awx:' "$file" | grep '^ *git_repo:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
    [[ -n "$val" ]] && GIT_REPO="$val"
    val=$(grep -A50 '^awx:' "$file" | grep '^ *git_branch:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
    [[ -n "$val" ]] && GIT_BRANCH="$val"
    val=$(grep -A50 '^awx:' "$file" | grep '^ *aether_url:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
    [[ -n "$val" ]] && AETHER_URL="$val"
    val=$(grep -A50 '^awx:' "$file" | grep '^ *aether_cluster_id:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
    [[ -n "$val" ]] && AETHER_CLUSTER_ID="$val"
}

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

vm_ref() {
    echo "${TARGET_REMOTE}:${VM_NAME}"
}

vm_exists() {
    incus info "$(vm_ref)" &>/dev/null 2>&1
}

vm_running() {
    local state
    state=$(incus info "$(vm_ref)" 2>/dev/null | grep -i '^status:' | awk '{print $2}')
    [[ "$state" == "RUNNING" || "$state" == "Running" ]]
}

vm_exec() {
    incus exec "$(vm_ref)" -- "$@"
}

awx_url() {
    # AWX is exposed via K3s NodePort on port 30080 (HTTP)
    # Traefik ingress returns 404 for IP-based access
    echo "http://${AWX_IP}:30080"
}

awx_api() {
    local method="$1" path="$2"
    shift 2
    curl -sk -X "$method" "$(awx_url)${path}" \
        -H "Content-Type: application/json" \
        "$@"
}

awx_api_auth() {
    local method="$1" path="$2"
    shift 2
    local token
    token=$(get_awx_admin_password)
    curl -sk -X "$method" "$(awx_url)${path}" \
        --user "admin:${token}" \
        -H "Content-Type: application/json" \
        "$@"
}

get_awx_admin_password() {
    vm_exec kubectl -n "$AWX_NAMESPACE" get secret awx-admin-password \
        -o jsonpath='{.data.password}' 2>/dev/null | base64 -d
}

wait_for_vm_agent() {
    local timeout="${1:-120}"
    local elapsed=0
    info "Waiting for VM agent (up to ${timeout}s)..."
    while ! vm_exec true &>/dev/null 2>&1; do
        sleep 5
        elapsed=$((elapsed + 5))
        if [[ $elapsed -ge $timeout ]]; then
            error "VM agent not available after ${timeout}s"
            return 1
        fi
    done
    success "VM agent connected (${elapsed}s)"
}

wait_for_awx_web() {
    local timeout="${1:-600}"
    local elapsed=0
    info "Waiting for AWX web interface (up to ${timeout}s)..."
    while ! curl -sk -o /dev/null -w '%{http_code}' "$(awx_url)/api/v2/ping/" 2>/dev/null | grep -q '200'; do
        sleep 10
        elapsed=$((elapsed + 10))
        if [[ $elapsed -ge $timeout ]]; then
            error "AWX web not responding after ${timeout}s"
            return 1
        fi
        [[ "$VERBOSE" == true ]] && detail "Still waiting... (${elapsed}s)"
    done
    success "AWX web interface ready (${elapsed}s)"
}

# ---------------------------------------------------------------------------
# Doctor — prerequisite checks
# ---------------------------------------------------------------------------

action_doctor() {
    step "Checking prerequisites"
    local ok=true

    # incus CLI
    if command -v incus &>/dev/null; then
        success "incus CLI: $(incus version 2>/dev/null || echo 'installed')"
    else
        error "incus CLI not found — install the Incus client"
        ok=false
    fi

    # curl
    if command -v curl &>/dev/null; then
        success "curl: $(curl --version | head -1 | awk '{print $2}')"
    else
        error "curl not found"
        ok=false
    fi

    # python3
    if command -v python3 &>/dev/null; then
        success "python3: $(python3 --version 2>&1 | awk '{print $2}')"
    else
        error "python3 not found"
        ok=false
    fi

    # base64
    if command -v base64 &>/dev/null; then
        success "base64: available"
    else
        error "base64 not found"
        ok=false
    fi

    # Manifest files
    if [[ -d "$MANIFESTS_DIR" ]]; then
        success "AWX manifests: $MANIFESTS_DIR"
    else
        error "AWX manifests not found at $MANIFESTS_DIR"
        ok=false
    fi

    # Target remote
    if incus remote list -f csv 2>/dev/null | grep -q "^${TARGET_REMOTE},"; then
        success "Incus remote '${TARGET_REMOTE}': reachable"
    else
        warn "Incus remote '${TARGET_REMOTE}' not found (needed for deployment)"
    fi

    # Existing VM
    if vm_exists; then
        info "AWX VM '$(vm_ref)' exists"
        if vm_running; then
            success "  VM is running"
            # Check K3s
            if vm_exec kubectl get nodes &>/dev/null 2>&1; then
                success "  K3s is responding"
            else
                warn "  K3s not responding"
            fi
            # Check AWX pods
            local pod_count
            pod_count=$(vm_exec kubectl -n "$AWX_NAMESPACE" get pods --no-headers 2>/dev/null | wc -l || echo 0)
            info "  AWX pods: ${pod_count}"
        else
            warn "  VM is not running"
        fi
    else
        info "AWX VM '$(vm_ref)' does not exist (--deploy to create)"
    fi

    echo
    if [[ "$ok" == true ]]; then
        success "All prerequisites satisfied"
    else
        error "Some prerequisites missing — fix the issues above"
        return 1
    fi
}

# ---------------------------------------------------------------------------
# Deploy — full deployment pipeline
# ---------------------------------------------------------------------------

action_deploy() {
    step "Deploying AWX on $(vm_ref)"
    echo
    info "  Target:  ${TARGET_REMOTE}"
    info "  VM:      ${VM_NAME}"
    info "  IP:      ${AWX_IP}/${AWX_PREFIX}"
    info "  CPU:     ${AWX_CPU}"
    info "  Memory:  ${AWX_MEMORY}"
    info "  Disk:    ${AWX_DISK}"
    echo

    if vm_exists; then
        if vm_running; then
            error "AWX VM already exists and is running. Use --status, --heal, or --cleanup first."
            return 1
        else
            warn "AWX VM exists but is not running — starting it"
            if dry_run_guard "incus start $(vm_ref)"; then
                incus start "$(vm_ref)"
            fi
            return 0
        fi
    fi

    phase_create_vm
    phase_configure_network
    phase_install_k3s
    phase_deploy_awx_operator
    phase_deploy_awx_instance
    phase_verify

    echo
    success "AWX deployment complete!"
    info "  Web UI:   $(awx_url)/"
    info "  API:      $(awx_url)/api/v2/ping/"
    local pw
    pw=$(get_awx_admin_password 2>/dev/null || echo "<retrieve with --status>")
    info "  Admin:    admin / ${pw}"
    echo
    info "Next steps:"
    info "  ${SCRIPT_NAME} --configure    # set up project, inventory, templates"
    info "  ${SCRIPT_NAME} --join-aether  # register with Aether"
}

phase_create_vm() {
    step "Phase 1: Create VM"

    if ! dry_run_guard "incus launch images:debian/12 $(vm_ref) --vm --target ${TARGET_REMOTE} ..."; then
        return 0
    fi

    incus launch images:debian/12 "$(vm_ref)" --vm \
        --target "$TARGET_REMOTE" \
        -c "limits.cpu=${AWX_CPU}" \
        -c "limits.memory=${AWX_MEMORY}" \
        -d "root,size=${AWX_DISK}"

    success "VM created"
    detail "Waiting 10s for initial boot..."
    sleep 10

    # Switch to macvlan for direct VLAN access
    info "Configuring macvlan NIC (parent=mgmt)"
    incus config device remove "$(vm_ref)" eth0 2>/dev/null || true
    incus config device add "$(vm_ref)" eth0 nic \
        nictype=macvlan parent=mgmt

    success "NIC switched to macvlan — VM will restart networking"
}

phase_configure_network() {
    step "Phase 2: Configure static IP + hostname"

    wait_for_vm_agent 120

    if ! dry_run_guard "Configure networking on $(vm_ref)"; then
        return 0
    fi

    # Debian 12 images use systemd-networkd (not netplan)
    # The macvlan NIC appears as enp5s0 in Incus VMs
    vm_exec bash -c "
        hostnamectl set-hostname '${VM_NAME}'

        # Disable any existing DHCP config
        rm -f /etc/network/interfaces.d/* 2>/dev/null || true

        # Configure static IP via systemd-networkd
        mkdir -p /etc/systemd/network
        cat > /etc/systemd/network/10-static.network << 'NETWORKD'
[Match]
Name=enp5s0

[Network]
Address=${AWX_IP}/${AWX_PREFIX}
Gateway=${AWX_GATEWAY}
DNS=${AWX_DNS}
NETWORKD

        systemctl enable systemd-networkd
        systemctl restart systemd-networkd
    "

    success "Static IP ${AWX_IP}/${AWX_PREFIX} configured"

    # Verify connectivity
    detail "Verifying network connectivity..."
    sleep 3
    if vm_exec ping -c 1 -W 3 "${AWX_GATEWAY}" &>/dev/null; then
        success "Gateway reachable from VM"
    else
        warn "Cannot reach gateway — check VLAN/network config"
    fi
}

phase_install_k3s() {
    step "Phase 3: Install K3s"

    if ! dry_run_guard "Install K3s on $(vm_ref)"; then
        return 0
    fi

    # Ensure curl is available (minimal Debian 12 may not have it)
    info "Installing prerequisites..."
    vm_exec bash -c "
        export DEBIAN_FRONTEND=noninteractive
        apt-get update -qq
        apt-get install -y -qq curl python3 2>&1 | tail -1
    "

    vm_exec bash -c "
        curl -sfL https://get.k3s.io | sh -s - --write-kubeconfig-mode 644
    "
    success "K3s installed"

    info "Waiting for K3s node to be ready..."
    vm_exec kubectl wait --for=condition=Ready node --all --timeout=120s
    success "K3s node ready"
}

phase_deploy_awx_operator() {
    step "Phase 4: Deploy AWX Operator"

    if ! dry_run_guard "Deploy AWX Operator on $(vm_ref)"; then
        return 0
    fi

    # Create namespace
    vm_exec kubectl create namespace "$AWX_NAMESPACE" 2>/dev/null || true

    # Push kustomization files
    info "Pushing operator manifests..."
    incus file push "${MANIFESTS_DIR}/operator/kustomization.yaml" \
        "$(vm_ref)/tmp/awx-operator-kustomization.yaml"

    vm_exec bash -c "
        mkdir -p /opt/awx/operator
        cp /tmp/awx-operator-kustomization.yaml /opt/awx/operator/kustomization.yaml
        kubectl apply -k /opt/awx/operator/
    "

    info "Waiting for AWX Operator (up to 5 min)..."
    vm_exec kubectl -n "$AWX_NAMESPACE" wait \
        --for=condition=Available \
        deployment/awx-operator-controller-manager \
        --timeout=300s

    success "AWX Operator deployed"
}

phase_deploy_awx_instance() {
    step "Phase 5: Deploy AWX instance"

    if ! dry_run_guard "Deploy AWX instance on $(vm_ref)"; then
        return 0
    fi

    # Push AWX CR manifests
    info "Pushing AWX instance manifests..."
    incus file push "${MANIFESTS_DIR}/base/awx.yaml" \
        "$(vm_ref)/tmp/awx-cr.yaml"
    incus file push "${MANIFESTS_DIR}/base/kustomization.yaml" \
        "$(vm_ref)/tmp/awx-kustomization.yaml"

    vm_exec bash -c "
        mkdir -p /opt/awx/base
        cp /tmp/awx-cr.yaml /opt/awx/base/awx.yaml
        cp /tmp/awx-kustomization.yaml /opt/awx/base/kustomization.yaml
        kubectl apply -k /opt/awx/base/
    "

    info "AWX instance deploying — this takes 5-10 minutes..."
    info "Waiting for AWX pods to start..."

    # Wait for AWX deployment to exist and become available
    local elapsed=0
    local timeout=600
    while ! vm_exec kubectl -n "$AWX_NAMESPACE" get deployment awx-web &>/dev/null 2>&1; do
        sleep 15
        elapsed=$((elapsed + 15))
        if [[ $elapsed -ge $timeout ]]; then
            error "AWX web deployment not created after ${timeout}s"
            warn "Check pod status: incus exec $(vm_ref) -- kubectl -n awx get pods"
            return 1
        fi
        [[ "$VERBOSE" == true ]] && detail "Waiting for AWX web deployment... (${elapsed}s)"
    done

    vm_exec kubectl -n "$AWX_NAMESPACE" wait \
        --for=condition=Available \
        deployment/awx-web \
        --timeout=300s || {
            warn "AWX web deployment not fully available yet — checking pods"
            vm_exec kubectl -n "$AWX_NAMESPACE" get pods
        }

    success "AWX instance deployed"
}

phase_verify() {
    step "Phase 6: Verify AWX"

    wait_for_awx_web 300

    local admin_pw
    admin_pw=$(get_awx_admin_password)
    success "Admin password retrieved"

    # Test API ping (NodePort 30080)
    local ping_result
    ping_result=$(curl -sk "$(awx_url)/api/v2/ping/" 2>/dev/null)
    if echo "$ping_result" | python3 -c "import sys,json; json.load(sys.stdin)" &>/dev/null; then
        success "AWX API responding"
    else
        warn "AWX API returned unexpected response"
    fi
}

# ---------------------------------------------------------------------------
# Status — health checks
# ---------------------------------------------------------------------------

action_status() {
    step "AWX Status for $(vm_ref)"
    echo

    # VM existence and state
    if ! vm_exists; then
        error "AWX VM does not exist — use --deploy to create"
        return 1
    fi

    if ! vm_running; then
        error "AWX VM exists but is not running"
        return 1
    fi
    success "VM: running"

    # K3s
    if vm_exec kubectl get nodes &>/dev/null 2>&1; then
        success "K3s: healthy"
        if [[ "$VERBOSE" == true ]]; then
            vm_exec kubectl get nodes -o wide
        fi
    else
        error "K3s: not responding"
        return 1
    fi

    # AWX pods
    echo
    info "AWX pods:"
    vm_exec kubectl -n "$AWX_NAMESPACE" get pods -o wide 2>/dev/null || warn "Cannot list pods"

    # Pod health summary
    local total running
    total=$(vm_exec kubectl -n "$AWX_NAMESPACE" get pods --no-headers 2>/dev/null | wc -l || echo 0)
    running=$(vm_exec kubectl -n "$AWX_NAMESPACE" get pods --no-headers 2>/dev/null | grep -c 'Running' || echo 0)
    echo
    if [[ "$total" -gt 0 && "$running" -eq "$total" ]]; then
        success "All ${total} pods running"
    else
        warn "Pods: ${running}/${total} running"
    fi

    # AWX web (NodePort 30080)
    local http_code
    http_code=$(curl -sk -o /dev/null -w '%{http_code}' "$(awx_url)/api/v2/ping/" 2>/dev/null || echo "000")
    if [[ "$http_code" == "200" ]]; then
        success "AWX API: responding at $(awx_url) (HTTP ${http_code})"
    else
        error "AWX API: not responding (HTTP ${http_code})"
    fi

    # Admin password
    local admin_pw
    admin_pw=$(get_awx_admin_password 2>/dev/null || echo "")
    if [[ -n "$admin_pw" ]]; then
        success "Admin password: ${admin_pw}"
    else
        warn "Cannot retrieve admin password"
    fi

    # AWX version
    local version
    version=$(curl -sk "$(awx_url)/api/v2/ping/" 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin).get('version','unknown'))" 2>/dev/null || echo "unknown")
    info "AWX version: ${version}"

    # Aether connectivity
    echo
    local aether_code
    aether_code=$(curl -sk -o /dev/null -w '%{http_code}' "${AETHER_URL}/api/health" 2>/dev/null || echo "000")
    if [[ "$aether_code" == "200" || "$aether_code" == "401" || "$aether_code" == "302" ]]; then
        success "Aether: reachable at ${AETHER_URL}"
    else
        warn "Aether: not reachable (HTTP ${aether_code})"
    fi
}

# ---------------------------------------------------------------------------
# Heal — fix common issues
# ---------------------------------------------------------------------------

action_heal() {
    step "Healing AWX on $(vm_ref)"
    echo

    if ! vm_exists || ! vm_running; then
        error "AWX VM is not running"
        return 1
    fi

    # 1. K3s service
    info "Checking K3s service..."
    if vm_exec systemctl is-active k3s &>/dev/null; then
        success "K3s service: active"
    else
        warn "K3s service not active — restarting"
        if dry_run_guard "systemctl restart k3s"; then
            vm_exec systemctl restart k3s
            sleep 10
        fi
    fi

    # 2. Unhealthy pods
    info "Checking for unhealthy pods..."
    local unhealthy
    unhealthy=$(vm_exec kubectl -n "$AWX_NAMESPACE" get pods --no-headers 2>/dev/null \
        | grep -vE 'Running|Completed' | awk '{print $1}' || true)
    if [[ -n "$unhealthy" ]]; then
        warn "Unhealthy pods found — deleting for operator to recreate"
        for pod in $unhealthy; do
            info "  Deleting pod: $pod"
            if dry_run_guard "kubectl delete pod $pod"; then
                vm_exec kubectl -n "$AWX_NAMESPACE" delete pod "$pod" --grace-period=30
            fi
        done
        info "Waiting 30s for pods to restart..."
        sleep 30
    else
        success "All pods healthy"
    fi

    # 3. AWX web responsiveness (NodePort 30080)
    info "Checking AWX web..."
    local http_code
    http_code=$(curl -sk -o /dev/null -w '%{http_code}' "$(awx_url)/api/v2/ping/" 2>/dev/null || echo "000")
    if [[ "$http_code" == "200" ]]; then
        success "AWX web: responding"
    else
        warn "AWX web not responding — restarting deployment"
        if dry_run_guard "kubectl rollout restart deployment/awx-web"; then
            vm_exec kubectl -n "$AWX_NAMESPACE" rollout restart deployment/awx-web 2>/dev/null || true
            vm_exec kubectl -n "$AWX_NAMESPACE" rollout restart deployment/awx-task 2>/dev/null || true
            wait_for_awx_web 300
        fi
    fi

    # 4. Project sync
    info "Triggering project sync..."
    local project_id
    project_id=$(awx_api_auth GET "/api/v2/projects/?name=incus-contrib" 2>/dev/null \
        | python3 -c "import sys,json; r=json.load(sys.stdin); print(r['results'][0]['id'] if r.get('count',0)>0 else '')" 2>/dev/null || true)
    if [[ -n "$project_id" ]]; then
        if dry_run_guard "POST /api/v2/projects/${project_id}/update/"; then
            awx_api_auth POST "/api/v2/projects/${project_id}/update/" 2>/dev/null || true
            success "Project sync triggered (ID: ${project_id})"
        fi
    else
        info "No project configured yet — use --configure"
    fi

    echo
    success "Heal complete"
}

# ---------------------------------------------------------------------------
# Configure — set up AWX project, inventory, templates
# ---------------------------------------------------------------------------

action_configure() {
    step "Configuring AWX"
    echo

    if ! vm_running; then
        error "AWX VM is not running"
        return 1
    fi

    wait_for_awx_web 60

    local admin_pw
    admin_pw=$(get_awx_admin_password)
    detail "Admin password: ${admin_pw}"

    # Helper for authenticated API calls with password
    awx_auth() {
        local method="$1" path="$2"
        shift 2
        curl -sk -X "$method" "$(awx_url)${path}" \
            --user "admin:${admin_pw}" \
            -H "Content-Type: application/json" \
            "$@"
    }

    # 1. Push Incus client cert/key to AWX project directory
    #    Playbooks use Incus REST API (not SSH) — cert is required.
    #    AWX EE mounts project dir at /runner/project/ during job execution.
    step "Pushing Incus client certificate to AWX"
    local incus_cert="${HOME}/.config/incus/client.crt"
    local incus_key="${HOME}/.config/incus/client.key"
    if [[ -f "$incus_cert" && -f "$incus_key" ]]; then
        local task_pod
        task_pod=$(vm_exec kubectl -n "$AWX_NAMESPACE" get pods \
            -l app.kubernetes.io/component=awx-task -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
        if [[ -n "$task_pod" ]]; then
            local project_dir="/var/lib/awx/projects/incus-contrib"
            vm_exec kubectl -n "$AWX_NAMESPACE" exec "$task_pod" -- mkdir -p "$project_dir"
            # Push cert via base64 to avoid file push permission issues
            local cert_b64 key_b64
            cert_b64=$(base64 -w0 "$incus_cert")
            key_b64=$(base64 -w0 "$incus_key")
            vm_exec kubectl -n "$AWX_NAMESPACE" exec "$task_pod" -- \
                bash -c "echo '${cert_b64}' | base64 -d > ${project_dir}/incus-client.crt"
            vm_exec kubectl -n "$AWX_NAMESPACE" exec "$task_pod" -- \
                bash -c "echo '${key_b64}' | base64 -d > ${project_dir}/incus-client.key"
            vm_exec kubectl -n "$AWX_NAMESPACE" exec "$task_pod" -- \
                chmod 600 "${project_dir}/incus-client.key"
            success "Incus client cert/key pushed to AWX task pod"
        else
            warn "Cannot find AWX task pod — push cert manually later"
        fi
    else
        warn "Incus client cert not found at ${incus_cert}"
        warn "Playbooks need the cert to talk to Incus API — push it manually"
    fi

    # 2. Create manual project (local_path)
    #    AWX EE containers cannot reach the private git repo, so we use a
    #    manual project with files pushed directly to the task pod.
    step "Creating project"
    local project_id
    project_id=$(awx_auth GET "/api/v2/projects/?name=incus-contrib" 2>/dev/null \
        | python3 -c "import sys,json; r=json.load(sys.stdin); print(r['results'][0]['id'] if r.get('count',0)>0 else '')" 2>/dev/null || true)

    if [[ -z "$project_id" ]]; then
        project_id=$(awx_auth POST "/api/v2/projects/" \
            -d '{"name":"incus-contrib","organization":1,"scm_type":"","local_path":"incus-contrib"}' 2>/dev/null \
            | python3 -c "import sys,json; print(json.load(sys.stdin).get('id',''))" 2>/dev/null || true)
        [[ -n "$project_id" ]] && success "Project created (ID: ${project_id}, local_path)" || warn "Failed to create project"
    else
        success "Project exists (ID: ${project_id})"
    fi

    # Push playbooks to AWX task pod project directory
    if [[ -n "$project_id" ]]; then
        local task_pod
        task_pod=$(vm_exec kubectl -n "$AWX_NAMESPACE" get pods \
            -l app.kubernetes.io/component=awx-task -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
        if [[ -n "$task_pod" ]]; then
            local project_dir="/var/lib/awx/projects/incus-contrib"
            vm_exec kubectl -n "$AWX_NAMESPACE" exec "$task_pod" -- mkdir -p "${project_dir}/playbooks"
            for playbook in post-deploy.yml decommission.yml; do
                local pb_path="${SCRIPT_DIR}/../ansible/playbooks/${playbook}"
                if [[ -f "$pb_path" ]]; then
                    local pb_b64
                    pb_b64=$(base64 -w0 "$pb_path")
                    vm_exec kubectl -n "$AWX_NAMESPACE" exec "$task_pod" -- \
                        bash -c "echo '${pb_b64}' | base64 -d > ${project_dir}/playbooks/${playbook}"
                    detail "Pushed ${playbook}"
                else
                    warn "Playbook not found: ${pb_path}"
                fi
            done
            success "Playbooks pushed to AWX project directory"
        else
            warn "Cannot find AWX task pod — push playbooks manually"
        fi
    fi

    # 3. Create inventory
    step "Creating inventory"
    local inventory_id
    inventory_id=$(awx_auth GET "/api/v2/inventories/?name=incus-instances" 2>/dev/null \
        | python3 -c "import sys,json; r=json.load(sys.stdin); print(r['results'][0]['id'] if r.get('count',0)>0 else '')" 2>/dev/null || true)

    if [[ -z "$inventory_id" ]]; then
        inventory_id=$(awx_auth POST "/api/v2/inventories/" \
            -d '{"name":"incus-instances","organization":1}' 2>/dev/null \
            | python3 -c "import sys,json; print(json.load(sys.stdin).get('id',''))" 2>/dev/null || true)
        [[ -n "$inventory_id" ]] && success "Inventory created (ID: ${inventory_id})" || warn "Failed to create inventory"
    else
        success "Inventory exists (ID: ${inventory_id})"
    fi

    # 4. Create machine credential (required by AWX templates, though
    #    playbooks use Incus API not SSH)
    step "Creating machine credential"
    local machine_cred_id
    machine_cred_id=$(awx_auth GET "/api/v2/credentials/?name=incus-instances" 2>/dev/null \
        | python3 -c "import sys,json; r=json.load(sys.stdin); print(r['results'][0]['id'] if r.get('count',0)>0 else '')" 2>/dev/null || true)

    if [[ -z "$machine_cred_id" ]]; then
        machine_cred_id=$(awx_auth POST "/api/v2/credentials/" \
            -d '{"name":"incus-instances","organization":1,"credential_type":1,"inputs":{"username":"root"}}' 2>/dev/null \
            | python3 -c "import sys,json; print(json.load(sys.stdin).get('id',''))" 2>/dev/null || true)
        [[ -n "$machine_cred_id" ]] && success "Machine credential created (ID: ${machine_cred_id})" || warn "Failed to create machine credential"
    else
        success "Machine credential exists (ID: ${machine_cred_id})"
    fi

    # 5. Create job templates
    step "Creating job templates"
    for tmpl_name in post-deploy decommission; do
        local tmpl_id
        tmpl_id=$(awx_auth GET "/api/v2/job_templates/?name=${tmpl_name}" 2>/dev/null \
            | python3 -c "import sys,json; r=json.load(sys.stdin); print(r['results'][0]['id'] if r.get('count',0)>0 else '')" 2>/dev/null || true)

        if [[ -z "$tmpl_id" ]]; then
            local tmpl_payload
            tmpl_payload=$(python3 -c "
import json
d = {
    'name': '${tmpl_name}',
    'organization': 1,
    'job_type': 'run',
    'ask_variables_on_launch': True,
    'playbook': '${PLAYBOOK_DIR}/${tmpl_name}.yml'
}
pid = '${project_id}'
iid = '${inventory_id}'
cid = '${machine_cred_id}'
if pid: d['project'] = int(pid)
if iid: d['inventory'] = int(iid)
print(json.dumps(d))")

            tmpl_id=$(awx_auth POST "/api/v2/job_templates/" -d "$tmpl_payload" 2>/dev/null \
                | python3 -c "import sys,json; print(json.load(sys.stdin).get('id',''))" 2>/dev/null || true)

            # Associate credential separately (templates use credential list)
            if [[ -n "$tmpl_id" && -n "$machine_cred_id" ]]; then
                awx_auth POST "/api/v2/job_templates/${tmpl_id}/credentials/" \
                    -d "{\"id\":${machine_cred_id}}" 2>/dev/null || true
            fi

            [[ -n "$tmpl_id" ]] && success "Template '${tmpl_name}' created (ID: ${tmpl_id})" || warn "Failed to create template '${tmpl_name}'"
        else
            success "Template '${tmpl_name}' exists (ID: ${tmpl_id})"
        fi
    done

    echo
    success "AWX configuration complete"
    echo
    info "Template IDs (for Aether binding):"
    for tmpl_name in post-deploy decommission; do
        local tid
        tid=$(awx_auth GET "/api/v2/job_templates/?name=${tmpl_name}" 2>/dev/null \
            | python3 -c "import sys,json; r=json.load(sys.stdin); print(r['results'][0]['id'] if r.get('count',0)>0 else '')" 2>/dev/null || true)
        info "  ${tmpl_name}: ${tid:-unknown}"
    done
    echo
    info "Next: ${SCRIPT_NAME} --join-aether"
}

# ---------------------------------------------------------------------------
# Join Aether — register AWX endpoint + bind to cluster
# ---------------------------------------------------------------------------

action_join_aether() {
    step "Registering AWX with Aether"
    echo

    if ! vm_running; then
        error "AWX VM is not running"
        return 1
    fi

    # 1. Create Personal Access Token
    step "Creating AWX Personal Access Token"
    local admin_pw
    admin_pw=$(get_awx_admin_password)

    local pat_response
    pat_response=$(curl -sk -X POST "$(awx_url)/api/v2/users/1/personal_tokens/" \
        --user "admin:${admin_pw}" \
        -H "Content-Type: application/json" \
        -d '{"description": "Aether integration", "scope": "write"}' 2>/dev/null)

    local awx_token
    awx_token=$(echo "$pat_response" | python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null || true)

    if [[ -z "$awx_token" ]]; then
        error "Failed to create PAT — check AWX admin credentials"
        detail "Response: ${pat_response}"
        return 1
    fi
    success "PAT created"

    # 2. Get template IDs
    local post_deploy_id decommission_id
    post_deploy_id=$(curl -sk "$(awx_url)/api/v2/job_templates/?name=post-deploy" \
        --user "admin:${admin_pw}" 2>/dev/null \
        | python3 -c "import sys,json; r=json.load(sys.stdin); print(r['results'][0]['id'] if r.get('count',0)>0 else '')" 2>/dev/null || true)
    decommission_id=$(curl -sk "$(awx_url)/api/v2/job_templates/?name=decommission" \
        --user "admin:${admin_pw}" 2>/dev/null \
        | python3 -c "import sys,json; r=json.load(sys.stdin); print(r['results'][0]['id'] if r.get('count',0)>0 else '')" 2>/dev/null || true)

    info "Post-deploy template ID: ${post_deploy_id:-not found}"
    info "Decommission template ID: ${decommission_id:-not found}"

    echo
    info "To complete Aether integration:"
    echo
    info "1. Log into Aether at ${AETHER_URL}"
    info "2. Navigate to Settings → Ansible Automation (/awx-endpoints)"
    info "3. Add Endpoint:"
    info "     Name:       lab-awx"
    info "     URL:        $(awx_url)"
    info "     Token:      ${awx_token}"
    info "     Verify SSL: unchecked"
    info "4. Configure cluster AWX binding:"
    info "     Post-deploy template ID:    ${post_deploy_id:-<configure first>}"
    info "     Decommission template ID:   ${decommission_id:-<configure first>}"
    info "     Job timeout:                600 seconds"
    echo

    # Print API commands for scripted registration
    if [[ "$VERBOSE" == true ]]; then
        detail "API commands for scripted registration:"
        detail ""
        detail "# Register endpoint:"
        detail "curl -sSk -b cookies.txt -H 'X-CSRF-Token: \$CSRF' \\"
        detail "  -H 'Referer: ${AETHER_URL}/' \\"
        detail "  -X POST ${AETHER_URL}/api/awx/endpoints \\"
        detail "  -H 'Content-Type: application/json' \\"
        detail "  -d '{\"name\":\"lab-awx\",\"url\":\"http://${AWX_IP}:30080\",\"token\":\"${awx_token}\",\"verify_ssl\":false}'"
        detail ""
        detail "# Bind cluster (replace ENDPOINT_ID):"
        detail "curl -sSk -b cookies.txt -H 'X-CSRF-Token: \$CSRF' \\"
        detail "  -H 'Referer: ${AETHER_URL}/' \\"
        detail "  -X PUT ${AETHER_URL}/api/clusters/${AETHER_CLUSTER_ID}/awx-config \\"
        detail "  -H 'Content-Type: application/json' \\"
        detail "  -d '{\"cluster_id\":${AETHER_CLUSTER_ID},\"awx_endpoint_id\":ENDPOINT_ID,\"awx_post_deploy_template_id\":${post_deploy_id:-0},\"awx_decommission_template_id\":${decommission_id:-0},\"awx_job_timeout_seconds\":600}'"
    fi
}

# ---------------------------------------------------------------------------
# Cleanup — destroy AWX VM
# ---------------------------------------------------------------------------

action_cleanup() {
    step "Cleaning up AWX VM"

    if ! vm_exists; then
        info "AWX VM does not exist — nothing to clean up"
        return 0
    fi

    echo
    warn "This will permanently destroy the AWX VM '$(vm_ref)'"
    warn "All AWX data (jobs, templates, credentials) will be lost."
    echo

    if [[ "$DRY_RUN" == true ]]; then
        info "(dry-run) Would destroy $(vm_ref)"
        return 0
    fi

    read -r -p "Type 'yes' to confirm: " confirm
    if [[ "$confirm" != "yes" ]]; then
        info "Aborted"
        return 0
    fi

    if vm_running; then
        info "Stopping VM..."
        incus stop "$(vm_ref)" --force 2>/dev/null || true
        sleep 3
    fi

    info "Deleting VM..."
    incus delete "$(vm_ref)" --force 2>/dev/null || true

    success "AWX VM destroyed"
}

# ---------------------------------------------------------------------------
# Argument parsing
# ---------------------------------------------------------------------------

parse_args() {
    while [[ $# -gt 0 ]]; do
        case "$1" in
            --deploy)      ACTION="deploy";;
            --status)      ACTION="status";;
            --heal)        ACTION="heal";;
            --configure)   ACTION="configure";;
            --join-aether) ACTION="join-aether";;
            --cleanup)     ACTION="cleanup";;
            --doctor)      ACTION="doctor";;
            -c|--config)   CONFIG_FILE="$2"; shift;;
            -n|--dry-run)  DRY_RUN=true;;
            -v|--verbose)  VERBOSE=true;;
            -q|--quiet)    QUIET=true;;
            -h|--help)     usage; exit 0;;
            *)
                error "Unknown option: $1"
                echo "Run '${SCRIPT_NAME} --help' for usage."
                exit 1
                ;;
        esac
        shift
    done

    if [[ -z "$ACTION" ]]; then
        error "No action specified"
        echo "Run '${SCRIPT_NAME} --help' for usage."
        exit 1
    fi
}

# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

main() {
    setup_colors
    parse_args "$@"

    # Load config file if provided
    if [[ -n "$CONFIG_FILE" ]]; then
        parse_config "$CONFIG_FILE"
    fi

    echo
    case "$ACTION" in
        deploy)      action_deploy;;
        status)      action_status;;
        heal)        action_heal;;
        configure)   action_configure;;
        join-aether) action_join_aether;;
        cleanup)     action_cleanup;;
        doctor)      action_doctor;;
    esac
}

main "$@"