#!/usr/bin/env bash
# deploy-observability - Deploy and manage the Incus observability stack
#
# Deploys Prometheus, Grafana, Loki, and Promtail in a monitoring container,
# plus node-exporter containers on each cluster node. Configures Prometheus
# to scrape Incus metrics (with client certs), HAProxy stats, and node
# exporters. Sets up Grafana with datasources and dashboards.
#
# Usage: deploy-observability [OPTIONS]
# Run 'deploy-observability --help' for full usage information.

set -euo pipefail

readonly VERSION="0.1.0"
readonly SCRIPT_NAME="deploy-observability"
readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly DASHBOARDS_DIR="${SCRIPT_DIR}/observability-dashboards"

# ---------------------------------------------------------------------------
# Defaults (overridden by CLI flags)
# ---------------------------------------------------------------------------

CLUSTER_REMOTE="oc-node-01"
OVN_NETWORK="net-prod"
MONITORING_IP="10.10.10.70"
MONITORING_TARGET="oc-node-02"
FORWARD_VIP="192.168.103.201"

# Fallback defaults — used when cluster is unreachable (e.g., --dry-run)
DEFAULT_INCUS_NODES=("192.168.102.140" "192.168.102.141" "192.168.102.142")
DEFAULT_NODE_EXP_TARGETS=("oc-node-01" "oc-node-02" "oc-node-03")
DEFAULT_NODE_EXP_IPS=("10.10.10.71" "10.10.10.72" "10.10.10.73")

INCUS_NODES=()
NODE_EXP_TARGETS=()
NODE_EXP_IPS=()
HAPROXY_IPS=("10.10.10.50" "10.10.10.51")

HAPROXY_CLUSTER_ID="52"
OC_SERVER_IP=""

# Runtime flags
DRY_RUN=false
VERBOSE=false
QUIET=false
RESUME=false
ACTION=""
MANUAL_NODES=""

# ---------------------------------------------------------------------------
# Color and formatting
# ---------------------------------------------------------------------------

setup_colors() {
    if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then
        RED='\033[0;31m'
        GREEN='\033[0;32m'
        YELLOW='\033[0;33m'
        BLUE='\033[0;34m'
        CYAN='\033[0;36m'
        BOLD='\033[1m'
        DIM='\033[2m'
        RESET='\033[0m'
    else
        RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' RESET=''
    fi
}

info()    { [[ "$QUIET" == true ]] && return; echo -e "${BLUE}[info]${RESET}    $*"; }
success() { [[ "$QUIET" == true ]] && return; echo -e "${GREEN}[ok]${RESET}      $*"; }
warn()    { echo -e "${YELLOW}[warn]${RESET}    $*" >&2; }
error()   { echo -e "${RED}[error]${RESET}   $*" >&2; }
step()    { [[ "$QUIET" == true ]] && return; echo -e "${CYAN}[step]${RESET}    ${BOLD}$*${RESET}"; }
detail()  { [[ "$VERBOSE" != true ]] && return; echo -e "${DIM}          $*${RESET}"; }

dry_run_guard() {
    if [[ "$DRY_RUN" == true ]]; then
        info "(dry-run) $*"
        return 1
    fi
    return 0
}

# ---------------------------------------------------------------------------
# Help
# ---------------------------------------------------------------------------

usage() {
    cat <<EOF
${BOLD}${SCRIPT_NAME}${RESET} v${VERSION} - Deploy the Incus observability stack

${BOLD}USAGE${RESET}
    ${SCRIPT_NAME} [OPTIONS]

${BOLD}ACTIONS${RESET}
    --deploy          Full deploy: monitoring container, Prometheus, Grafana,
                      Loki, Promtail, node-exporters, ACLs, network forward,
                      HAProxy exporter route, dashboards
    --workloads       Deploy dummy workloads to generate varied metrics
    --status          Show status of all observability components
    --cleanup         Remove all observability infrastructure
    --doctor          Diagnose common issues

${BOLD}OPTIONS${RESET}
    -r, --remote NAME         Incus remote (default: ${CLUSTER_REMOTE})
    --nodes NODE1,NODE2       Manual node list (skip auto-discovery)
    --monitoring-target NODE  Node to deploy monitoring container on (default: ${MONITORING_TARGET})
    --monitoring-ip IP        IP for monitoring container (default: ${MONITORING_IP})
    --forward-vip IP          OVN external IP for Grafana/Prometheus forward (default: ${FORWARD_VIP})
    --oc-server IP            Monitor oc-server at this IP (proxy device for 9100)
    -n, --dry-run         Preview actions without executing
    -v, --verbose         Show detailed output
    -q, --quiet           Suppress informational output
    -h, --help            Show this help message

${BOLD}COMPONENTS${RESET}
    Monitoring container (Debian 12, 2 GiB RAM, 20 GiB disk):
        Prometheus    :9090  - metrics collection and alerting
        Grafana       :3000  - dashboards and visualization
        Loki          :3100  - log aggregation
        Promtail              - log shipping agent

    Node-exporter containers (Alpine, 128 MiB, privileged):
        Auto-discovered from cluster (or --nodes override), :9100

    Network forward:
        ${FORWARD_VIP} -> Grafana (:3000), Prometheus (:9090)

${BOLD}EXAMPLES${RESET}
    # Check prerequisites and connectivity
    ${SCRIPT_NAME} --doctor

    # Preview what would be deployed
    ${SCRIPT_NAME} --deploy --dry-run

    # Full deployment (auto-discovers cluster nodes)
    ${SCRIPT_NAME} --deploy

    # Deploy with oc-server monitoring
    ${SCRIPT_NAME} --deploy --oc-server 192.168.102.120

    # Deploy with manual node list
    ${SCRIPT_NAME} --deploy --nodes oc-node-01,oc-node-02,oc-node-03

    # Check health of all components
    ${SCRIPT_NAME} --status

    # Tear down everything
    ${SCRIPT_NAME} --cleanup
EOF
}

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

container_ref() {
    echo "${CLUSTER_REMOTE}:$1"
}

container_exists() {
    incus info "$(container_ref "$1")" &>/dev/null 2>&1
}

container_running() {
    local state
    state=$(incus info "$(container_ref "$1")" 2>/dev/null | grep -i '^status:' | awk '{print $2}')
    [[ "$state" == "RUNNING" || "$state" == "Running" ]]
}

container_exec() {
    local name="$1"
    shift
    incus exec "$(container_ref "$name")" -- "$@"
}

wait_for_container_agent() {
    local name="$1"
    local timeout="${2:-60}"
    local elapsed=0
    while ! container_exec "$name" true &>/dev/null 2>&1; do
        sleep 2
        elapsed=$((elapsed + 2))
        if [[ $elapsed -ge $timeout ]]; then
            error "Container agent for '${name}' not available after ${timeout}s"
            return 1
        fi
    done
}

node_exp_name() {
    local idx="$1"
    printf "node-exp-%02d" "$idx"
}

# ---------------------------------------------------------------------------
# Cluster auto-discovery
# ---------------------------------------------------------------------------

discover_cluster_nodes() {
    # Manual override via --nodes takes priority
    if [[ -n "$MANUAL_NODES" ]]; then
        info "Using manually specified nodes: ${MANUAL_NODES}"
        INCUS_NODES=()
        NODE_EXP_TARGETS=()
        NODE_EXP_IPS=()
        local exp_octet=71
        IFS=',' read -ra names <<< "$MANUAL_NODES"
        for name in "${names[@]}"; do
            # Look up address from Incus remote
            local addr
            addr=$(incus cluster list "${CLUSTER_REMOTE}:" --format csv 2>/dev/null \
                | grep "^${name}," | head -1 | cut -d',' -f2 | sed 's|https://||;s|:.*||') || true
            if [[ -z "$addr" ]]; then
                warn "Could not resolve address for node '${name}' — skipping"
                continue
            fi
            INCUS_NODES+=("$addr")
            NODE_EXP_TARGETS+=("$name")
            NODE_EXP_IPS+=("10.10.10.${exp_octet}")
            exp_octet=$((exp_octet + 1))
        done
        return 0
    fi

    # Auto-discover from cluster
    local csv
    csv=$(incus cluster list "${CLUSTER_REMOTE}:" --format csv 2>/dev/null) || true

    if [[ -z "$csv" ]]; then
        if [[ "$DRY_RUN" == true ]]; then
            info "Cluster not reachable — using fallback defaults for dry-run"
            INCUS_NODES=("${DEFAULT_INCUS_NODES[@]}")
            NODE_EXP_TARGETS=("${DEFAULT_NODE_EXP_TARGETS[@]}")
            NODE_EXP_IPS=("${DEFAULT_NODE_EXP_IPS[@]}")
            return 0
        fi
        error "Cannot reach cluster via remote '${CLUSTER_REMOTE}'"
        error "Check that 'incus remote list' includes '${CLUSTER_REMOTE}'."
        return 1
    fi

    INCUS_NODES=()
    NODE_EXP_TARGETS=()
    NODE_EXP_IPS=()

    local exp_octet=71
    while IFS=',' read -r name url roles _rest; do
        # Extract IP from URL: https://192.168.102.140:8443 → 192.168.102.140
        local addr="${url#https://}"
        addr="${addr%%:*}"
        INCUS_NODES+=("$addr")
        NODE_EXP_TARGETS+=("$name")
        NODE_EXP_IPS+=("10.10.10.${exp_octet}")
        exp_octet=$((exp_octet + 1))
    done <<< "$csv"

    info "Discovered ${#INCUS_NODES[@]} cluster node(s): ${NODE_EXP_TARGETS[*]}"
}

# ---------------------------------------------------------------------------
# OC server helpers
# ---------------------------------------------------------------------------

oc_server_remote_exists() {
    incus remote list --format csv 2>/dev/null | grep -q "^oc-server,"
}

oc_server_container_ref() {
    echo "oc-server:$1"
}

oc_server_container_exists() {
    incus info "$(oc_server_container_ref "$1")" &>/dev/null 2>&1
}

oc_server_container_running() {
    local state
    state=$(incus info "$(oc_server_container_ref "$1")" 2>/dev/null | grep -i '^status:' | awk '{print $2}')
    [[ "$state" == "RUNNING" || "$state" == "Running" ]]
}

oc_server_container_exec() {
    local name="$1"
    shift
    incus exec "$(oc_server_container_ref "$name")" -- "$@"
}

# ---------------------------------------------------------------------------
# Deploy action
# ---------------------------------------------------------------------------

action_deploy() {
    step "Deploying observability stack on ${CLUSTER_REMOTE}"
    echo
    info "  Monitoring:     ${MONITORING_IP} (on ${MONITORING_TARGET})"
    info "  Node-exporters: ${NODE_EXP_IPS[*]}"
    info "  Incus nodes:    ${INCUS_NODES[*]}"
    info "  HAProxy stats:  ${HAPROXY_IPS[*]}"
    info "  Network fwd:    ${FORWARD_VIP} -> Grafana/Prometheus"
    [[ -n "$OC_SERVER_IP" ]] && info "  OC server:      ${OC_SERVER_IP} (proxy :9100)"
    echo

    if container_exists "monitoring"; then
        if [[ "$RESUME" == true ]]; then
            info "Monitoring container already exists — resuming from phase 5b"
        else
            error "Monitoring container already exists"
            error "Use --status to check health, --cleanup first, or --resume to continue."
            return 1
        fi
    fi

    if [[ "$RESUME" != true ]]; then
        phase_deploy_monitoring
        phase_install_prometheus
        phase_install_loki
        phase_install_grafana
        phase_deploy_node_exporters
    fi
    phase_deploy_oc_server_exporter
    phase_create_acl
    phase_configure_haproxy_exporter
    phase_create_network_forward
    phase_upload_dashboards

    echo
    success "Observability stack deployed!"
    echo
    info "  Grafana:     http://${FORWARD_VIP}:3000  (admin / admin)"
    info "  Prometheus:  http://${FORWARD_VIP}:9090"
    info "  Loki:        http://${MONITORING_IP}:3100  (internal)"
    echo
    info "Next steps:"
    info "  ${SCRIPT_NAME} --status    # verify all targets are up"
    info "  Open Grafana and explore the pre-loaded dashboards"
}

# ---------------------------------------------------------------------------
# Phase 1: Create monitoring container
# ---------------------------------------------------------------------------

phase_deploy_monitoring() {
    step "Phase 1: Create monitoring container"

    if ! dry_run_guard "Launch monitoring on ${OVN_NETWORK} at ${MONITORING_IP}"; then
        return 0
    fi

    incus launch images:debian/12 "$(container_ref monitoring)" \
        -n "$OVN_NETWORK" \
        --target "$MONITORING_TARGET" \
        -c limits.memory=2GiB \
        -d root,size=20GiB

    # Tell OVN the static IP we want — port security drops traffic from
    # IPs that OVN didn't assign, so this must be set at the device level.
    incus config device set "$(container_ref monitoring)" eth0 \
        ipv4.address="${MONITORING_IP}"

    info "Waiting for container agent..."
    wait_for_container_agent monitoring 60

    # Discover upstream DNS from the UPLINK network config.
    # OVN DHCP advertises this automatically, but we need it for the static
    # networkd config (which bypasses DHCP).
    local upstream_dns
    upstream_dns=$(incus network get "${CLUSTER_REMOTE}:UPLINK" dns.nameservers 2>/dev/null) || true
    upstream_dns="${upstream_dns:-192.168.100.1}"
    detail "Using upstream DNS: ${upstream_dns}"

    # Build DNS lines (handles comma-separated list from UPLINK config)
    local dns_lines=""
    IFS=',' read -ra dns_addrs <<< "$upstream_dns"
    for addr in "${dns_addrs[@]}"; do
        dns_lines="${dns_lines}DNS=${addr}"$'\n'
    done

    # Configure matching static IP inside the container (systemd-networkd
    # bypasses DHCP, so we must specify DNS explicitly).
    container_exec monitoring bash -c "
        cat > /etc/systemd/network/10-static.network << NETCFG
[Match]
Name=eth0

[Network]
Address=${MONITORING_IP}/24
Gateway=10.10.10.1
${dns_lines}NETCFG
        systemctl restart systemd-networkd systemd-resolved
    "

    sleep 3
    success "Monitoring container created at ${MONITORING_IP}"
}

# ---------------------------------------------------------------------------
# Phase 2: Install and configure Prometheus
# ---------------------------------------------------------------------------

phase_install_prometheus() {
    step "Phase 2: Install and configure Prometheus"

    if ! dry_run_guard "Install Prometheus in monitoring container"; then
        return 0
    fi

    # Copy Incus client certs into the container for metrics scraping
    local incus_cert="${HOME}/.config/incus/client.crt"
    local incus_key="${HOME}/.config/incus/client.key"

    if [[ ! -f "$incus_cert" || ! -f "$incus_key" ]]; then
        error "Incus client certificates not found at ~/.config/incus/"
        error "Prometheus needs these to scrape Incus metrics endpoints."
        error "Install the Incus client and run 'incus remote add' first."
        return 1
    fi

    incus file push "$incus_cert" "$(container_ref monitoring)/tmp/incus-client.crt"
    incus file push "$incus_key" "$(container_ref monitoring)/tmp/incus-client.key"

    container_exec monitoring bash -c "
        export DEBIAN_FRONTEND=noninteractive

        # Add Grafana apt repository (provides grafana, loki, promtail)
        apt-get update -qq
        apt-get install -y -qq ca-certificates curl gnupg apt-transport-https 2>&1 | tail -1
        curl -fsSL https://apt.grafana.com/gpg.key | gpg --dearmor -o /usr/share/keyrings/grafana-archive-keyring.gpg
        echo 'deb [signed-by=/usr/share/keyrings/grafana-archive-keyring.gpg] https://apt.grafana.com stable main' \
            > /etc/apt/sources.list.d/grafana.list
        apt-get update -qq

        apt-get install -y -qq prometheus 2>&1 | tail -1

        # Place Incus client certs where Prometheus can read them
        mkdir -p /etc/prometheus/certs
        cp /tmp/incus-client.crt /etc/prometheus/certs/
        cp /tmp/incus-client.key /etc/prometheus/certs/
        chown -R prometheus:prometheus /etc/prometheus/certs
        chmod 600 /etc/prometheus/certs/incus-client.key
    "

    # Build scrape targets
    local incus_targets=""
    local idx=0
    for node in "${INCUS_NODES[@]}"; do
        if [[ $idx -gt 0 ]]; then incus_targets+=", "; fi
        incus_targets+="'${node}:8443'"
        idx=$((idx + 1))
    done
    # Add oc-server to Incus API scraping
    if [[ -n "$OC_SERVER_IP" ]]; then
        if [[ $idx -gt 0 ]]; then incus_targets+=", "; fi
        incus_targets+="'${OC_SERVER_IP}:8443'"
    fi

    local haproxy_targets=""
    idx=0
    for ip in "${HAPROXY_IPS[@]}"; do
        if [[ $idx -gt 0 ]]; then haproxy_targets+=", "; fi
        haproxy_targets+="'${ip}:8404'"
        idx=$((idx + 1))
    done

    local node_exp_config=""
    idx=0
    for ip in "${NODE_EXP_IPS[@]}"; do
        local target_name="${NODE_EXP_TARGETS[$idx]}"
        node_exp_config+="
      - targets: ['${ip}:9100']
        labels:
          nodename: '${target_name}'"
        idx=$((idx + 1))
    done
    # Add oc-server node-exporter (via proxy device on host IP)
    if [[ -n "$OC_SERVER_IP" ]]; then
        node_exp_config+="
      - targets: ['${OC_SERVER_IP}:9100']
        labels:
          nodename: 'oc-server'"
    fi

    container_exec monitoring bash -c "cat > /etc/prometheus/prometheus.yml << 'PROMCFG'
global:
  scrape_interval: 15s
  evaluation_interval: 15s

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'incus'
    scheme: https
    tls_config:
      cert_file: /etc/prometheus/certs/incus-client.crt
      key_file: /etc/prometheus/certs/incus-client.key
      insecure_skip_verify: true
    metrics_path: /1.0/metrics
    static_configs:
      - targets: [${incus_targets}]
    metric_relabel_configs:
      - source_labels: [__name__]
        regex: 'node_.*'
        action: drop

  - job_name: 'haproxy'
    static_configs:
      - targets: [${haproxy_targets}]

  - job_name: 'node-exporter'
    static_configs:${node_exp_config}
    relabel_configs:
      - source_labels: [nodename]
        target_label: instance
PROMCFG"

    container_exec monitoring systemctl enable prometheus
    container_exec monitoring systemctl restart prometheus
    sleep 2

    success "Prometheus installed and configured"
}

# ---------------------------------------------------------------------------
# Phase 3: Install and configure Loki + Promtail
# ---------------------------------------------------------------------------

phase_install_loki() {
    step "Phase 3: Install Loki and Promtail"

    if ! dry_run_guard "Install Loki + Promtail in monitoring container"; then
        return 0
    fi

    container_exec monitoring bash -c "
        export DEBIAN_FRONTEND=noninteractive
        apt-get install -y -qq loki promtail 2>&1 | tail -1
    "

    # Configure Loki
    container_exec monitoring bash -c "cat > /etc/loki/config.yml << 'LOKICFG'
auth_enabled: false

server:
  http_listen_port: 3100
  grpc_listen_port: 9096

common:
  path_prefix: /var/lib/loki
  storage:
    filesystem:
      chunks_directory: /var/lib/loki/chunks
      rules_directory: /var/lib/loki/rules
  replication_factor: 1
  ring:
    kvstore:
      store: inmemory

schema_config:
  configs:
    - from: 2020-10-24
      store: tsdb
      object_store: filesystem
      schema: v13
      index:
        prefix: index_
        period: 24h

limits_config:
  reject_old_samples: true
  reject_old_samples_max_age: 168h
  allow_structured_metadata: false
LOKICFG"

    # Configure Promtail to ship local logs
    container_exec monitoring bash -c "cat > /etc/promtail/config.yml << 'PTCFG'
server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /var/lib/promtail/positions.yaml

clients:
  - url: http://localhost:3100/loki/api/v1/push

scrape_configs:
  - job_name: system
    static_configs:
      - targets:
          - localhost
        labels:
          job: varlogs
          host: monitoring
          __path__: /var/log/**/*.log

  - job_name: journal
    journal:
      max_age: 12h
      labels:
        job: systemd-journal
        host: monitoring
    relabel_configs:
      - source_labels: ['__journal__systemd_unit']
        target_label: unit
      - source_labels: ['__journal_priority_keyword']
        target_label: level
      - source_labels: ['__journal_syslog_identifier']
        target_label: service_name
PTCFG"

    # Ensure data directories exist with correct ownership
    container_exec monitoring bash -c "mkdir -p /var/lib/loki && chown loki:nogroup /var/lib/loki"
    container_exec monitoring bash -c "mkdir -p /var/lib/promtail && chown promtail:nogroup /var/lib/promtail"

    # Promtail needs adm group for /var/log access, systemd-journal for journal
    container_exec monitoring bash -c "usermod -aG adm,systemd-journal promtail"

    container_exec monitoring systemctl enable loki
    container_exec monitoring systemctl restart loki
    sleep 2

    container_exec monitoring systemctl enable promtail
    container_exec monitoring systemctl restart promtail

    success "Loki and Promtail installed and configured"
}

# ---------------------------------------------------------------------------
# Phase 4: Install and configure Grafana
# ---------------------------------------------------------------------------

phase_install_grafana() {
    step "Phase 4: Install and configure Grafana"

    if ! dry_run_guard "Install Grafana in monitoring container"; then
        return 0
    fi

    container_exec monitoring bash -c "
        export DEBIAN_FRONTEND=noninteractive
        apt-get install -y -qq grafana 2>&1 | tail -1
    "

    # Install Logs Drilldown plugin for interactive log exploration
    container_exec monitoring grafana-cli plugins install grafana-lokiexplore-app 2>&1 | tail -1

    # Provision datasources with explicit UIDs for dashboard portability
    container_exec monitoring bash -c "mkdir -p /etc/grafana/provisioning/datasources
cat > /etc/grafana/provisioning/datasources/observability.yaml << 'DSCFG'
apiVersion: 1
datasources:
  - name: Prometheus
    type: prometheus
    uid: prometheus
    access: proxy
    url: http://localhost:9090
    isDefault: true
    editable: true

  - name: Loki
    type: loki
    uid: loki
    access: proxy
    url: http://localhost:3100
    editable: true
DSCFG"

    container_exec monitoring systemctl enable grafana-server
    container_exec monitoring systemctl restart grafana-server
    sleep 2

    success "Grafana installed with Prometheus + Loki datasources"
}

# ---------------------------------------------------------------------------
# Phase 5: Deploy node-exporter containers
# ---------------------------------------------------------------------------

phase_deploy_node_exporters() {
    step "Phase 5: Deploy node-exporter containers"

    local idx=1
    for ip in "${NODE_EXP_IPS[@]}"; do
        local name
        name=$(node_exp_name "$idx")
        local target="${NODE_EXP_TARGETS[$((idx - 1))]}"

        if container_exists "$name"; then
            if container_running "$name"; then
                success "  ${name} already running"
                idx=$((idx + 1))
                continue
            fi
        fi

        if ! dry_run_guard "Launch ${name} on ${target} at ${ip}"; then
            idx=$((idx + 1))
            continue
        fi

        info "Launching ${name} (${ip}) on ${target}..."
        incus launch images:alpine/3.20 "$(container_ref "$name")" \
            -n "$OVN_NETWORK" \
            --target "$target" \
            -c limits.memory=128MiB \
            -c security.privileged=true

        # Reserve this IP in OVN before switching to static inside the container
        incus config device set "$(container_ref "$name")" eth0 \
            ipv4.address="${ip}"

        wait_for_container_agent "$name" 30

        # Mount host filesystems for full node metrics
        incus config device add "$(container_ref "$name")" host-proc disk \
            source=/proc path=/host/proc readonly=true
        incus config device add "$(container_ref "$name")" host-sys disk \
            source=/sys path=/host/sys readonly=true
        incus config device add "$(container_ref "$name")" host-root disk \
            source=/ path=/host/root readonly=true

        # Install and configure node_exporter
        container_exec "$name" sh -c "
            apk add --no-cache prometheus-node-exporter > /dev/null 2>&1

            # Configure to use host mount paths
            mkdir -p /etc/conf.d
            cat > /etc/conf.d/node-exporter << 'NEXPCFG'
ARGS=\"--path.procfs=/host/proc --path.sysfs=/host/sys --path.rootfs=/host/root\"
NEXPCFG

            rc-update add node-exporter default 2>/dev/null || true
            rc-service node-exporter start 2>/dev/null || true
        "

        # Configure static IP
        container_exec "$name" sh -c "
            cat > /etc/network/interfaces << NETCFG
auto lo
iface lo inet loopback

auto eth0
iface eth0 inet static
    address ${ip}/24
    gateway 10.10.10.1
NETCFG
            rc-service networking restart 2>/dev/null || true
        "

        success "  ${name} deployed at ${ip} on ${target}"
        idx=$((idx + 1))
    done

    success "Node-exporter containers deployed"
}

# ---------------------------------------------------------------------------
# Phase 5b: Deploy node-exporter on oc-server (if --oc-server set)
# ---------------------------------------------------------------------------

phase_deploy_oc_server_exporter() {
    [[ -z "$OC_SERVER_IP" ]] && return 0

    step "Phase 5b: Deploy node-exporter on oc-server (${OC_SERVER_IP})"

    # Validate oc-server remote exists
    if ! oc_server_remote_exists; then
        error "Incus remote 'oc-server' not found."
        error "Add it with:"
        error "  incus remote add oc-server https://${OC_SERVER_IP}:8443 --accept-certificate"
        return 1
    fi

    local name="node-exp-oc-server"

    if oc_server_container_exists "$name"; then
        if oc_server_container_running "$name"; then
            success "  ${name} already running on oc-server"
            return 0
        fi
    fi

    if ! dry_run_guard "Launch ${name} on oc-server at ${OC_SERVER_IP}"; then
        return 0
    fi

    info "Launching ${name} on oc-server..."
    incus launch images:alpine/3.20 "$(oc_server_container_ref "$name")" \
        -c limits.memory=128MiB \
        -c security.privileged=true

    wait_for_container_agent_oc_server "$name" 30

    # Mount host filesystems for full node metrics
    incus config device add "$(oc_server_container_ref "$name")" host-proc disk \
        source=/proc path=/host/proc readonly=true
    incus config device add "$(oc_server_container_ref "$name")" host-sys disk \
        source=/sys path=/host/sys readonly=true
    incus config device add "$(oc_server_container_ref "$name")" host-root disk \
        source=/ path=/host/root readonly=true

    # Expose 9100 on the host IP via proxy device (oc-server is on incusbr0,
    # not net-prod, so Prometheus reaches it via the host address)
    incus config device add "$(oc_server_container_ref "$name")" proxy9100 proxy \
        listen=tcp:${OC_SERVER_IP}:9100 connect=tcp:127.0.0.1:9100

    # Install and configure node_exporter
    oc_server_container_exec "$name" sh -c "
        apk add --no-cache prometheus-node-exporter > /dev/null 2>&1

        mkdir -p /etc/conf.d
        cat > /etc/conf.d/node-exporter << 'NEXPCFG'
ARGS=\"--path.procfs=/host/proc --path.sysfs=/host/sys --path.rootfs=/host/root\"
NEXPCFG

        rc-update add node-exporter default 2>/dev/null || true
        rc-service node-exporter start 2>/dev/null || true
    "

    success "  ${name} deployed on oc-server with proxy ${OC_SERVER_IP}:9100"
}

wait_for_container_agent_oc_server() {
    local name="$1"
    local timeout="${2:-60}"
    local elapsed=0
    while ! oc_server_container_exec "$name" true &>/dev/null 2>&1; do
        sleep 2
        elapsed=$((elapsed + 2))
        if [[ $elapsed -ge $timeout ]]; then
            error "Container agent for '${name}' on oc-server not available after ${timeout}s"
            return 1
        fi
    done
}

# ---------------------------------------------------------------------------
# Phase 6: Create ACL and attach to containers
# ---------------------------------------------------------------------------

phase_create_acl() {
    step "Phase 6: Create monitoring ACL"

    if ! dry_run_guard "Create monitoring-allow ACL"; then
        return 0
    fi

    # Create the ACL if it does not exist
    if ! incus network acl show "$(container_ref monitoring-allow)" &>/dev/null 2>&1; then
        incus network acl create "$(container_ref monitoring-allow)"

        # Add allow-all egress and ingress rules
        incus network acl rule add "$(container_ref monitoring-allow)" egress \
            action=allow
        incus network acl rule add "$(container_ref monitoring-allow)" ingress \
            action=allow
        success "ACL 'monitoring-allow' created with allow-all rules"
    else
        success "ACL 'monitoring-allow' already exists"
    fi

    # Attach ACL to monitoring container
    incus config device set "$(container_ref monitoring)" eth0 \
        security.acls=monitoring-allow security.acls.default.ingress.action=allow \
        security.acls.default.egress.action=allow 2>/dev/null || true
    detail "ACL attached to monitoring"

    # Attach ACL to node-exporter containers
    local idx=1
    for ip in "${NODE_EXP_IPS[@]}"; do
        local name
        name=$(node_exp_name "$idx")
        if container_exists "$name"; then
            incus config device set "$(container_ref "$name")" eth0 \
                security.acls=monitoring-allow security.acls.default.ingress.action=allow \
                security.acls.default.egress.action=allow 2>/dev/null || true
            detail "ACL attached to ${name}"
        fi
        idx=$((idx + 1))
    done

    # Changing security.acls can bring the NIC down. Bring all NICs back up.
    sleep 2
    container_exec monitoring bash -c "ip link set eth0 up; systemctl restart systemd-networkd" 2>/dev/null || true
    local nic_idx=1
    for ip in "${NODE_EXP_IPS[@]}"; do
        local nname
        nname=$(node_exp_name "$nic_idx")
        if container_exists "$nname"; then
            container_exec "$nname" sh -c "ip link set eth0 up; rc-service networking restart" 2>/dev/null || true
        fi
        nic_idx=$((nic_idx + 1))
    done

    success "ACL attached to all observability containers"
}

# ---------------------------------------------------------------------------
# Phase 7: Configure HAProxy prometheus exporter
# ---------------------------------------------------------------------------

phase_configure_haproxy_exporter() {
    step "Phase 7: Configure HAProxy prometheus exporter + ACL rules"

    if ! dry_run_guard "Add prometheus exporter to HAProxy containers"; then
        return 0
    fi

    local haproxy_remote="${CLUSTER_REMOTE}"

    for n in 01 02; do
        local cname="ffsdn-haproxy-${HAPROXY_CLUSTER_ID}-${n}"

        if ! container_exists "$cname"; then
            warn "HAProxy container ${cname} not found -- skipping"
            continue
        fi

        # Add prometheus exporter line to the stats frontend in haproxy.cfg
        # Only add if not already present
        local has_exporter
        has_exporter=$(container_exec "$cname" grep -c 'prometheus-exporter' /etc/haproxy/haproxy.cfg 2>/dev/null || echo "0")
        if [[ "$has_exporter" == "0" ]]; then
            container_exec "$cname" bash -c "
                # Insert the prometheus exporter line into the stats section
                sed -i '/^[[:space:]]*stats enable/a\\    http-request use-service prometheus-exporter if { path /metrics }' /etc/haproxy/haproxy.cfg
                haproxy -c -f /etc/haproxy/haproxy.cfg 2>/dev/null && systemctl reload haproxy || true
            "
            success "  ${cname}: prometheus exporter route added"
        else
            success "  ${cname}: prometheus exporter already configured"
        fi

        # Add monitoring ingress rules to Aether ACL
        local acl_name="${HAPROXY_CLUSTER_ID}-${cname}-aether-acl"
        local has_monitoring_rule
        has_monitoring_rule=$(incus network acl show "${haproxy_remote}:${acl_name}" 2>/dev/null \
            | grep -c "monitoring-prometheus" || echo "0")

        if [[ "$has_monitoring_rule" == "0" ]]; then
            incus network acl rule add "${haproxy_remote}:${acl_name}" ingress \
                action=allow protocol=tcp source="${MONITORING_IP}/32" destination_port=8404 \
                description="monitoring-prometheus" 2>/dev/null || true
            incus network acl rule add "${haproxy_remote}:${acl_name}" ingress \
                action=allow protocol=icmp source="${MONITORING_IP}/32" \
                description="monitoring-icmp" 2>/dev/null || true
            success "  ${cname}: monitoring ACL rules added"
        else
            success "  ${cname}: monitoring ACL rules already present"
        fi
    done

    success "HAProxy prometheus exporter configured"
}

# ---------------------------------------------------------------------------
# Phase 8: Create OVN network forward
# ---------------------------------------------------------------------------

phase_create_network_forward() {
    step "Phase 8: Create network forward ${FORWARD_VIP}"

    if ! dry_run_guard "Create network forward ${FORWARD_VIP}"; then
        return 0
    fi

    # Check if forward already exists
    if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then
        success "Network forward ${FORWARD_VIP} already exists"
        return 0
    fi

    incus network forward create "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP"

    # Grafana port
    incus network forward port add "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" \
        tcp 3000 "${MONITORING_IP}" 3000

    # Prometheus port
    incus network forward port add "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" \
        tcp 9090 "${MONITORING_IP}" 9090

    success "Network forward created: ${FORWARD_VIP} -> Grafana(:3000), Prometheus(:9090)"
}

# ---------------------------------------------------------------------------
# Phase 9: Upload Grafana dashboards
# ---------------------------------------------------------------------------

phase_upload_dashboards() {
    step "Phase 9: Install Grafana dashboards"

    if ! dry_run_guard "Install dashboards via manage-dashboards"; then
        return 0
    fi

    local manage_script="${SCRIPT_DIR}/manage-dashboards"
    if [[ ! -x "$manage_script" ]]; then
        warn "manage-dashboards script not found at ${manage_script}"
        warn "Falling back to direct file upload."

        if [[ ! -d "$DASHBOARDS_DIR" ]]; then
            warn "Dashboard directory not found: ${DASHBOARDS_DIR}"
            return 0
        fi

        local count=0
        for dashboard in "${DASHBOARDS_DIR}"/core/*.json; do
            [[ -f "$dashboard" ]] || continue
            local basename
            basename=$(basename "$dashboard")
            incus file push "$dashboard" "$(container_ref monitoring)/var/lib/grafana/dashboards/${basename}"
            detail "Uploaded ${basename}"
            count=$((count + 1))
        done

        if [[ $count -gt 0 ]]; then
            container_exec monitoring systemctl restart grafana-server 2>/dev/null || true
            success "${count} dashboard(s) uploaded (direct push)"
        else
            info "No dashboard files found in ${DASHBOARDS_DIR}/core/"
        fi
        return 0
    fi

    # Wait for Grafana API to be ready
    info "Waiting for Grafana API..."
    local elapsed=0
    while true; do
        local code
        code=$(container_exec monitoring curl -s -o /dev/null -w '%{http_code}' \
            http://localhost:3000/api/health 2>/dev/null || echo "000")
        if [[ "$code" == "200" ]]; then
            break
        fi
        sleep 2
        elapsed=$((elapsed + 2))
        if [[ $elapsed -ge 30 ]]; then
            warn "Grafana API not ready after 30s -- falling back to file push"
            return 0
        fi
    done

    # Use manage-dashboards to install with proper folder organization
    local flags="-r ${CLUSTER_REMOTE}"
    [[ "$VERBOSE" == true ]] && flags="${flags} -v"
    [[ "$QUIET" == true ]] && flags="${flags} -q"

    "${manage_script}" --install ${flags}
}

# ---------------------------------------------------------------------------
# Workloads action
# ---------------------------------------------------------------------------

WORKLOAD_WEB_IP="10.10.10.80"
WORKLOAD_API_IP="10.10.10.81"

action_workloads() {
    step "Deploying dummy workloads for metric generation"
    echo
    info "  workload-web: ${WORKLOAD_WEB_IP} (nginx + CPU/disk/network crons)"
    info "  workload-api: ${WORKLOAD_API_IP} (Python HTTP + memory/network crons)"
    echo

    phase_deploy_workload_web
    phase_deploy_workload_api

    echo
    success "Dummy workloads deployed!"
    info "  Metrics will appear within 1-2 minutes"
    info "  workload-web: nginx on ${WORKLOAD_WEB_IP}:80, CPU/disk/net crons"
    info "  workload-api: Python on ${WORKLOAD_API_IP}:8080, memory sawtooth cron"
}

phase_deploy_workload_web() {
    step "Deploy workload-web (nginx + varied load)"

    if container_exists "workload-web"; then
        if container_running "workload-web"; then
            success "workload-web already running"
            return 0
        fi
    fi

    if ! dry_run_guard "Launch workload-web on ${OVN_NETWORK} at ${WORKLOAD_WEB_IP}"; then
        return 0
    fi

    incus launch images:alpine/3.20 "$(container_ref workload-web)" \
        -n "$OVN_NETWORK" \
        --target "oc-node-01" \
        -c limits.memory=128MiB

    incus config device set "$(container_ref workload-web)" eth0 \
        ipv4.address="${WORKLOAD_WEB_IP}"

    wait_for_container_agent workload-web 30

    container_exec workload-web sh -c "
        apk add --no-cache nginx curl gzip coreutils > /dev/null 2>&1

        # Configure nginx
        mkdir -p /var/www/html
        cat > /var/www/html/index.html << 'WEBHTML'
<html><body><h1>workload-web</h1><p>Observability test workload</p></body></html>
WEBHTML
        cat > /etc/nginx/http.d/default.conf << 'NGXCFG'
server {
    listen 80 default_server;
    root /var/www/html;
    index index.html;
    location / { try_files \$uri \$uri/ =404; }
}
NGXCFG
        rc-update add nginx default 2>/dev/null || true
        rc-service nginx start 2>/dev/null || true

        # Cron: every minute -> 10 HTTP requests (network traffic)
        # Cron: every 5 min -> gzip random data (CPU spike)
        # Cron: every 15 min -> dd 5 MiB write+delete (disk I/O burst)
        cat > /etc/crontabs/root << 'CRONCFG'
* * * * * for i in \$(seq 1 10); do wget -q -O /dev/null http://localhost/ 2>/dev/null; done
*/5 * * * * dd if=/dev/urandom bs=1M count=2 2>/dev/null | gzip > /dev/null 2>&1
*/15 * * * * dd if=/dev/urandom of=/tmp/disktest bs=1M count=5 2>/dev/null && rm -f /tmp/disktest
CRONCFG
        rc-update add crond default 2>/dev/null || true
        rc-service crond start 2>/dev/null || true
    "

    # Configure static IP
    container_exec workload-web sh -c "
        cat > /etc/network/interfaces << NETCFG
auto lo
iface lo inet loopback

auto eth0
iface eth0 inet static
    address ${WORKLOAD_WEB_IP}/24
    gateway 10.10.10.1
NETCFG
        rc-service networking restart 2>/dev/null || true
    "

    # Attach monitoring ACL
    incus config device set "$(container_ref workload-web)" eth0 \
        security.acls=monitoring-allow security.acls.default.ingress.action=allow \
        security.acls.default.egress.action=allow 2>/dev/null || true

    success "workload-web deployed at ${WORKLOAD_WEB_IP}"
}

phase_deploy_workload_api() {
    step "Deploy workload-api (Python HTTP + memory patterns)"

    if container_exists "workload-api"; then
        if container_running "workload-api"; then
            success "workload-api already running"
            return 0
        fi
    fi

    if ! dry_run_guard "Launch workload-api on ${OVN_NETWORK} at ${WORKLOAD_API_IP}"; then
        return 0
    fi

    incus launch images:alpine/3.20 "$(container_ref workload-api)" \
        -n "$OVN_NETWORK" \
        --target "oc-node-03" \
        -c limits.memory=128MiB

    incus config device set "$(container_ref workload-api)" eth0 \
        ipv4.address="${WORKLOAD_API_IP}"

    wait_for_container_agent workload-api 30

    container_exec workload-api sh -c "
        apk add --no-cache python3 curl coreutils > /dev/null 2>&1

        # Create a simple JSON status endpoint
        mkdir -p /opt/api
        cat > /opt/api/server.py << 'PYAPI'
import http.server
import json
import time
import os

class Handler(http.server.BaseHTTPRequestHandler):
    def do_GET(self):
        self.send_response(200)
        self.send_header('Content-Type', 'application/json')
        self.end_headers()
        data = {
            'status': 'ok',
            'uptime': time.time() - START_TIME,
            'hostname': os.uname().nodename,
            'pid': os.getpid()
        }
        self.wfile.write(json.dumps(data).encode())

    def log_message(self, format, *args):
        pass  # suppress access logs

START_TIME = time.time()
http.server.HTTPServer(('0.0.0.0', 8080), Handler).serve_forever()
PYAPI

        # Create memory stress script
        cat > /opt/api/mem-stress.py << 'PYMEM'
import time
# Allocate ~20 MiB, hold for 30 seconds, then release
data = bytearray(20 * 1024 * 1024)
time.sleep(30)
del data
PYMEM

        # Start the API server as a background service
        cat > /etc/init.d/api-server << 'INITSCRIPT'
#!/sbin/openrc-run
name=\"api-server\"
command=\"/usr/bin/python3\"
command_args=\"/opt/api/server.py\"
command_background=true
pidfile=\"/run/api-server.pid\"
INITSCRIPT
        chmod +x /etc/init.d/api-server
        rc-update add api-server default 2>/dev/null || true
        rc-service api-server start 2>/dev/null || true

        # Cron: every 2 min -> curl workload-web (inter-container traffic)
        # Cron: every 10 min -> allocate 20 MiB for 30s (memory sawtooth)
        cat > /etc/crontabs/root << 'CRONCFG'
*/2 * * * * wget -q -O /dev/null http://${WORKLOAD_WEB_IP}/ 2>/dev/null
*/10 * * * * python3 /opt/api/mem-stress.py 2>/dev/null &
CRONCFG
        rc-update add crond default 2>/dev/null || true
        rc-service crond start 2>/dev/null || true
    "

    # Configure static IP
    container_exec workload-api sh -c "
        cat > /etc/network/interfaces << NETCFG
auto lo
iface lo inet loopback

auto eth0
iface eth0 inet static
    address ${WORKLOAD_API_IP}/24
    gateway 10.10.10.1
NETCFG
        rc-service networking restart 2>/dev/null || true
    "

    # Attach monitoring ACL
    incus config device set "$(container_ref workload-api)" eth0 \
        security.acls=monitoring-allow security.acls.default.ingress.action=allow \
        security.acls.default.egress.action=allow 2>/dev/null || true

    success "workload-api deployed at ${WORKLOAD_API_IP}"
}

# ---------------------------------------------------------------------------
# Status action
# ---------------------------------------------------------------------------

action_status() {
    step "Observability Stack Status"
    echo

    # --- Container status ---
    info "Containers:"
    local all_ok=true

    for cname in monitoring; do
        if container_exists "$cname"; then
            if container_running "$cname"; then
                success "  ${cname}: running"
            else
                error "  ${cname}: stopped"
                all_ok=false
            fi
        else
            error "  ${cname}: not found"
            all_ok=false
        fi
    done

    local idx=1
    for ip in "${NODE_EXP_IPS[@]}"; do
        local name
        name=$(node_exp_name "$idx")
        if container_exists "$name"; then
            if container_running "$name"; then
                success "  ${name} (${ip}): running"
            else
                warn "  ${name} (${ip}): stopped"
            fi
        else
            warn "  ${name} (${ip}): not found"
        fi
        idx=$((idx + 1))
    done

    # OC server node-exporter (optional)
    if [[ -n "$OC_SERVER_IP" ]] && oc_server_remote_exists; then
        if oc_server_container_exists "node-exp-oc-server"; then
            if oc_server_container_running "node-exp-oc-server"; then
                success "  node-exp-oc-server (${OC_SERVER_IP}:9100): running"
            else
                warn "  node-exp-oc-server (${OC_SERVER_IP}:9100): stopped"
            fi
        else
            warn "  node-exp-oc-server: not found on oc-server"
        fi
    fi

    # Workload containers (optional)
    for wl in workload-web workload-api; do
        if container_exists "$wl"; then
            if container_running "$wl"; then
                success "  ${wl}: running"
            else
                warn "  ${wl}: stopped"
            fi
        fi
    done

    if [[ "$all_ok" == false ]]; then
        echo
        error "Some containers are missing or stopped."
        echo
    fi

    # --- Service status ---
    if container_exists monitoring && container_running monitoring; then
        echo
        info "Services:"
        for svc in prometheus grafana-server loki promtail; do
            local status
            status=$(container_exec monitoring systemctl is-active "$svc" 2>/dev/null || echo "inactive")
            if [[ "$status" == "active" ]]; then
                success "  ${svc}: active"
            else
                error "  ${svc}: ${status}"
            fi
        done

        # --- Prometheus targets ---
        echo
        info "Prometheus targets:"
        local targets_json
        targets_json=$(curl -s --connect-timeout 5 "http://${MONITORING_IP}:9090/api/v1/targets" 2>/dev/null || echo "")
        if [[ -n "$targets_json" ]]; then
            echo "$targets_json" | python3 -c "
import sys, json
try:
    data = json.load(sys.stdin)
    targets = data.get('data', {}).get('activeTargets', [])
    for t in targets:
        job = t.get('labels', {}).get('job', 'unknown')
        instance = t.get('labels', {}).get('instance', 'unknown')
        health = t.get('health', 'unknown')
        icon = 'UP' if health == 'up' else 'DOWN'
        print(f'  {job:20s} {instance:30s} {icon}')
except Exception as e:
    print(f'  (error parsing targets: {e})')
" 2>/dev/null || warn "  Could not parse Prometheus targets"
        else
            warn "  Prometheus not reachable at http://${MONITORING_IP}:9090"
        fi

        # --- Grafana health ---
        echo
        info "Grafana health:"
        local grafana_code
        grafana_code=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 5 \
            "http://${MONITORING_IP}:3000/api/health" 2>/dev/null || echo "000")
        if [[ "$grafana_code" == "200" ]]; then
            success "  Grafana API: healthy (HTTP ${grafana_code})"
        else
            error "  Grafana API: HTTP ${grafana_code}"
        fi

        # --- Loki labels ---
        echo
        info "Loki status:"
        local loki_labels
        loki_labels=$(curl -s --connect-timeout 5 "http://${MONITORING_IP}:3100/loki/api/v1/labels" 2>/dev/null || echo "")
        if [[ -n "$loki_labels" ]]; then
            local label_count
            label_count=$(echo "$loki_labels" | python3 -c "
import sys, json
try:
    data = json.load(sys.stdin)
    labels = data.get('data', [])
    print(len(labels))
except:
    print(0)
" 2>/dev/null || echo "0")
            if [[ "$label_count" -gt 0 ]]; then
                success "  Loki: responding (${label_count} label(s))"
            else
                info "  Loki: responding (no labels yet -- logs may not have been ingested)"
            fi
        else
            warn "  Loki not reachable at http://${MONITORING_IP}:3100"
        fi
    fi

    # --- Network forward ---
    echo
    info "Network forward:"
    if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then
        success "  ${FORWARD_VIP}: configured"
        local grafana_fwd
        grafana_fwd=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 5 \
            "http://${FORWARD_VIP}:3000/api/health" 2>/dev/null || echo "000")
        if [[ "$grafana_fwd" == "200" ]]; then
            success "  Grafana via forward: reachable (HTTP ${grafana_fwd})"
        else
            warn "  Grafana via forward: HTTP ${grafana_fwd}"
        fi
    else
        warn "  ${FORWARD_VIP}: not found"
    fi
}

# ---------------------------------------------------------------------------
# Cleanup action
# ---------------------------------------------------------------------------

action_cleanup() {
    step "Cleaning up observability stack"
    echo

    local has_anything=false

    if container_exists monitoring; then has_anything=true; fi
    local idx=1
    for ip in "${NODE_EXP_IPS[@]}"; do
        if container_exists "$(node_exp_name "$idx")"; then has_anything=true; fi
        idx=$((idx + 1))
    done
    for wl in workload-web workload-api; do
        if container_exists "$wl"; then has_anything=true; fi
    done
    if [[ -n "$OC_SERVER_IP" ]] && oc_server_remote_exists && oc_server_container_exists "node-exp-oc-server"; then
        has_anything=true
    fi
    if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then
        has_anything=true
    fi

    if [[ "$has_anything" == false ]]; then
        info "Nothing to clean up"
        return 0
    fi

    warn "This will delete:"
    if container_exists monitoring; then
        warn "  - monitoring container (Prometheus, Grafana, Loki data)"
    fi
    idx=1
    for ip in "${NODE_EXP_IPS[@]}"; do
        local name
        name=$(node_exp_name "$idx")
        if container_exists "$name"; then
            warn "  - ${name}"
        fi
        idx=$((idx + 1))
    done
    if [[ -n "$OC_SERVER_IP" ]] && oc_server_remote_exists && oc_server_container_exists "node-exp-oc-server"; then
        warn "  - node-exp-oc-server (on oc-server)"
    fi
    for wl in workload-web workload-api; do
        if container_exists "$wl"; then
            warn "  - ${wl}"
        fi
    done
    if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then
        warn "  - network forward ${FORWARD_VIP}"
    fi
    if incus network acl show "$(container_ref monitoring-allow)" &>/dev/null 2>&1; then
        warn "  - ACL monitoring-allow"
    fi
    echo

    if [[ "$DRY_RUN" == true ]]; then
        info "(dry-run) Would delete all observability infrastructure"
        return 0
    fi

    read -r -p "Type 'yes' to confirm: " confirm
    if [[ "$confirm" != "yes" ]]; then
        info "Aborted"
        return 0
    fi

    # Remove monitoring ACL rules from HAProxy ACLs
    step "Removing monitoring rules from HAProxy ACLs"
    for n in 01 02; do
        local cname="ffsdn-haproxy-${HAPROXY_CLUSTER_ID}-${n}"
        local acl_name="${HAPROXY_CLUSTER_ID}-${cname}-aether-acl"
        # Remove rules with monitoring descriptions
        local rule_ids
        rule_ids=$(incus network acl show "${CLUSTER_REMOTE}:${acl_name}" 2>/dev/null \
            | python3 -c "
import sys, yaml
try:
    data = yaml.safe_load(sys.stdin)
    for rule in data.get('ingress', []):
        desc = rule.get('description', '')
        if desc.startswith('monitoring-'):
            # ACL rules are removed by matching all fields, not by ID
            print(desc)
except: pass
" 2>/dev/null || true)
        if [[ -n "$rule_ids" ]]; then
            for desc in $rule_ids; do
                incus network acl rule remove "${CLUSTER_REMOTE}:${acl_name}" ingress \
                    description="$desc" 2>/dev/null || true
            done
            info "  Removed monitoring rules from ${acl_name}"
        fi
    done

    # Delete containers
    step "Deleting containers"
    if container_exists monitoring; then
        info "Deleting monitoring..."
        incus delete "$(container_ref monitoring)" --force 2>/dev/null || true
        success "Deleted monitoring"
    fi

    idx=1
    for ip in "${NODE_EXP_IPS[@]}"; do
        local name
        name=$(node_exp_name "$idx")
        if container_exists "$name"; then
            info "Deleting ${name}..."
            incus delete "$(container_ref "$name")" --force 2>/dev/null || true
            success "Deleted ${name}"
        fi
        idx=$((idx + 1))
    done

    # Delete oc-server node-exporter
    if [[ -n "$OC_SERVER_IP" ]] && oc_server_remote_exists && oc_server_container_exists "node-exp-oc-server"; then
        info "Deleting node-exp-oc-server on oc-server..."
        incus delete "$(oc_server_container_ref "node-exp-oc-server")" --force 2>/dev/null || true
        success "Deleted node-exp-oc-server"
    fi

    # Delete workload containers
    for wl in workload-web workload-api; do
        if container_exists "$wl"; then
            info "Deleting ${wl}..."
            incus delete "$(container_ref "$wl")" --force 2>/dev/null || true
            success "Deleted ${wl}"
        fi
    done

    # Delete network forward
    step "Removing network forward"
    if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then
        incus network forward delete "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP"
        success "Deleted network forward ${FORWARD_VIP}"
    fi

    # Delete ACL
    step "Removing ACL"
    if incus network acl show "$(container_ref monitoring-allow)" &>/dev/null 2>&1; then
        incus network acl delete "$(container_ref monitoring-allow)" 2>/dev/null || true
        success "Deleted ACL monitoring-allow"
    fi

    echo
    success "Cleanup complete"
}

# ---------------------------------------------------------------------------
# Doctor action
# ---------------------------------------------------------------------------

action_doctor() {
    step "Diagnosing observability stack"
    echo

    if ! container_exists monitoring || ! container_running monitoring; then
        error "Monitoring container is not running"
        error "Deploy with: ${SCRIPT_NAME} --deploy"
        return 1
    fi

    local issues=0

    # Check container networking
    info "Container networking:"
    if container_exec monitoring ping -c 1 -W 3 10.10.10.1 &>/dev/null; then
        success "  Gateway (10.10.10.1): reachable"
    else
        error "  Gateway (10.10.10.1): unreachable"
        issues=$((issues + 1))
    fi

    local idx=1
    for ip in "${NODE_EXP_IPS[@]}"; do
        local name
        name=$(node_exp_name "$idx")
        if container_exec monitoring ping -c 1 -W 3 "$ip" &>/dev/null; then
            success "  ${name} (${ip}): reachable"
        else
            warn "  ${name} (${ip}): unreachable"
            issues=$((issues + 1))
        fi
        idx=$((idx + 1))
    done

    for ip in "${HAPROXY_IPS[@]}"; do
        if container_exec monitoring ping -c 1 -W 3 "$ip" &>/dev/null; then
            success "  HAProxy (${ip}): reachable"
        else
            warn "  HAProxy (${ip}): unreachable"
            issues=$((issues + 1))
        fi
    done

    # Check Prometheus scrape errors
    echo
    info "Prometheus scrape health:"
    local targets_json
    targets_json=$(curl -s --connect-timeout 5 "http://${MONITORING_IP}:9090/api/v1/targets" 2>/dev/null || echo "")
    if [[ -n "$targets_json" ]]; then
        echo "$targets_json" | python3 -c "
import sys, json
try:
    data = json.load(sys.stdin)
    targets = data.get('data', {}).get('activeTargets', [])
    down = [t for t in targets if t.get('health') != 'up']
    if down:
        for t in down:
            job = t.get('labels', {}).get('job', '?')
            inst = t.get('labels', {}).get('instance', '?')
            err = t.get('lastError', 'unknown error')
            print(f'  DOWN: {job} {inst}')
            print(f'        {err}')
    else:
        print(f'  All {len(targets)} target(s) are UP')
except Exception as e:
    print(f'  (parse error: {e})')
" 2>/dev/null || warn "  Could not query Prometheus"
    else
        error "  Prometheus not reachable"
        issues=$((issues + 1))
    fi

    # Check systemd service failures
    echo
    info "Service health:"
    for svc in prometheus grafana-server loki promtail; do
        local status
        status=$(container_exec monitoring systemctl is-active "$svc" 2>/dev/null || echo "inactive")
        if [[ "$status" == "active" ]]; then
            success "  ${svc}: active"
        else
            error "  ${svc}: ${status}"
            # Show recent logs
            local logs
            logs=$(container_exec monitoring journalctl -u "$svc" --no-pager -n 5 --no-hostname 2>/dev/null || true)
            if [[ -n "$logs" ]]; then
                echo "$logs" | while IFS= read -r line; do
                    detail "    ${line}"
                done
            fi
            issues=$((issues + 1))
        fi
    done

    # Check disk space
    echo
    info "Disk space (monitoring container):"
    local disk_usage
    disk_usage=$(container_exec monitoring df -h / 2>/dev/null | tail -1 || echo "")
    if [[ -n "$disk_usage" ]]; then
        local use_pct
        use_pct=$(echo "$disk_usage" | awk '{print $5}' | tr -d '%')
        if [[ -n "$use_pct" ]] && [[ "$use_pct" -gt 85 ]]; then
            warn "  Disk usage: ${use_pct}% -- consider increasing disk size"
            issues=$((issues + 1))
        else
            success "  Disk usage: ${use_pct}%"
        fi
    else
        warn "  Could not check disk usage"
    fi

    echo
    if [[ $issues -eq 0 ]]; then
        success "No issues found"
    else
        warn "${issues} issue(s) found -- review the output above"
    fi
}

# ---------------------------------------------------------------------------
# Argument parsing
# ---------------------------------------------------------------------------

parse_args() {
    while [[ $# -gt 0 ]]; do
        case "$1" in
            --deploy)      ACTION="deploy";;
            --workloads)   ACTION="workloads";;
            --status)      ACTION="status";;
            --cleanup)     ACTION="cleanup";;
            --doctor)      ACTION="doctor";;
            -r|--remote)           CLUSTER_REMOTE="$2"; shift;;
            --nodes)               MANUAL_NODES="$2"; shift;;
            --monitoring-target)   MONITORING_TARGET="$2"; shift;;
            --monitoring-ip)       MONITORING_IP="$2"; shift;;
            --forward-vip)         FORWARD_VIP="$2"; shift;;
            --oc-server)           OC_SERVER_IP="$2"; shift;;
            --resume)              RESUME=true;;
            -n|--dry-run)  DRY_RUN=true;;
            -v|--verbose)  VERBOSE=true;;
            -q|--quiet)    QUIET=true;;
            -h|--help)     usage; exit 0;;
            *)
                error "Unknown option: $1"
                echo "Run '${SCRIPT_NAME} --help' for usage."
                exit 1
                ;;
        esac
        shift
    done

    if [[ -z "$ACTION" ]]; then
        error "No action specified"
        echo "Run '${SCRIPT_NAME} --help' for usage."
        exit 1
    fi
}

# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

main() {
    setup_colors
    parse_args "$@"

    # Discover cluster nodes (populates INCUS_NODES, NODE_EXP_TARGETS, NODE_EXP_IPS)
    discover_cluster_nodes

    echo
    case "$ACTION" in
        deploy)    action_deploy;;
        workloads) action_workloads;;
        status)    action_status;;
        cleanup)   action_cleanup;;
        doctor)    action_doctor;;
    esac
}

main "$@"
