Add cluster auto-discovery, oc-server monitoring, fix node-exporter RAM bug

Three changes to deploy-observability:

1. Fix node-exporter showing 128 MiB RAM: the conf.d template wrote
   NODE_EXPORTER_OPTS but Alpine's init script reads ARGS, so
   node-exporter started with no flags and read container /proc
   instead of host /proc. Changed to ARGS.

2. Auto-discover cluster nodes from `incus cluster list` instead of
   hardcoded arrays. Old values become fallback defaults for --dry-run.
   Added --nodes flag for manual override.

3. Add --oc-server flag to monitor the standalone OC management VM.
   Deploys node-exporter with host mounts and a proxy device to expose
   9100 on the host IP (oc-server uses incusbr0, not OVN). Adds to
   both Incus API and node-exporter Prometheus targets. Updated status
   and cleanup to handle the oc-server container.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

incusos/deploy-observability

@@ -26,18 +26,25 @@ MONITORING_IP="10.10.10.70"
 MONITORING_TARGET="oc-node-02"
 FORWARD_VIP="192.168.103.201"
 
-INCUS_NODES=("192.168.102.140" "192.168.102.141" "192.168.102.142")
+# Fallback defaults — used when cluster is unreachable (e.g., --dry-run)
+DEFAULT_INCUS_NODES=("192.168.102.140" "192.168.102.141" "192.168.102.142")
+DEFAULT_NODE_EXP_TARGETS=("oc-node-01" "oc-node-02" "oc-node-03")
+DEFAULT_NODE_EXP_IPS=("10.10.10.71" "10.10.10.72" "10.10.10.73")
+
+INCUS_NODES=()
+NODE_EXP_TARGETS=()
+NODE_EXP_IPS=()
 HAPROXY_IPS=("10.10.10.50" "10.10.10.51")
-NODE_EXP_IPS=("10.10.10.71" "10.10.10.72" "10.10.10.73")
-NODE_EXP_TARGETS=("oc-node-01" "oc-node-02" "oc-node-03")
 
 HAPROXY_CLUSTER_ID="52"
+OC_SERVER_IP=""
 
 # Runtime flags
 DRY_RUN=false
 VERBOSE=false
 QUIET=false
 ACTION=""
+MANUAL_NODES=""
 
 # ---------------------------------------------------------------------------
 # Color and formatting
@@ -95,6 +102,8 @@ ${BOLD}ACTIONS${RESET}
 
 ${BOLD}OPTIONS${RESET}
     -r, --remote NAME     Incus remote (default: ${CLUSTER_REMOTE})
+    --nodes NODE1,NODE2   Manual node list (skip auto-discovery)
+    --oc-server IP        Monitor oc-server at this IP (proxy device for 9100)
     -n, --dry-run         Preview actions without executing
     -v, --verbose         Show detailed output
     -q, --quiet           Suppress informational output
@@ -108,7 +117,7 @@ ${BOLD}COMPONENTS${RESET}
         Promtail              - log shipping agent
 
     Node-exporter containers (Alpine, 128 MiB, privileged):
-        node-exp-01/02/03 on cluster nodes, :9100
+        Auto-discovered from cluster (or --nodes override), :9100
 
     Network forward:
         ${FORWARD_VIP} -> Grafana (:3000), Prometheus (:9090)
@@ -120,9 +129,15 @@ ${BOLD}EXAMPLES${RESET}
     # Preview what would be deployed
     ${SCRIPT_NAME} --deploy --dry-run
 
-    # Full deployment
+    # Full deployment (auto-discovers cluster nodes)
     ${SCRIPT_NAME} --deploy
 
+    # Deploy with oc-server monitoring
+    ${SCRIPT_NAME} --deploy --oc-server 192.168.102.120
+
+    # Deploy with manual node list
+    ${SCRIPT_NAME} --deploy --nodes oc-node-01,oc-node-02,oc-node-03
+
     # Check health of all components
     ${SCRIPT_NAME} --status
 
@@ -174,6 +189,99 @@ node_exp_name() {
     printf "node-exp-%02d" "$idx"
 }
 
+# ---------------------------------------------------------------------------
+# Cluster auto-discovery
+# ---------------------------------------------------------------------------
+
+discover_cluster_nodes() {
+    # Manual override via --nodes takes priority
+    if [[ -n "$MANUAL_NODES" ]]; then
+        info "Using manually specified nodes: ${MANUAL_NODES}"
+        INCUS_NODES=()
+        NODE_EXP_TARGETS=()
+        NODE_EXP_IPS=()
+        local exp_octet=71
+        IFS=',' read -ra names <<< "$MANUAL_NODES"
+        for name in "${names[@]}"; do
+            # Look up address from Incus remote
+            local addr
+            addr=$(incus cluster list "${CLUSTER_REMOTE}:" --format csv 2>/dev/null \
+                | grep "^${name}," | head -1 | cut -d',' -f2 | sed 's|https://||;s|:.*||') || true
+            if [[ -z "$addr" ]]; then
+                warn "Could not resolve address for node '${name}' — skipping"
+                continue
+            fi
+            INCUS_NODES+=("$addr")
+            NODE_EXP_TARGETS+=("$name")
+            NODE_EXP_IPS+=("10.10.10.${exp_octet}")
+            exp_octet=$((exp_octet + 1))
+        done
+        return 0
+    fi
+
+    # Auto-discover from cluster
+    local csv
+    csv=$(incus cluster list "${CLUSTER_REMOTE}:" --format csv 2>/dev/null) || true
+
+    if [[ -z "$csv" ]]; then
+        if [[ "$DRY_RUN" == true ]]; then
+            info "Cluster not reachable — using fallback defaults for dry-run"
+            INCUS_NODES=("${DEFAULT_INCUS_NODES[@]}")
+            NODE_EXP_TARGETS=("${DEFAULT_NODE_EXP_TARGETS[@]}")
+            NODE_EXP_IPS=("${DEFAULT_NODE_EXP_IPS[@]}")
+            return 0
+        fi
+        error "Cannot reach cluster via remote '${CLUSTER_REMOTE}'"
+        error "Check that 'incus remote list' includes '${CLUSTER_REMOTE}'."
+        return 1
+    fi
+
+    INCUS_NODES=()
+    NODE_EXP_TARGETS=()
+    NODE_EXP_IPS=()
+
+    local exp_octet=71
+    while IFS=',' read -r name url roles _rest; do
+        # Extract IP from URL: https://192.168.102.140:8443 → 192.168.102.140
+        local addr="${url#https://}"
+        addr="${addr%%:*}"
+        INCUS_NODES+=("$addr")
+        NODE_EXP_TARGETS+=("$name")
+        NODE_EXP_IPS+=("10.10.10.${exp_octet}")
+        exp_octet=$((exp_octet + 1))
+    done <<< "$csv"
+
+    info "Discovered ${#INCUS_NODES[@]} cluster node(s): ${NODE_EXP_TARGETS[*]}"
+}
+
+# ---------------------------------------------------------------------------
+# OC server helpers
+# ---------------------------------------------------------------------------
+
+oc_server_remote_exists() {
+    incus remote list --format csv 2>/dev/null | grep -q "^oc-server,"
+}
+
+oc_server_container_ref() {
+    echo "oc-server:$1"
+}
+
+oc_server_container_exists() {
+    incus info "$(oc_server_container_ref "$1")" &>/dev/null 2>&1
+}
+
+oc_server_container_running() {
+    local state
+    state=$(incus info "$(oc_server_container_ref "$1")" 2>/dev/null | grep -i '^status:' | awk '{print $2}')
+    [[ "$state" == "RUNNING" || "$state" == "Running" ]]
+}
+
+oc_server_container_exec() {
+    local name="$1"
+    shift
+    incus exec "$(oc_server_container_ref "$name")" -- "$@"
+}
+
 # ---------------------------------------------------------------------------
 # Deploy action
 # ---------------------------------------------------------------------------
@@ -186,6 +294,7 @@ action_deploy() {
     info "  Incus nodes:    ${INCUS_NODES[*]}"
     info "  HAProxy stats:  ${HAPROXY_IPS[*]}"
     info "  Network fwd:    ${FORWARD_VIP} -> Grafana/Prometheus"
+    [[ -n "$OC_SERVER_IP" ]] && info "  OC server:      ${OC_SERVER_IP} (proxy :9100)"
     echo
 
     if container_exists "monitoring"; then
@@ -199,6 +308,7 @@ action_deploy() {
     phase_install_loki
     phase_install_grafana
     phase_deploy_node_exporters
+    phase_deploy_oc_server_exporter
     phase_create_acl
     phase_configure_haproxy_exporter
     phase_create_network_forward
@@ -328,6 +438,11 @@ phase_install_prometheus() {
         incus_targets+="'${node}:8443'"
         idx=$((idx + 1))
     done
+    # Add oc-server to Incus API scraping
+    if [[ -n "$OC_SERVER_IP" ]]; then
+        if [[ $idx -gt 0 ]]; then incus_targets+=", "; fi
+        incus_targets+="'${OC_SERVER_IP}:8443'"
+    fi
 
     local haproxy_targets=""
     idx=0
@@ -347,6 +462,13 @@ phase_install_prometheus() {
           nodename: '${target_name}'"
         idx=$((idx + 1))
     done
+    # Add oc-server node-exporter (via proxy device on host IP)
+    if [[ -n "$OC_SERVER_IP" ]]; then
+        node_exp_config+="
+      - targets: ['${OC_SERVER_IP}:9100']
+        labels:
+          nodename: 'oc-server'"
+    fi
 
     container_exec monitoring bash -c "cat > /etc/prometheus/prometheus.yml << 'PROMCFG'
 global:
@@ -596,7 +718,7 @@ phase_deploy_node_exporters() {
             # Configure to use host mount paths
             mkdir -p /etc/conf.d
             cat > /etc/conf.d/node-exporter << 'NEXPCFG'
-NODE_EXPORTER_OPTS=\"--path.procfs=/host/proc --path.sysfs=/host/sys --path.rootfs=/host/root\"
+ARGS=\"--path.procfs=/host/proc --path.sysfs=/host/sys --path.rootfs=/host/root\"
 NEXPCFG
 
             rc-update add node-exporter default 2>/dev/null || true
@@ -624,6 +746,86 @@ NETCFG
     success "Node-exporter containers deployed"
 }
 
+# ---------------------------------------------------------------------------
+# Phase 5b: Deploy node-exporter on oc-server (if --oc-server set)
+# ---------------------------------------------------------------------------
+
+phase_deploy_oc_server_exporter() {
+    [[ -z "$OC_SERVER_IP" ]] && return 0
+
+    step "Phase 5b: Deploy node-exporter on oc-server (${OC_SERVER_IP})"
+
+    # Validate oc-server remote exists
+    if ! oc_server_remote_exists; then
+        error "Incus remote 'oc-server' not found."
+        error "Add it with:"
+        error "  incus remote add oc-server https://${OC_SERVER_IP}:8443 --accept-certificate"
+        return 1
+    fi
+
+    local name="node-exp-oc-server"
+
+    if oc_server_container_exists "$name"; then
+        if oc_server_container_running "$name"; then
+            success "  ${name} already running on oc-server"
+            return 0
+        fi
+    fi
+
+    if ! dry_run_guard "Launch ${name} on oc-server at ${OC_SERVER_IP}"; then
+        return 0
+    fi
+
+    info "Launching ${name} on oc-server..."
+    incus launch images:alpine/3.20 "$(oc_server_container_ref "$name")" \
+        -c limits.memory=128MiB \
+        -c security.privileged=true
+
+    wait_for_container_agent_oc_server "$name" 30
+
+    # Mount host filesystems for full node metrics
+    incus config device add "$(oc_server_container_ref "$name")" host-proc disk \
+        source=/proc path=/host/proc readonly=true
+    incus config device add "$(oc_server_container_ref "$name")" host-sys disk \
+        source=/sys path=/host/sys readonly=true
+    incus config device add "$(oc_server_container_ref "$name")" host-root disk \
+        source=/ path=/host/root readonly=true
+
+    # Expose 9100 on the host IP via proxy device (oc-server is on incusbr0,
+    # not net-prod, so Prometheus reaches it via the host address)
+    incus config device add "$(oc_server_container_ref "$name")" proxy9100 proxy \
+        listen=tcp:${OC_SERVER_IP}:9100 connect=tcp:127.0.0.1:9100
+
+    # Install and configure node_exporter
+    oc_server_container_exec "$name" sh -c "
+        apk add --no-cache prometheus-node-exporter > /dev/null 2>&1
+
+        mkdir -p /etc/conf.d
+        cat > /etc/conf.d/node-exporter << 'NEXPCFG'
+ARGS=\"--path.procfs=/host/proc --path.sysfs=/host/sys --path.rootfs=/host/root\"
+NEXPCFG
+
+        rc-update add node-exporter default 2>/dev/null || true
+        rc-service node-exporter start 2>/dev/null || true
+    "
+
+    success "  ${name} deployed on oc-server with proxy ${OC_SERVER_IP}:9100"
+}
+
+wait_for_container_agent_oc_server() {
+    local name="$1"
+    local timeout="${2:-60}"
+    local elapsed=0
+    while ! oc_server_container_exec "$name" true &>/dev/null 2>&1; do
+        sleep 2
+        elapsed=$((elapsed + 2))
+        if [[ $elapsed -ge $timeout ]]; then
+            error "Container agent for '${name}' on oc-server not available after ${timeout}s"
+            return 1
+        fi
+    done
+}
+
 # ---------------------------------------------------------------------------
 # Phase 6: Create ACL and attach to containers
 # ---------------------------------------------------------------------------
@@ -1091,6 +1293,19 @@ action_status() {
         idx=$((idx + 1))
     done
 
+    # OC server node-exporter (optional)
+    if [[ -n "$OC_SERVER_IP" ]] && oc_server_remote_exists; then
+        if oc_server_container_exists "node-exp-oc-server"; then
+            if oc_server_container_running "node-exp-oc-server"; then
+                success "  node-exp-oc-server (${OC_SERVER_IP}:9100): running"
+            else
+                warn "  node-exp-oc-server (${OC_SERVER_IP}:9100): stopped"
+            fi
+        else
+            warn "  node-exp-oc-server: not found on oc-server"
+        fi
+    fi
+
     # Workload containers (optional)
     for wl in workload-web workload-api; do
         if container_exists "$wl"; then
@@ -1221,6 +1436,9 @@ action_cleanup() {
     for wl in workload-web workload-api; do
         if container_exists "$wl"; then has_anything=true; fi
     done
+    if [[ -n "$OC_SERVER_IP" ]] && oc_server_remote_exists && oc_server_container_exists "node-exp-oc-server"; then
+        has_anything=true
+    fi
     if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then
         has_anything=true
     fi
@@ -1243,6 +1461,9 @@ action_cleanup() {
         fi
         idx=$((idx + 1))
     done
+    if [[ -n "$OC_SERVER_IP" ]] && oc_server_remote_exists && oc_server_container_exists "node-exp-oc-server"; then
+        warn "  - node-exp-oc-server (on oc-server)"
+    fi
     for wl in workload-web workload-api; do
         if container_exists "$wl"; then
             warn "  - ${wl}"
@@ -1315,6 +1536,13 @@ except: pass
         idx=$((idx + 1))
     done
 
+    # Delete oc-server node-exporter
+    if [[ -n "$OC_SERVER_IP" ]] && oc_server_remote_exists && oc_server_container_exists "node-exp-oc-server"; then
+        info "Deleting node-exp-oc-server on oc-server..."
+        incus delete "$(oc_server_container_ref "node-exp-oc-server")" --force 2>/dev/null || true
+        success "Deleted node-exp-oc-server"
+    fi
+
     # Delete workload containers
     for wl in workload-web workload-api; do
         if container_exists "$wl"; then
@@ -1479,6 +1707,8 @@ parse_args() {
             --cleanup)     ACTION="cleanup";;
             --doctor)      ACTION="doctor";;
             -r|--remote)   CLUSTER_REMOTE="$2"; shift;;
+            --nodes)       MANUAL_NODES="$2"; shift;;
+            --oc-server)   OC_SERVER_IP="$2"; shift;;
             -n|--dry-run)  DRY_RUN=true;;
             -v|--verbose)  VERBOSE=true;;
             -q|--quiet)    QUIET=true;;
@@ -1507,6 +1737,9 @@ main() {
     setup_colors
     parse_args "$@"
 
+    # Discover cluster nodes (populates INCUS_NODES, NODE_EXP_TARGETS, NODE_EXP_IPS)
+    discover_cluster_nodes
+
     echo
     case "$ACTION" in
         deploy)    action_deploy;;